diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy index ba7311fee6f..c7f4316ee04 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy @@ -131,8 +131,9 @@ public class RestTestsFromSnippetsTask extends SnippetsTask { } private void response(Snippet response) { - current.println(" - response_body: |") - response.contents.eachLine { current.println(" $it") } + current.println(" - match: ") + current.println(" \$body: ") + response.contents.eachLine { current.println(" $it") } } void emitDo(String method, String pathAndQuery, diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml index de47736913f..fe726062706 100644 --- a/buildSrc/src/main/resources/checkstyle.xml +++ b/buildSrc/src/main/resources/checkstyle.xml @@ -39,6 +39,25 @@ + + + waiting for 3 nodes to be up"); - assertBusy(new Runnable() { - @Override - public void run() { - NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get(); - assertThat(resp.getNodes().size(), equalTo(3)); - } + assertBusy(() -> { + NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get(); + assertThat(resp.getNodes().size(), equalTo(3)); }); logger.info("--> creating 'test' index"); @@ -126,7 +123,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase { Map explanations = cae.getNodeExplanations(); - Float noAttrWeight = -1f; Float barAttrWeight = -1f; Float fooBarAttrWeight = -1f; for (Map.Entry entry : explanations.entrySet()) { @@ -134,7 +130,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase { String nodeName = node.getName(); NodeExplanation explanation = entry.getValue(); ClusterAllocationExplanation.FinalDecision finalDecision = explanation.getFinalDecision(); - String finalExplanation = explanation.getFinalExplanation(); ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy(); Decision d = explanation.getDecision(); float weight = explanation.getWeight(); @@ -143,7 +138,6 @@ public final class ClusterAllocationExplainIT extends ESIntegTestCase { assertEquals(d.type(), Decision.Type.NO); if (noAttrNode.equals(nodeName)) { assertThat(d.toString(), containsString("node does not match index include filters [foo:\"bar\"]")); - noAttrWeight = weight; assertNull(storeStatus); assertEquals("the shard cannot be assigned because one or more allocation decider returns a 'NO' decision", explanation.getFinalExplanation()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java index d5cefc6d1f3..6c23d1604b8 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplainTests.java @@ -47,7 +47,6 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase { NodeExplanation explanation = cae.getNodeExplanations().values().iterator().next(); ClusterAllocationExplanation.FinalDecision fd = explanation.getFinalDecision(); ClusterAllocationExplanation.StoreCopy storeCopy = explanation.getStoreCopy(); - String finalExplanation = explanation.getFinalExplanation(); Decision d = explanation.getDecision(); assertNotNull("should have a decision", d); assertEquals(Decision.Type.NO, d.type()); @@ -76,7 +75,6 @@ public final class ClusterAllocationExplainTests extends ESSingleNodeTestCase { d = explanation.getDecision(); fd = explanation.getFinalDecision(); storeCopy = explanation.getStoreCopy(); - finalExplanation = explanation.getFinalExplanation(); assertNotNull("should have a decision", d); assertEquals(Decision.Type.NO, d.type()); assertEquals(ClusterAllocationExplanation.FinalDecision.ALREADY_ASSIGNED, fd); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java index d0e8ef14d01..1561ab7a77c 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -205,7 +204,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase { "assignedNode", allocationDelay, remainingDelay, null, false, nodeExplanations); BytesStreamOutput out = new BytesStreamOutput(); cae.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); ClusterAllocationExplanation cae2 = new ClusterAllocationExplanation(in); assertEquals(shard, cae2.getShard()); assertTrue(cae2.isPrimary()); @@ -215,9 +214,7 @@ public final class ClusterAllocationExplanationTests extends ESTestCase { assertEquals(allocationDelay, cae2.getAllocationDelayMillis()); assertEquals(remainingDelay, cae2.getRemainingDelayMillis()); for (Map.Entry entry : cae2.getNodeExplanations().entrySet()) { - DiscoveryNode node = entry.getKey(); NodeExplanation explanation = entry.getValue(); - IndicesShardStoresResponse.StoreStatus status = explanation.getStoreStatus(); assertNotNull(explanation.getStoreStatus()); assertNotNull(explanation.getDecision()); assertEquals(nodeWeight, explanation.getWeight()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java index 704c1348b7e..d0d452df478 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthResponsesTests.java @@ -84,7 +84,7 @@ public class ClusterHealthResponsesTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); clusterHealth.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); clusterHealth = ClusterHealthResponse.readResponseFrom(in); } return clusterHealth; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index c4d49d899b9..56756ad9fdb 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -249,7 +249,7 @@ public class TransportTasksActionTests extends TaskManagerTestCase { /** * Test class for testing task operations */ - static abstract class TestTasksAction extends TransportTasksAction { + abstract static class TestTasksAction extends TransportTasksAction { protected TestTasksAction(Settings settings, String actionName, ThreadPool threadPool, ClusterService clusterService, TransportService transportService) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java index b736751b781..4f553dfb88a 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteRequestTests.java @@ -166,7 +166,7 @@ public class ClusterRerouteRequestTests extends ESTestCase { private ClusterRerouteRequest roundTripThroughBytes(ClusterRerouteRequest original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { ClusterRerouteRequest copy = new ClusterRerouteRequest(); copy.readFrom(in); return copy; diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java index 00fcbf60a5a..dac878eefec 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteTests.java @@ -66,7 +66,7 @@ public class ClusterRerouteTests extends ESAllocationTestCase { BytesReference bytes = out.bytes(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); new NetworkModule(null, Settings.EMPTY, true, namedWriteableRegistry); - StreamInput wrap = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes.toBytes()), + StreamInput wrap = new NamedWriteableAwareStreamInput(bytes.streamInput(), namedWriteableRegistry); ClusterRerouteRequest deserializedReq = new ClusterRerouteRequest(); deserializedReq.readFrom(wrap); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java index fc04de81254..b515829b72a 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestTests.java @@ -45,7 +45,7 @@ public class ClusterStateRequestTests extends ESTestCase { output.setVersion(testVersion); clusterStateRequest.writeTo(output); - StreamInput streamInput = StreamInput.wrap(output.bytes()); + StreamInput streamInput = output.bytes().streamInput(); streamInput.setVersion(testVersion); ClusterStateRequest deserializedCSRequest = new ClusterStateRequest(); deserializedCSRequest.readFrom(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java index e4c2849b907..2e9239a2c3b 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/GetStoredScriptRequestTests.java @@ -37,7 +37,7 @@ public class GetStoredScriptRequestTests extends ESTestCase { out.setVersion(randomVersion(random())); request.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(out.getVersion()); GetStoredScriptRequest request2 = new GetStoredScriptRequest(); request2.readFrom(in); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java index 04f6037f64b..7040c92ec1d 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/flush/SyncedFlushUnitTests.java @@ -84,7 +84,7 @@ public class SyncedFlushUnitTests extends ESTestCase { assertThat(testPlan.result.restStatus(), equalTo(testPlan.totalCounts.failed > 0 ? RestStatus.CONFLICT : RestStatus.OK)); BytesStreamOutput out = new BytesStreamOutput(); testPlan.result.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); SyncedFlushResponse readResponse = new SyncedFlushResponse(); readResponse.readFrom(in); assertThat(readResponse.totalShards(), equalTo(testPlan.totalCounts.total)); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java index 6e3e5d76224..1cd1704e164 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shards/IndicesShardStoreRequestIT.java @@ -215,7 +215,7 @@ public class IndicesShardStoreRequestIT extends ESIntegTestCase { client().admin().indices().prepareFlush().setForce(true).setWaitIfOngoing(true).execute().actionGet(); } - private final static class IndexNodePredicate implements Predicate { + private static final class IndexNodePredicate implements Predicate { private final Set nodesWithShard; public IndexNodePredicate(String index) { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java index 726dccee597..dfc10169e70 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.stats; +import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.engine.CommitStats; @@ -26,6 +27,8 @@ import org.elasticsearch.index.engine.SegmentsStats; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.util.List; + import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasKey; @@ -108,4 +111,12 @@ public class IndicesStatsTests extends ESSingleNodeTestCase { } } + /** + * Gives access to package private IndicesStatsResponse constructor for test purpose. + **/ + public static IndicesStatsResponse newIndicesStatsResponse(ShardStats[] shards, int totalShards, int successfulShards, + int failedShards, List shardFailures) { + return new IndicesStatsResponse(shards, totalShards, successfulShards, failedShards, shardFailures); + } + } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index 337f881d41b..142fb282c20 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -55,9 +55,9 @@ public class BulkRequestTests extends ESTestCase { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, null); assertThat(bulkRequest.numberOfActions(), equalTo(3)); - assertThat(((IndexRequest) bulkRequest.requests().get(0)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }").toBytes())); + assertThat(((IndexRequest) bulkRequest.requests().get(0)).source(), equalTo(new BytesArray("{ \"field1\" : \"value1\" }"))); assertThat(bulkRequest.requests().get(1), instanceOf(DeleteRequest.class)); - assertThat(((IndexRequest) bulkRequest.requests().get(2)).source().toBytes(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }").toBytes())); + assertThat(((IndexRequest) bulkRequest.requests().get(2)).source(), equalTo(new BytesArray("{ \"field1\" : \"value3\" }"))); } public void testSimpleBulk2() throws Exception { @@ -81,7 +81,7 @@ public class BulkRequestTests extends ESTestCase { assertThat(bulkRequest.numberOfActions(), equalTo(4)); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).id(), equalTo("1")); assertThat(((UpdateRequest) bulkRequest.requests().get(0)).retryOnConflict(), equalTo(2)); - assertThat(((UpdateRequest) bulkRequest.requests().get(0)).doc().source().toUtf8(), equalTo("{\"field\":\"value\"}")); + assertThat(((UpdateRequest) bulkRequest.requests().get(0)).doc().source().utf8ToString(), equalTo("{\"field\":\"value\"}")); assertThat(((UpdateRequest) bulkRequest.requests().get(1)).id(), equalTo("0")); assertThat(((UpdateRequest) bulkRequest.requests().get(1)).type(), equalTo("type1")); assertThat(((UpdateRequest) bulkRequest.requests().get(1)).index(), equalTo("index1")); @@ -93,7 +93,7 @@ public class BulkRequestTests extends ESTestCase { assertThat(scriptParams, notNullValue()); assertThat(scriptParams.size(), equalTo(1)); assertThat(((Integer) scriptParams.get("param1")), equalTo(1)); - assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().toUtf8(), equalTo("{\"counter\":1}")); + assertThat(((UpdateRequest) bulkRequest.requests().get(1)).upsertRequest().source().utf8ToString(), equalTo("{\"counter\":1}")); } public void testBulkAllowExplicitIndex() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java index b26d2531ff0..bb406366d25 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java @@ -29,11 +29,11 @@ public class BulkShardRequestTests extends ESTestCase { public void testToString() { String index = randomSimpleString(random(), 10); int count = between(1, 100); - BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]); + BulkShardRequest r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests", r.toString()); - r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]); + r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests and a refresh", r.toString()); - r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]); + r = new BulkShardRequest(new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests blocking until refresh", r.toString()); } } diff --git a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 6ae7559ba62..6111c4c9953 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -59,7 +59,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; public class TransportBulkActionTookTests extends ESTestCase { - static private ThreadPool threadPool; + private static ThreadPool threadPool; private ClusterService clusterService; @BeforeClass diff --git a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java index 451ade62584..ef259463139 100644 --- a/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/get/MultiGetShardRequestTests.java @@ -70,7 +70,7 @@ public class MultiGetShardRequestTests extends ESTestCase { out.setVersion(randomVersion(random())); multiGetShardRequest.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(out.getVersion()); MultiGetShardRequest multiGetShardRequest2 = new MultiGetShardRequest(); multiGetShardRequest2.readFrom(in); diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java index 323a8c0aaa6..544e2932b44 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateDocumentSimpleResultTests.java @@ -45,7 +45,7 @@ public class SimulateDocumentSimpleResultTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); simulateDocumentBaseResult.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); SimulateDocumentBaseResult otherSimulateDocumentBaseResult = new SimulateDocumentBaseResult(streamInput); if (isFailure) { diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java index 1376ca4280e..576e8e01724 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineResponseTests.java @@ -73,7 +73,7 @@ public class SimulatePipelineResponseTests extends ESTestCase { SimulatePipelineResponse response = new SimulatePipelineResponse(randomAsciiOfLengthBetween(1, 10), isVerbose, results); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); SimulatePipelineResponse otherResponse = new SimulatePipelineResponse(); otherResponse.readFrom(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java index f612f36c9d6..ccf3a674944 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/SimulateProcessorResultTests.java @@ -48,7 +48,7 @@ public class SimulateProcessorResultTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); simulateProcessorResult.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); SimulateProcessorResult otherSimulateProcessorResult = new SimulateProcessorResult(streamInput); assertThat(otherSimulateProcessorResult.getProcessorTag(), equalTo(simulateProcessorResult.getProcessorTag())); if (isFailure) { diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java index 3f252c37072..00327603ba8 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/WritePipelineResponseTests.java @@ -35,7 +35,7 @@ public class WritePipelineResponseTests extends ESTestCase { response = new WritePipelineResponse(isAcknowledged); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); WritePipelineResponse otherResponse = new WritePipelineResponse(); otherResponse.readFrom(streamInput); @@ -46,7 +46,7 @@ public class WritePipelineResponseTests extends ESTestCase { WritePipelineResponse response = new WritePipelineResponse(); BytesStreamOutput out = new BytesStreamOutput(); response.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); WritePipelineResponse otherResponse = new WritePipelineResponse(); otherResponse.readFrom(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java index a7ce842913d..b4908846e97 100644 --- a/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java +++ b/core/src/test/java/org/elasticsearch/action/ingest/WriteableIngestDocumentTests.java @@ -112,7 +112,7 @@ public class WriteableIngestDocumentTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); writeableIngestDocument.writeTo(out); - StreamInput streamInput = StreamInput.wrap(out.bytes()); + StreamInput streamInput = out.bytes().streamInput(); WriteableIngestDocument otherWriteableIngestDocument = new WriteableIngestDocument(streamInput); assertIngestDocument(otherWriteableIngestDocument.getIngestDocument(), writeableIngestDocument.getIngestDocument()); } diff --git a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java index 2bff71d3c40..3e592d1c341 100644 --- a/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/main/MainActionTests.java @@ -64,7 +64,7 @@ public class MainActionTests extends ESTestCase { BytesStreamOutput streamOutput = new BytesStreamOutput(); mainResponse.writeTo(streamOutput); final MainResponse serialized = new MainResponse(); - serialized.readFrom(new ByteBufferStreamInput(ByteBuffer.wrap(streamOutput.bytes().toBytes()))); + serialized.readFrom(streamOutput.bytes().streamInput()); assertThat(serialized.getNodeName(), equalTo(nodeName)); assertThat(serialized.getClusterName(), equalTo(clusterName)); diff --git a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java index d5ed5302b97..d656e0f62a9 100644 --- a/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java @@ -38,7 +38,7 @@ public class IndicesOptionsTests extends ESTestCase { output.setVersion(outputVersion); indicesOptions.writeIndicesOptions(output); - StreamInput streamInput = StreamInput.wrap(output.bytes()); + StreamInput streamInput = output.bytes().streamInput(); streamInput.setVersion(randomVersion(random())); IndicesOptions indicesOptions2 = IndicesOptions.readIndicesOptions(streamInput); diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java index dc40fda3f8e..d0e26c22cfb 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ClusterStateCreationUtils.java @@ -220,7 +220,6 @@ public class ClusterStateCreationUtils { * Creates a cluster state with no index */ public static ClusterState stateWithNoShard() { - int numberOfNodes = 2; DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder(); discoBuilder.localNodeId(newNode(0).getId()); discoBuilder.masterNodeId(newNode(1).getId()); @@ -260,7 +259,7 @@ public class ClusterStateCreationUtils { new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); } - static private String selectAndRemove(Set strings) { + private static String selectAndRemove(Set strings) { String selection = randomFrom(strings.toArray(new String[strings.size()])); strings.remove(selection); return selection; diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java index 208945a6179..d9f351120b2 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/AbstractTermVectorsTestCase.java @@ -68,10 +68,10 @@ import static org.hamcrest.Matchers.equalTo; public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase { protected static class TestFieldSetting { - final public String name; - final public boolean storedOffset; - final public boolean storedPayloads; - final public boolean storedPositions; + public final String name; + public final boolean storedOffset; + public final boolean storedPayloads; + public final boolean storedPositions; public TestFieldSetting(String name, boolean storedOffset, boolean storedPayloads, boolean storedPositions) { this.name = name; @@ -124,9 +124,9 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase { } protected static class TestDoc { - final public String id; - final public TestFieldSetting[] fieldSettings; - final public String[] fieldContent; + public final String id; + public final TestFieldSetting[] fieldSettings; + public final String[] fieldContent; public String index = "test"; public String alias = "alias"; public String type = "type1"; @@ -163,11 +163,11 @@ public abstract class AbstractTermVectorsTestCase extends ESIntegTestCase { } protected static class TestConfig { - final public TestDoc doc; - final public String[] selectedFields; - final public boolean requestPositions; - final public boolean requestOffsets; - final public boolean requestPayloads; + public final TestDoc doc; + public final String[] selectedFields; + public final boolean requestPositions; + public final boolean requestOffsets; + public final boolean requestPayloads; public Class expectedException = null; public TestConfig(TestDoc doc, String[] selectedFields, boolean requestPositions, boolean requestOffsets, boolean requestPayloads) { diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java index 37a1bc92e9c..1611c63d2ba 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsCheckDocFreqIT.java @@ -140,7 +140,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -196,7 +196,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, null); xBuilder.endObject(); - String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; @@ -255,7 +255,7 @@ public class GetTermVectorsCheckDocFreqIT extends ESIntegTestCase { xBuilder.startObject(); response.toXContent(xBuilder, ToXContent.EMPTY_PARAMS); xBuilder.endObject(); - String utf8 = xBuilder.bytes().toUtf8().replaceFirst("\"took\":\\d+,", "");; + String utf8 = xBuilder.bytes().utf8ToString().replaceFirst("\"took\":\\d+,", "");; String expectedString = "{\"_index\":\"test\",\"_type\":\"type1\",\"_id\":\"" + i + "\",\"_version\":1,\"found\":true,\"term_vectors\":{\"field\":{\"field_statistics\":{\"sum_doc_freq\":120,\"doc_count\":15,\"sum_ttf\":135},\"terms\":{\"brown\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":2,\"start_offset\":10,\"end_offset\":15,\"payload\":\"d29yZA==\"}]},\"dog\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":8,\"start_offset\":40,\"end_offset\":43,\"payload\":\"d29yZA==\"}]},\"fox\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":3,\"start_offset\":16,\"end_offset\":19,\"payload\":\"d29yZA==\"}]},\"jumps\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":4,\"start_offset\":20,\"end_offset\":25,\"payload\":\"d29yZA==\"}]},\"lazy\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":7,\"start_offset\":35,\"end_offset\":39,\"payload\":\"d29yZA==\"}]},\"over\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":5,\"start_offset\":26,\"end_offset\":30,\"payload\":\"d29yZA==\"}]},\"quick\":{\"doc_freq\":15,\"ttf\":15,\"term_freq\":1,\"tokens\":[{\"position\":1,\"start_offset\":4,\"end_offset\":9,\"payload\":\"d29yZA==\"}]},\"the\":{\"doc_freq\":15,\"ttf\":30,\"term_freq\":2,\"tokens\":[{\"position\":0,\"start_offset\":0,\"end_offset\":3,\"payload\":\"d29yZA==\"},{\"position\":6,\"start_offset\":31,\"end_offset\":34,\"payload\":\"d29yZA==\"}]}}}}}"; diff --git a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java index 12af9f8a2c2..6ba31fdb88e 100644 --- a/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java +++ b/core/src/test/java/org/elasticsearch/action/termvectors/GetTermVectorsIT.java @@ -39,7 +39,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.FieldMapper; -import org.hamcrest.Matcher; import java.io.IOException; import java.util.ArrayList; @@ -55,7 +54,6 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -963,21 +961,6 @@ public class GetTermVectorsIT extends AbstractTermVectorsTestCase { return randomBoolean() ? "test" : "alias"; } - private Map getFieldStatistics(Map stats, String fieldName) throws IOException { - return (Map) ((Map) stats.get(fieldName)).get("field_statistics"); - } - - private Map getTermStatistics(Map stats, String fieldName, String term) { - return (Map) ((Map) ((Map) stats.get(fieldName)).get("terms")).get(term); - } - - private Matcher equalOrLessThanTo(Integer value, boolean isEqual) { - if (isEqual) { - return equalTo(value); - } - return lessThan(value); - } - public void testTermVectorsWithVersion() { assertAcked(prepareCreate("test").addAlias(new Alias("alias")) .setSettings(Settings.builder().put("index.refresh_interval", -1))); diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index 597a2a4db39..d105a4bf63b 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -135,7 +135,7 @@ public class UpdateRequestTests extends ESTestCase { TimeValue providedTTLValue = TimeValue.parseTimeValue(randomTimeValue(), null, "ttl"); Settings settings = settings(Version.CURRENT).build(); - UpdateHelper updateHelper = new UpdateHelper(settings, null, null); + UpdateHelper updateHelper = new UpdateHelper(settings, null); // We just upsert one document with ttl IndexRequest indexRequest = new IndexRequest("test", "type1", "1") diff --git a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java index ffe82f9388d..699b919cf05 100644 --- a/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java +++ b/core/src/test/java/org/elasticsearch/blocks/SimpleBlocksIT.java @@ -141,15 +141,6 @@ public class SimpleBlocksIT extends ESIntegTestCase { } } - private void canNotIndexExists(String index) { - try { - IndicesExistsResponse r = client().admin().indices().prepareExists(index).execute().actionGet(); - fail(); - } catch (ClusterBlockException e) { - // all is well - } - } - private void setIndexReadOnly(String index, Object value) { HashMap newSettings = new HashMap<>(); newSettings.put(IndexMetaData.SETTING_READ_ONLY, value); diff --git a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java index 40995ff778b..be0848ed4dc 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/BasicAnalysisBackwardCompatibilityIT.java @@ -41,7 +41,7 @@ import static org.hamcrest.Matchers.equalTo; public class BasicAnalysisBackwardCompatibilityIT extends ESBackcompatTestCase { // This pattern match characters with Line_Break = Complex_Content. - final static Pattern complexUnicodeChars = Pattern.compile("[\u17B4\u17B5\u17D3\u17CB-\u17D1\u17DD\u1036\u17C6\u1A74\u1038\u17C7\u0E4E\u0E47-\u0E4D\u0EC8-\u0ECD\uAABF\uAAC1\u1037\u17C8-\u17CA\u1A75-\u1A7C\u1AA8-\u1AAB\uAADE\uAADF\u1AA0-\u1AA6\u1AAC\u1AAD\u109E\u109F\uAA77-\uAA79\u0E46\u0EC6\u17D7\u1AA7\uA9E6\uAA70\uAADD\u19DA\u0E01-\u0E3A\u0E40-\u0E45\u0EDE\u0E81\u0E82\u0E84\u0E87\u0E88\u0EAA\u0E8A\u0EDF\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAB\u0EDC\u0EDD\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\uAA80-\uAABE\uAAC0\uAAC2\uAADB\uAADC\u1000\u1075\u1001\u1076\u1002\u1077\uAA60\uA9E9\u1003\uA9E0\uA9EA\u1004\u105A\u1005\u1078\uAA61\u1006\uA9E1\uAA62\uAA7E\u1007\uAA63\uA9EB\u1079\uAA72\u1008\u105B\uA9E2\uAA64\uA9EC\u1061\uAA7F\u1009\u107A\uAA65\uA9E7\u100A\u100B\uAA66\u100C\uAA67\u100D\uAA68\uA9ED\u100E\uAA69\uA9EE\u100F\u106E\uA9E3\uA9EF\u1010-\u1012\u107B\uA9FB\u1013\uAA6A\uA9FC\u1014\u107C\uAA6B\u105E\u1015\u1016\u107D\u107E\uAA6F\u108E\uA9E8\u1017\u107F\uA9FD\u1018\uA9E4\uA9FE\u1019\u105F\u101A\u103B\u101B\uAA73\uAA7A\u103C\u101C\u1060\u101D\u103D\u1082\u1080\u1050\u1051\u1065\u101E\u103F\uAA6C\u101F\u1081\uAA6D\u103E\uAA6E\uAA71\u1020\uA9FA\u105C\u105D\u106F\u1070\u1066\u1021-\u1026\u1052-\u1055\u1027-\u102A\u102C\u102B\u1083\u1072\u109C\u102D\u1071\u102E\u1033\u102F\u1073\u1074\u1030\u1056-\u1059\u1031\u1084\u1035\u1085\u1032\u109D\u1034\u1062\u1067\u1068\uA9E5\u1086\u1039\u103A\u1063\u1064\u1069-\u106D\u1087\u108B\u1088\u108C\u108D\u1089\u108A\u108F\u109A\u109B\uAA7B-\uAA7D\uAA74-\uAA76\u1780-\u17A2\u17DC\u17A3-\u17B3\u17B6-\u17C5\u17D2\u1950-\u196D\u1970-\u1974\u1980-\u199C\u19DE\u19DF\u199D-\u19AB\u19B0-\u19C9\u1A20-\u1A26\u1A58\u1A59\u1A27-\u1A3B\u1A5A\u1A5B\u1A3C-\u1A46\u1A54\u1A47-\u1A4C\u1A53\u1A6B\u1A55-\u1A57\u1A5C-\u1A5E\u1A4D-\u1A52\u1A61\u1A6C\u1A62-\u1A6A\u1A6E\u1A6F\u1A73\u1A70-\u1A72\u1A6D\u1A60]"); + static final Pattern complexUnicodeChars = Pattern.compile("[\u17B4\u17B5\u17D3\u17CB-\u17D1\u17DD\u1036\u17C6\u1A74\u1038\u17C7\u0E4E\u0E47-\u0E4D\u0EC8-\u0ECD\uAABF\uAAC1\u1037\u17C8-\u17CA\u1A75-\u1A7C\u1AA8-\u1AAB\uAADE\uAADF\u1AA0-\u1AA6\u1AAC\u1AAD\u109E\u109F\uAA77-\uAA79\u0E46\u0EC6\u17D7\u1AA7\uA9E6\uAA70\uAADD\u19DA\u0E01-\u0E3A\u0E40-\u0E45\u0EDE\u0E81\u0E82\u0E84\u0E87\u0E88\u0EAA\u0E8A\u0EDF\u0E8D\u0E94-\u0E97\u0E99-\u0E9F\u0EA1-\u0EA3\u0EA5\u0EA7\u0EAB\u0EDC\u0EDD\u0EAD-\u0EB9\u0EBB-\u0EBD\u0EC0-\u0EC4\uAA80-\uAABE\uAAC0\uAAC2\uAADB\uAADC\u1000\u1075\u1001\u1076\u1002\u1077\uAA60\uA9E9\u1003\uA9E0\uA9EA\u1004\u105A\u1005\u1078\uAA61\u1006\uA9E1\uAA62\uAA7E\u1007\uAA63\uA9EB\u1079\uAA72\u1008\u105B\uA9E2\uAA64\uA9EC\u1061\uAA7F\u1009\u107A\uAA65\uA9E7\u100A\u100B\uAA66\u100C\uAA67\u100D\uAA68\uA9ED\u100E\uAA69\uA9EE\u100F\u106E\uA9E3\uA9EF\u1010-\u1012\u107B\uA9FB\u1013\uAA6A\uA9FC\u1014\u107C\uAA6B\u105E\u1015\u1016\u107D\u107E\uAA6F\u108E\uA9E8\u1017\u107F\uA9FD\u1018\uA9E4\uA9FE\u1019\u105F\u101A\u103B\u101B\uAA73\uAA7A\u103C\u101C\u1060\u101D\u103D\u1082\u1080\u1050\u1051\u1065\u101E\u103F\uAA6C\u101F\u1081\uAA6D\u103E\uAA6E\uAA71\u1020\uA9FA\u105C\u105D\u106F\u1070\u1066\u1021-\u1026\u1052-\u1055\u1027-\u102A\u102C\u102B\u1083\u1072\u109C\u102D\u1071\u102E\u1033\u102F\u1073\u1074\u1030\u1056-\u1059\u1031\u1084\u1035\u1085\u1032\u109D\u1034\u1062\u1067\u1068\uA9E5\u1086\u1039\u103A\u1063\u1064\u1069-\u106D\u1087\u108B\u1088\u108C\u108D\u1089\u108A\u108F\u109A\u109B\uAA7B-\uAA7D\uAA74-\uAA76\u1780-\u17A2\u17DC\u17A3-\u17B3\u17B6-\u17C5\u17D2\u1950-\u196D\u1970-\u1974\u1980-\u199C\u19DE\u19DF\u199D-\u19AB\u19B0-\u19C9\u1A20-\u1A26\u1A58\u1A59\u1A27-\u1A3B\u1A5A\u1A5B\u1A3C-\u1A46\u1A54\u1A47-\u1A4C\u1A53\u1A6B\u1A55-\u1A57\u1A5C-\u1A5E\u1A4D-\u1A52\u1A61\u1A6C\u1A62-\u1A6A\u1A6E\u1A6F\u1A73\u1A70-\u1A72\u1A6D\u1A60]"); /** * Simple upgrade test for analyzers to make sure they analyze to the same tokens after upgrade diff --git a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java index c9d5f0b622e..4601c1bcfcf 100644 --- a/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java +++ b/core/src/test/java/org/elasticsearch/bwcompat/NodesStatsBasicBackwardsCompatIT.java @@ -22,7 +22,6 @@ package org.elasticsearch.bwcompat; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsRequestBuilder; -import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESBackcompatTestCase; @@ -46,7 +45,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { for (NodeInfo n : nodesInfo.getNodes()) { TransportClient tc = TransportClient.builder().settings(settings).build().addTransportAddress(n.getNode().getAddress()); // Just verify that the NS can be sent and serialized/deserialized between nodes with basic indices - NodesStatsResponse ns = tc.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); + tc.admin().cluster().prepareNodesStats().setIndices(true).execute().actionGet(); tc.close(); } } @@ -78,7 +77,7 @@ public class NodesStatsBasicBackwardsCompatIT extends ESBackcompatTestCase { method.invoke(nsBuilder); } } - NodesStatsResponse ns = nsBuilder.execute().actionGet(); + nsBuilder.execute().actionGet(); tc.close(); } diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java index b82b5e0ba60..da196d5d1d3 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterStateDiffIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.RepositoriesMetaData; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.snapshots.SnapshotId; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -121,7 +122,7 @@ public class ClusterStateDiffIT extends ESIntegTestCase { Diff diffBeforeSerialization = clusterState.diff(previousClusterState); BytesStreamOutput os = new BytesStreamOutput(); diffBeforeSerialization.writeTo(os); - byte[] diffBytes = os.bytes().toBytes(); + byte[] diffBytes = BytesReference.toBytes(os.bytes()); Diff diff; try (StreamInput input = StreamInput.wrap(diffBytes)) { diff = previousClusterStateFromDiffs.readDiffFrom(input); diff --git a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java index 7f2d0828128..a7fe1b918c0 100644 --- a/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/block/ClusterBlockTests.java @@ -55,7 +55,7 @@ public class ClusterBlockTests extends ESTestCase { out.setVersion(version); clusterBlock.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(version); ClusterBlock result = ClusterBlock.readClusterBlock(in); diff --git a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java index fd1e3e62466..0918358a510 100644 --- a/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/health/ClusterStateHealthTests.java @@ -169,7 +169,7 @@ public class ClusterStateHealthTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); clusterStateHealth.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); clusterStateHealth = new ClusterStateHealth(in); } return clusterStateHealth; diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java index aec701052fb..8dd950ba8e6 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexGraveyardTests.java @@ -60,8 +60,7 @@ public class IndexGraveyardTests extends ESTestCase { final IndexGraveyard graveyard = createRandom(); final BytesStreamOutput out = new BytesStreamOutput(); graveyard.writeTo(out); - final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); - assertThat(IndexGraveyard.fromStream(in), equalTo(graveyard)); + assertThat(IndexGraveyard.fromStream(out.bytes().streamInput()), equalTo(graveyard)); } public void testXContent() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 0c9827587ea..5fef33be388 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -69,7 +69,7 @@ public class IndexMetaDataTests extends ESTestCase { final BytesStreamOutput out = new BytesStreamOutput(); metaData.writeTo(out); - IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(StreamInput.wrap(out.bytes())); + IndexMetaData deserialized = IndexMetaData.PROTO.readFrom(out.bytes().streamInput()); assertEquals(metaData, deserialized); assertEquals(metaData.hashCode(), deserialized.hashCode()); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index e1e3a39122c..cf040fb3c7f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.Version; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -34,7 +33,6 @@ import org.elasticsearch.index.Index; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.nio.ByteBuffer; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -185,8 +183,7 @@ public class MetaDataTests extends ESTestCase { final MetaData originalMeta = MetaData.builder().indexGraveyard(graveyard).build(); final BytesStreamOutput out = new BytesStreamOutput(); originalMeta.writeTo(out); - final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); - final MetaData fromStreamMeta = MetaData.PROTO.readFrom(in); + final MetaData fromStreamMeta = MetaData.PROTO.readFrom(out.bytes().streamInput()); assertThat(fromStreamMeta.indexGraveyard(), equalTo(fromStreamMeta.indexGraveyard())); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java index 2d3e44db68a..32072282d6f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/PrimaryTermsTests.java @@ -50,7 +50,7 @@ public class PrimaryTermsTests extends ESAllocationTestCase { private RoutingTable testRoutingTable; private int numberOfShards; private int numberOfReplicas; - private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); private AllocationService allocationService; private ClusterState clusterState; diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java index 2d1a467a001..9da5e76ed1f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/RoutingTableTests.java @@ -47,7 +47,7 @@ public class RoutingTableTests extends ESAllocationTestCase { private int numberOfReplicas; private int shardsPerIndex; private int totalNumberOfShards; - private final static Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); + private static final Settings DEFAULT_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); private final AllocationService ALLOCATION_SERVICE = createAllocationService(Settings.builder() .put("cluster.routing.allocation.node_concurrent_recoveries", Integer.MAX_VALUE) // don't limit recoveries .put("cluster.routing.allocation.node_initial_primaries_recoveries", Integer.MAX_VALUE) diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java index 7267252b19f..fa9133f6d36 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/ShardRoutingTests.java @@ -206,7 +206,7 @@ public class ShardRoutingTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); routing.writeTo(out); - routing = new ShardRouting(StreamInput.wrap(out.bytes())); + routing = new ShardRouting(out.bytes().streamInput()); } if (routing.initializing() || routing.relocating()) { assertEquals(routing.toString(), byteSize, routing.getExpectedShardSize()); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java index 75300a4beb8..ec33a3cd5fc 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java @@ -82,7 +82,7 @@ public class UnassignedInfoTests extends ESAllocationTestCase { meta.writeTo(out); out.close(); - UnassignedInfo read = new UnassignedInfo(StreamInput.wrap(out.bytes())); + UnassignedInfo read = new UnassignedInfo(out.bytes().streamInput()); assertThat(read.getReason(), equalTo(meta.getReason())); assertThat(read.getUnassignedTimeInMillis(), equalTo(meta.getUnassignedTimeInMillis())); assertThat(read.getMessage(), equalTo(meta.getMessage())); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java index 28f27b8988c..f95fb687c76 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationCommandsTests.java @@ -430,7 +430,7 @@ public class AllocationCommandsTests extends ESAllocationTestCase { ); BytesStreamOutput bytes = new BytesStreamOutput(); AllocationCommands.writeTo(commands, bytes); - StreamInput in = StreamInput.wrap(bytes.bytes()); + StreamInput in = bytes.bytes().streamInput(); // Since the commands are named writeable we need to register them and wrap the input stream NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java index 7b81d3ece27..4fa6615ac45 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/ClusterSerializationTests.java @@ -80,7 +80,7 @@ public class ClusterSerializationTests extends ESAllocationTestCase { BytesStreamOutput outStream = new BytesStreamOutput(); source.writeTo(outStream); - StreamInput inStream = StreamInput.wrap(outStream.bytes().toBytes()); + StreamInput inStream = outStream.bytes().streamInput(); RoutingTable target = RoutingTable.Builder.readFrom(inStream); assertThat(target.prettyPrint(), equalTo(source.prettyPrint())); diff --git a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java index 452c6054576..611c261e334 100644 --- a/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/serialization/DiffableTests.java @@ -310,7 +310,7 @@ public class DiffableTests extends ESTestCase { logger.debug("--> serializing diff"); BytesStreamOutput out = new BytesStreamOutput(); diffMap.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); logger.debug("--> reading diff back"); diffMap = readDiff(in); } diff --git a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java index 4f2bad36d4a..c0cb3482b0e 100644 --- a/core/src/test/java/org/elasticsearch/common/ChannelsTests.java +++ b/core/src/test/java/org/elasticsearch/common/ChannelsTests.java @@ -19,14 +19,11 @@ package org.elasticsearch.common; -import org.elasticsearch.common.bytes.ByteBufferBytesReference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Channels; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; -import org.jboss.netty.buffer.ByteBufferBackedChannelBuffer; -import org.jboss.netty.buffer.ChannelBuffer; import org.junit.After; import org.junit.Before; @@ -85,7 +82,7 @@ public class ChannelsTests extends ESTestCase { BytesReference source = new BytesArray(randomBytes, offset + offsetToRead, lengthToRead); BytesReference read = new BytesArray(readBytes, offset + offsetToRead, lengthToRead); - assertThat("read bytes didn't match written bytes", source.toBytes(), Matchers.equalTo(read.toBytes())); + assertThat("read bytes didn't match written bytes", BytesReference.toBytes(source), Matchers.equalTo(BytesReference.toBytes(read))); } public void testBufferReadPastEOFWithException() throws Exception { @@ -157,7 +154,9 @@ public class ChannelsTests extends ESTestCase { copy.flip(); BytesReference sourceRef = new BytesArray(randomBytes, offset + offsetToRead, lengthToRead); - BytesReference copyRef = new ByteBufferBytesReference(copy); + byte[] tmp = new byte[copy.remaining()]; + copy.duplicate().get(tmp); + BytesReference copyRef = new BytesArray(tmp); assertTrue("read bytes didn't match written bytes", sourceRef.equals(copyRef)); } diff --git a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java index f31d7c69325..90922327732 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.ByteArray; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; -import org.hamcrest.Matchers; import java.io.EOFException; import java.io.IOException; @@ -66,12 +65,9 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int sliceLength = Math.max(0, length - sliceOffset - 1); BytesReference slice = pbr.slice(sliceOffset, sliceLength); assertEquals(sliceLength, slice.length()); - - if (slice.hasArray()) { - assertEquals(sliceOffset, slice.arrayOffset()); - } else { - expectThrows(IllegalStateException.class, () -> - slice.arrayOffset()); + BytesRef singlePageOrNull = getSinglePageOrNull(slice); + if (singlePageOrNull != null) { + assertEquals(sliceOffset, singlePageOrNull.offset); } } @@ -109,7 +105,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // bulk-read all si.readFully(targetBuf); - assertArrayEquals(pbr.toBytes(), targetBuf); + assertArrayEquals(BytesReference.toBytes(pbr), targetBuf); // continuing to read should now fail with EOFException try { @@ -141,7 +137,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // now do NOT reset the stream - keep the stream's offset! // buffer to compare remaining bytes against bulk read - byte[] pbrBytesWithOffset = Arrays.copyOfRange(pbr.toBytes(), offset, length); + byte[] pbrBytesWithOffset = Arrays.copyOfRange(BytesReference.toBytes(pbr), offset, length); // randomized target buffer to ensure no stale slots byte[] targetBytes = new byte[pbrBytesWithOffset.length]; random().nextBytes(targetBytes); @@ -178,7 +174,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { } assertEquals(pbr.length(), target.length()); BytesRef targetBytes = target.get(); - assertArrayEquals(pbr.toBytes(), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length)); + assertArrayEquals(BytesReference.toBytes(pbr), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length)); } public void testSliceStreamInput() throws IOException { @@ -208,11 +204,11 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { assertEquals(sliceInput.available(), 0); // compare slice content with upper half of original - byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length); + byte[] pbrSliceBytes = Arrays.copyOfRange(BytesReference.toBytes(pbr), sliceOffset, length); assertArrayEquals(pbrSliceBytes, sliceBytes); // compare slice bytes with bytes read from slice via streamInput :D - byte[] sliceToBytes = slice.toBytes(); + byte[] sliceToBytes = BytesReference.toBytes(slice); assertEquals(sliceBytes.length, sliceToBytes.length); assertArrayEquals(sliceBytes, sliceToBytes); @@ -233,7 +229,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); pbr.writeTo(out); assertEquals(pbr.length(), out.size()); - assertArrayEquals(pbr.toBytes(), out.bytes().toBytes()); + assertArrayEquals(BytesReference.toBytes(pbr), BytesReference.toBytes(out.bytes())); out.close(); } @@ -246,7 +242,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesStreamOutput sliceOut = new BytesStreamOutput(sliceLength); slice.writeTo(sliceOut); assertEquals(slice.length(), sliceOut.size()); - assertArrayEquals(slice.toBytes(), sliceOut.bytes().toBytes()); + assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(sliceOut.bytes())); sliceOut.close(); } @@ -254,16 +250,16 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); - byte[] bytes = pbr.toBytes(); + byte[] bytes = BytesReference.toBytes(pbr); assertEquals(sizes[i], bytes.length); } } - public void testToBytesArraySharedPage() throws IOException { + public void testToBytesRefSharedPage() throws IOException { int length = randomIntBetween(10, PAGE_SIZE); BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.toBytesArray(); - BytesArray ba2 = pbr.toBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef()); + BytesArray ba2 = new BytesArray(pbr.toBytesRef()); assertNotNull(ba); assertNotNull(ba2); assertEquals(pbr.length(), ba.length()); @@ -272,46 +268,46 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { assertSame(ba.array(), ba2.array()); } - public void testToBytesArrayMaterializedPages() throws IOException { + public void testToBytesRefMaterializedPages() throws IOException { // we need a length != (n * pagesize) to avoid page sharing at boundaries int length = 0; while ((length % PAGE_SIZE) == 0) { length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5)); } BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.toBytesArray(); - BytesArray ba2 = pbr.toBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef()); + BytesArray ba2 = new BytesArray(pbr.toBytesRef()); assertNotNull(ba); assertNotNull(ba2); assertEquals(pbr.length(), ba.length()); assertEquals(ba.length(), ba2.length()); } - public void testCopyBytesArray() throws IOException { + public void testCopyBytesRefSharesBytes() throws IOException { // small PBR which would normally share the first page int length = randomIntBetween(10, PAGE_SIZE); BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.copyBytesArray(); - BytesArray ba2 = pbr.copyBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef(), true); + BytesArray ba2 = new BytesArray(pbr.toBytesRef(), true); assertNotNull(ba); assertNotSame(ba, ba2); assertNotSame(ba.array(), ba2.array()); } - public void testSliceCopyBytesArray() throws IOException { + public void testSliceCopyBytesRef() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); - BytesArray ba1 = slice.copyBytesArray(); - BytesArray ba2 = slice.copyBytesArray(); + BytesArray ba1 = new BytesArray(slice.toBytesRef(), true); + BytesArray ba2 = new BytesArray(slice.toBytesRef(), true); assertNotNull(ba1); assertNotNull(ba2); assertNotSame(ba1.array(), ba2.array()); - assertArrayEquals(slice.toBytes(), ba1.array()); - assertArrayEquals(slice.toBytes(), ba2.array()); + assertArrayEquals(BytesReference.toBytes(slice), ba1.array()); + assertArrayEquals(BytesReference.toBytes(slice), ba2.array()); assertArrayEquals(ba1.array(), ba2.array()); } @@ -329,14 +325,14 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { while((ref = iterator.next()) != null) { builder.append(ref); } - assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); + assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); } public void testSliceIterator() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); BytesRefIterator iterator = slice.iterator(); BytesRef ref = null; @@ -344,7 +340,7 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { while((ref = iterator.next()) != null) { builder.append(ref); } - assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); + assertArrayEquals(BytesReference.toBytes(slice), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); } public void testIteratorRandom() throws IOException { @@ -352,12 +348,12 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesReference pbr = newBytesReference(length); if (randomBoolean()) { int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); pbr = pbr.slice(sliceOffset, sliceLength); } if (randomBoolean()) { - pbr = pbr.toBytesArray(); + pbr = new BytesArray(pbr.toBytesRef()); } BytesRefIterator iterator = pbr.iterator(); BytesRef ref = null; @@ -365,29 +361,15 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { while((ref = iterator.next()) != null) { builder.append(ref); } - assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); - } - - public void testArray() throws IOException { - int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; - - for (int i = 0; i < sizes.length; i++) { - BytesReference pbr = newBytesReference(sizes[i]); - byte[] array = pbr.array(); - assertNotNull(array); - assertEquals(sizes[i], array.length); - assertSame(array, pbr.array()); - } + assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); } public void testArrayOffset() throws IOException { int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); - if (pbr.hasArray()) { - assertEquals(0, pbr.arrayOffset()); - } else { - expectThrows(IllegalStateException.class, () -> - pbr.arrayOffset()); + BytesRef singlePageOrNull = getSinglePageOrNull(pbr); + if (singlePageOrNull != null) { + assertEquals(0, singlePageOrNull.offset); } } @@ -395,20 +377,24 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length() - 1); // an offset to the end would be len 0 - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); - if (slice.hasArray()) { - assertEquals(sliceOffset, slice.arrayOffset()); - } else { - expectThrows(IllegalStateException.class, () -> - slice.arrayOffset()); + BytesRef singlePageOrNull = getSinglePageOrNull(slice); + if (singlePageOrNull != null) { + if (getSinglePageOrNull(pbr) == null) { + // original reference has pages + assertEquals(sliceOffset % PAGE_SIZE, singlePageOrNull.offset); + } else { + // orig ref has no pages ie. BytesArray + assertEquals(sliceOffset, singlePageOrNull.offset); + } } } public void testToUtf8() throws IOException { // test empty BytesReference pbr = newBytesReference(0); - assertEquals("", pbr.toUtf8()); + assertEquals("", pbr.utf8ToString()); // TODO: good way to test? } @@ -417,7 +403,6 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesReference pbr = newBytesReference(length); BytesRef ref = pbr.toBytesRef(); assertNotNull(ref); - assertEquals(pbr.arrayOffset(), ref.offset); assertEquals(pbr.length(), ref.length); } @@ -426,21 +411,13 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { BytesReference pbr = newBytesReference(length); // get a BytesRef from a slice int sliceOffset = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); + int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset); BytesRef sliceRef = pbr.slice(sliceOffset, sliceLength).toBytesRef(); // note that these are only true if we have <= than a page, otherwise offset/length are shifted assertEquals(sliceOffset, sliceRef.offset); assertEquals(sliceLength, sliceRef.length); } - public void testCopyBytesRef() throws IOException { - int length = randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5)); - BytesReference pbr = newBytesReference(length); - BytesRef ref = pbr.copyBytesRef(); - assertNotNull(ref); - assertEquals(pbr.length(), ref.length); - } - public void testHashCode() throws IOException { // empty content must have hash 1 (JDK compat) BytesReference pbr = newBytesReference(0); @@ -448,40 +425,36 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // test with content pbr = newBytesReference(randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5))); - int jdkHash = Arrays.hashCode(pbr.toBytes()); + int jdkHash = Arrays.hashCode(BytesReference.toBytes(pbr)); int pbrHash = pbr.hashCode(); assertEquals(jdkHash, pbrHash); // test hashes of slices int sliceFrom = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom); + int sliceLength = randomIntBetween(0, pbr.length() - sliceFrom); BytesReference slice = pbr.slice(sliceFrom, sliceLength); - int sliceJdkHash = Arrays.hashCode(slice.toBytes()); + int sliceJdkHash = Arrays.hashCode(BytesReference.toBytes(slice)); int sliceHash = slice.hashCode(); assertEquals(sliceJdkHash, sliceHash); } - public void testEquals() { - int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); - ByteArray ba1 = bigarrays.newByteArray(length, false); - ByteArray ba2 = bigarrays.newByteArray(length, false); - - // copy contents - for (long i = 0; i < length; i++) { - ba2.set(i, ba1.get(i)); - } + public void testEquals() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5))); + BytesReference copy = bytesReference.slice(0, bytesReference.length()); // get refs & compare - BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length); - BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length); - assertEquals(pbr, pbr2); - } + assertEquals(copy, bytesReference); + int sliceFrom = randomIntBetween(0, bytesReference.length()); + int sliceLength = randomIntBetween(0, bytesReference.length() - sliceFrom); + assertEquals(copy.slice(sliceFrom, sliceLength), bytesReference.slice(sliceFrom, sliceLength)); - public void testEqualsPeerClass() throws IOException { - int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); - BytesReference pbr = newBytesReference(length); - BytesReference ba = new BytesArray(pbr.toBytes()); - assertEquals(pbr, ba); + BytesRef bytesRef = BytesRef.deepCopyOf(copy.toBytesRef()); + assertEquals(new BytesArray(bytesRef), copy); + + int offsetToFlip = randomIntBetween(0, bytesRef.length - 1); + int value = ~Byte.toUnsignedInt(bytesRef.bytes[bytesRef.offset+offsetToFlip]); + bytesRef.bytes[bytesRef.offset+offsetToFlip] = (byte)value; + assertNotEquals(new BytesArray(bytesRef), copy); } public void testSliceEquals() { @@ -491,19 +464,118 @@ public abstract class AbstractBytesReferenceTestCase extends ESTestCase { // test equality of slices int sliceFrom = randomIntBetween(0, pbr.length()); - int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom); + int sliceLength = randomIntBetween(0, pbr.length() - sliceFrom); BytesReference slice1 = pbr.slice(sliceFrom, sliceLength); BytesReference slice2 = pbr.slice(sliceFrom, sliceLength); - assertArrayEquals(slice1.toBytes(), slice2.toBytes()); + assertArrayEquals(BytesReference.toBytes(slice1), BytesReference.toBytes(slice2)); // test a slice with same offset but different length, // unless randomized testing gave us a 0-length slice. if (sliceLength > 0) { BytesReference slice3 = pbr.slice(sliceFrom, sliceLength / 2); - assertFalse(Arrays.equals(slice1.toBytes(), slice3.toBytes())); + assertFalse(Arrays.equals(BytesReference.toBytes(slice1), BytesReference.toBytes(slice3))); } } protected abstract BytesReference newBytesReference(int length) throws IOException; + public void testCompareTo() throws IOException { + final int iters = randomIntBetween(5, 10); + for (int i = 0; i < iters; i++) { + int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); + BytesReference bytesReference = newBytesReference(length); + assertTrue(bytesReference.compareTo(new BytesArray("")) > 0); + assertTrue(new BytesArray("").compareTo(bytesReference) < 0); + + + assertEquals(0, bytesReference.compareTo(bytesReference)); + int sliceFrom = randomIntBetween(0, bytesReference.length()); + int sliceLength = randomIntBetween(0, bytesReference.length() - sliceFrom); + BytesReference slice = bytesReference.slice(sliceFrom, sliceLength); + + assertEquals(bytesReference.toBytesRef().compareTo(slice.toBytesRef()), + new BytesArray(bytesReference.toBytesRef(), true).compareTo(new BytesArray(slice.toBytesRef(), true))); + + assertEquals(bytesReference.toBytesRef().compareTo(slice.toBytesRef()), + bytesReference.compareTo(slice)); + assertEquals(slice.toBytesRef().compareTo(bytesReference.toBytesRef()), + slice.compareTo(bytesReference)); + + assertEquals(0, slice.compareTo(new BytesArray(slice.toBytesRef()))); + assertEquals(0, new BytesArray(slice.toBytesRef()).compareTo(slice)); + + final int crazyLength = length + randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); + ReleasableBytesStreamOutput crazyStream = new ReleasableBytesStreamOutput(length, bigarrays); + final int offset = randomIntBetween(0, crazyLength - length); + for (int j = 0; j < offset; j++) { + crazyStream.writeByte((byte) random().nextInt(1 << 8)); + } + bytesReference.writeTo(crazyStream); + for (int j = crazyStream.size(); j < crazyLength; j++) { + crazyStream.writeByte((byte) random().nextInt(1 << 8)); + } + PagedBytesReference crazyReference = crazyStream.bytes(); + + assertFalse(crazyReference.compareTo(bytesReference) == 0); + assertEquals(0, crazyReference.slice(offset, length).compareTo( + bytesReference)); + assertEquals(0, bytesReference.compareTo( + crazyReference.slice(offset, length))); + } + } + + public static BytesRef getSinglePageOrNull(BytesReference ref) throws IOException { + if (ref.length() > 0) { + BytesRefIterator iterator = ref.iterator(); + BytesRef next = iterator.next(); + BytesRef retVal = next.clone(); + if (iterator.next() == null) { + return retVal; + } + } else { + return new BytesRef(); + } + return null; + } + + public static int getNumPages(BytesReference ref) throws IOException { + int num = 0; + if (ref.length() > 0) { + BytesRefIterator iterator = ref.iterator(); + while(iterator.next() != null) { + num++; + } + } + return num; + } + + + public void testBasicEquals() { + final int len = randomIntBetween(0, randomBoolean() ? 10: 100000); + final int offset1 = randomInt(5); + final byte[] array1 = new byte[offset1 + len + randomInt(5)]; + random().nextBytes(array1); + final int offset2 = randomInt(offset1); + final byte[] array2 = Arrays.copyOfRange(array1, offset1 - offset2, array1.length); + + final BytesArray b1 = new BytesArray(array1, offset1, len); + final BytesArray b2 = new BytesArray(array2, offset2, len); + assertEquals(b1, b2); + assertEquals(Arrays.hashCode(BytesReference.toBytes(b1)), b1.hashCode()); + assertEquals(Arrays.hashCode(BytesReference.toBytes(b2)), b2.hashCode()); + + // test same instance + assertEquals(b1, b1); + assertEquals(b2, b2); + + if (len > 0) { + // test different length + BytesArray differentLen = new BytesArray(array1, offset1, randomInt(len - 1)); + assertNotEquals(b1, differentLen); + + // test changed bytes + array1[offset1 + randomInt(len - 1)] += 13; + assertNotEquals(b1, b2); + } + } } diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java index 61d24ef44c3..fff030200b7 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/BytesArrayTests.java @@ -32,10 +32,28 @@ public class BytesArrayTests extends AbstractBytesReferenceTestCase { out.writeByte((byte) random().nextInt(1 << 8)); } assertEquals(length, out.size()); - BytesArray ref = out.bytes().toBytesArray(); + BytesArray ref = new BytesArray(out.bytes().toBytesRef()); assertEquals(length, ref.length()); assertTrue(ref instanceof BytesArray); assertThat(ref.length(), Matchers.equalTo(length)); return ref; } + + public void testArray() throws IOException { + int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; + + for (int i = 0; i < sizes.length; i++) { + BytesArray pbr = (BytesArray) newBytesReference(sizes[i]); + byte[] array = pbr.array(); + assertNotNull(array); + assertEquals(sizes[i], array.length); + assertSame(array, pbr.array()); + } + } + + public void testArrayOffset() throws IOException { + int length = randomInt(PAGE_SIZE * randomIntBetween(2, 5)); + BytesArray pbr = (BytesArray) newBytesReference(length); + assertEquals(0, pbr.offset()); + } } diff --git a/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java deleted file mode 100644 index 60f4983dd19..00000000000 --- a/core/src/test/java/org/elasticsearch/common/bytes/BytesReferenceTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.bytes; - - -import org.elasticsearch.test.ESTestCase; - -import java.util.Arrays; - -public class BytesReferenceTests extends ESTestCase { - - public void testEquals() { - final int len = randomIntBetween(0, randomBoolean() ? 10: 100000); - final int offset1 = randomInt(5); - final byte[] array1 = new byte[offset1 + len + randomInt(5)]; - random().nextBytes(array1); - final int offset2 = randomInt(offset1); - final byte[] array2 = Arrays.copyOfRange(array1, offset1 - offset2, array1.length); - - final BytesArray b1 = new BytesArray(array1, offset1, len); - final BytesArray b2 = new BytesArray(array2, offset2, len); - assertTrue(BytesReference.Helper.bytesEqual(b1, b2)); - assertTrue(BytesReference.Helper.bytesEquals(b1, b2)); - assertEquals(Arrays.hashCode(b1.toBytes()), b1.hashCode()); - assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b2)); - - // test same instance - assertTrue(BytesReference.Helper.bytesEqual(b1, b1)); - assertTrue(BytesReference.Helper.bytesEquals(b1, b1)); - assertEquals(BytesReference.Helper.bytesHashCode(b1), BytesReference.Helper.slowHashCode(b1)); - - if (len > 0) { - // test different length - BytesArray differentLen = new BytesArray(array1, offset1, randomInt(len - 1)); - assertFalse(BytesReference.Helper.bytesEqual(b1, differentLen)); - - // test changed bytes - array1[offset1 + randomInt(len - 1)] += 13; - assertFalse(BytesReference.Helper.bytesEqual(b1, b2)); - assertFalse(BytesReference.Helper.bytesEquals(b1, b2)); - } - } - -} diff --git a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java index 5a299d82de8..6ae2b3cf943 100644 --- a/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/common/bytes/PagedBytesReferenceTests.java @@ -50,15 +50,15 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase { return ref; } - public void testToBytesArrayMaterializedPages() throws IOException { + public void testToBytesRefMaterializedPages() throws IOException { // we need a length != (n * pagesize) to avoid page sharing at boundaries int length = 0; while ((length % PAGE_SIZE) == 0) { length = randomIntBetween(PAGE_SIZE, PAGE_SIZE * randomIntBetween(2, 5)); } BytesReference pbr = newBytesReference(length); - BytesArray ba = pbr.toBytesArray(); - BytesArray ba2 = pbr.toBytesArray(); + BytesArray ba = new BytesArray(pbr.toBytesRef()); + BytesArray ba2 = new BytesArray(pbr.toBytesRef()); assertNotNull(ba); assertNotNull(ba2); assertEquals(pbr.length(), ba.length()); @@ -67,23 +67,23 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase { assertNotSame(ba.array(), ba2.array()); } - public void testArray() throws IOException { + public void testSinglePage() throws IOException { int[] sizes = {0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5))}; for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); // verify that array() is cheap for small payloads if (sizes[i] <= PAGE_SIZE) { - byte[] array = pbr.array(); + BytesRef page = getSinglePageOrNull(pbr); + assertNotNull(page); + byte[] array = page.bytes; assertNotNull(array); assertEquals(sizes[i], array.length); - assertSame(array, pbr.array()); + assertSame(array, page.bytes); } else { - try { - pbr.array(); - fail("expected IllegalStateException"); - } catch (IllegalStateException isx) { - // expected + BytesRef page = getSinglePageOrNull(pbr); + if (pbr.length() > 0) { + assertNull(page); } } } @@ -94,22 +94,42 @@ public class PagedBytesReferenceTests extends AbstractBytesReferenceTestCase { for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); - byte[] bytes = pbr.toBytes(); + byte[] bytes = BytesReference.toBytes(pbr); assertEquals(sizes[i], bytes.length); // verify that toBytes() is cheap for small payloads if (sizes[i] <= PAGE_SIZE) { - assertSame(bytes, pbr.toBytes()); + assertSame(bytes, BytesReference.toBytes(pbr)); } else { - assertNotSame(bytes, pbr.toBytes()); + assertNotSame(bytes, BytesReference.toBytes(pbr)); } } } - public void testHasArray() throws IOException { + public void testHasSinglePage() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(1, 3)); BytesReference pbr = newBytesReference(length); // must return true for <= pagesize - assertEquals(length <= PAGE_SIZE, pbr.hasArray()); + assertEquals(length <= PAGE_SIZE, getNumPages(pbr) == 1); + } + + public void testEquals() { + int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); + ByteArray ba1 = bigarrays.newByteArray(length, false); + ByteArray ba2 = bigarrays.newByteArray(length, false); + + // copy contents + for (long i = 0; i < length; i++) { + ba2.set(i, ba1.get(i)); + } + + // get refs & compare + BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length); + BytesReference pbr2 = new PagedBytesReference(bigarrays, ba2, length); + assertEquals(pbr, pbr2); + int offsetToFlip = randomIntBetween(0, length - 1); + int value = ~Byte.toUnsignedInt(ba1.get(offsetToFlip)); + ba2.set(offsetToFlip, (byte)value); + assertNotEquals(pbr, pbr2); } } diff --git a/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java index 72866d082ae..0ce95077965 100644 --- a/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java +++ b/core/src/test/java/org/elasticsearch/common/compress/DeflateCompressedXContentTests.java @@ -91,8 +91,8 @@ public class DeflateCompressedXContentTests extends ESTestCase { // of different size are being used assertFalse(b1.equals(b2)); // we used the compressed representation directly and did not recompress - assertArrayEquals(b1.toBytes(), new CompressedXContent(b1).compressed()); - assertArrayEquals(b2.toBytes(), new CompressedXContent(b2).compressed()); + assertArrayEquals(BytesReference.toBytes(b1), new CompressedXContent(b1).compressed()); + assertArrayEquals(BytesReference.toBytes(b2), new CompressedXContent(b2).compressed()); // but compressedstring instances are still equal assertEquals(new CompressedXContent(b1), new CompressedXContent(b2)); } diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java index 407c9790dbe..416299f8e7e 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/GeoDistanceTests.java @@ -46,7 +46,7 @@ public class GeoDistanceTests extends ESTestCase { GeoDistance geoDistance = randomFrom(GeoDistance.PLANE, GeoDistance.FACTOR, GeoDistance.ARC, GeoDistance.SLOPPY_ARC); try (BytesStreamOutput out = new BytesStreamOutput()) { geoDistance.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) {; + try (StreamInput in = out.bytes().streamInput()) {; GeoDistance copy = GeoDistance.readFromStream(in); assertEquals(copy.toString() + " vs. " + geoDistance.toString(), copy, geoDistance); } @@ -60,7 +60,7 @@ public class GeoDistanceTests extends ESTestCase { } else { out.writeVInt(randomIntBetween(Integer.MIN_VALUE, -1)); } - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { GeoDistance.readFromStream(in); } catch (IOException e) { assertThat(e.getMessage(), containsString("Unknown GeoDistance ordinal [")); diff --git a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java index 566d2148cae..76376a4d30d 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/GeoJSONShapeParserTests.java @@ -56,7 +56,7 @@ import static org.elasticsearch.common.geo.builders.ShapeBuilder.SPATIAL_CONTEXT */ public class GeoJSONShapeParserTests extends ESTestCase { - private final static GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory(); + private static final GeometryFactory GEOMETRY_FACTORY = SPATIAL_CONTEXT.getGeometryFactory(); public void testParse_simplePoint() throws IOException { String pointGeoJson = XContentFactory.jsonBuilder().startObject().field("type", "Point") diff --git a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java index 6ee6a4fff83..e4eaa17874c 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/ShapeRelationTests.java @@ -39,21 +39,21 @@ public class ShapeRelationTests extends ESTestCase { public void testwriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { ShapeRelation.INTERSECTS.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { ShapeRelation.DISJOINT.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { ShapeRelation.WITHIN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(2)); } } @@ -62,19 +62,19 @@ public class ShapeRelationTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.INTERSECTS)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.DISJOINT)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(2); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(ShapeRelation.readFromStream(in), equalTo(ShapeRelation.WITHIN)); } } @@ -83,7 +83,7 @@ public class ShapeRelationTests extends ESTestCase { public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(3, Integer.MAX_VALUE)); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { ShapeRelation.readFromStream(in); fail("Expected IOException"); } catch(IOException e) { diff --git a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java index c2f29e6ecd7..b6eae97932f 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java +++ b/core/src/test/java/org/elasticsearch/common/geo/SpatialStrategyTests.java @@ -38,14 +38,14 @@ public class SpatialStrategyTests extends ESTestCase { public void testwriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { SpatialStrategy.TERM.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { SpatialStrategy.RECURSIVE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } @@ -54,13 +54,13 @@ public class SpatialStrategyTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(SpatialStrategy.readFromStream(in), equalTo(SpatialStrategy.TERM)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(SpatialStrategy.readFromStream(in), equalTo(SpatialStrategy.RECURSIVE)); } } @@ -69,7 +69,7 @@ public class SpatialStrategyTests extends ESTestCase { public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { SpatialStrategy.readFromStream(in); fail("Expected IOException"); } catch(IOException e) { diff --git a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java index 9cbd4bb769d..4003a96e26f 100644 --- a/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/geo/builders/AbstractShapeBuilderTestCase.java @@ -137,7 +137,7 @@ public abstract class AbstractShapeBuilderTestCase exte static ShapeBuilder copyShape(ShapeBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return namedWriteableRegistry.getReader(ShapeBuilder.class, original.getWriteableName()).read(in); } } diff --git a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java index 5c6c1e1789b..76b52c08a85 100644 --- a/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/StreamsTests.java @@ -84,7 +84,7 @@ public class StreamsTests extends ESTestCase { byte stuff[] = new byte[] { 0, 1, 2, 3 }; BytesRef stuffRef = new BytesRef(stuff, 2, 2); BytesArray stuffArray = new BytesArray(stuffRef); - StreamInput input = StreamInput.wrap(stuffArray); + StreamInput input = stuffArray.streamInput(); assertEquals(2, input.read()); assertEquals(3, input.read()); assertEquals(-1, input.read()); diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java index a4d15173a7c..dc57b0c70d4 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/AbstractWriteableEnumTestCase.java @@ -60,7 +60,7 @@ public abstract class AbstractWriteableEnumTestCase extends ESTestCase { protected static void assertWriteToStream(final Writeable writeableEnum, final int ordinal) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { writeableEnum.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(ordinal)); } } @@ -70,7 +70,7 @@ public abstract class AbstractWriteableEnumTestCase extends ESTestCase { protected void assertReadFromStream(final int ordinal, final Writeable expected) throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(ordinal); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(reader.read(in), equalTo(expected)); } } diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index 9fcbb708156..94f07369770 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.common.io.stream; import org.apache.lucene.util.Constants; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.util.BigArrays; @@ -48,7 +49,7 @@ public class BytesStreamsTests extends ESTestCase { // test empty stream to array assertEquals(0, out.size()); - assertEquals(0, out.bytes().toBytes().length); + assertEquals(0, out.bytes().length()); out.close(); } @@ -63,7 +64,7 @@ public class BytesStreamsTests extends ESTestCase { // write single byte out.writeByte(expectedData[0]); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -80,7 +81,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -108,14 +109,14 @@ public class BytesStreamsTests extends ESTestCase { byte[] expectedData = randomizedByteArrayWithSize(expectedSize); out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); // bulk-write again with actual bytes expectedSize = 10; expectedData = randomizedByteArrayWithSize(expectedSize); out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -130,7 +131,7 @@ public class BytesStreamsTests extends ESTestCase { out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -149,7 +150,7 @@ public class BytesStreamsTests extends ESTestCase { // now write the rest - more than fits into the remaining first page out.writeBytes(expectedData, initialOffset, additionalLength); assertEquals(expectedData.length, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -168,7 +169,7 @@ public class BytesStreamsTests extends ESTestCase { // ie. we cross over into a third out.writeBytes(expectedData, initialOffset, additionalLength); assertEquals(expectedData.length, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -185,7 +186,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -202,7 +203,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -219,7 +220,7 @@ public class BytesStreamsTests extends ESTestCase { } assertEquals(expectedSize, out.size()); - assertArrayEquals(expectedData, out.bytes().toBytes()); + assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); } @@ -235,7 +236,7 @@ public class BytesStreamsTests extends ESTestCase { out.seek(position += BigArrays.BYTE_PAGE_SIZE + 10); out.seek(position += BigArrays.BYTE_PAGE_SIZE * 2); assertEquals(position, out.position()); - assertEquals(position, out.bytes().toBytes().length); + assertEquals(position, BytesReference.toBytes(out.bytes()).length); out.close(); } @@ -288,8 +289,8 @@ public class BytesStreamsTests extends ESTestCase { out.writeTimeZone(DateTimeZone.forID("CET")); out.writeOptionalTimeZone(DateTimeZone.getDefault()); out.writeOptionalTimeZone(null); - final byte[] bytes = out.bytes().toBytes(); - StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + final byte[] bytes = BytesReference.toBytes(out.bytes()); + StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); assertEquals(in.available(), bytes.length); assertThat(in.readBoolean(), equalTo(false)); assertThat(in.readByte(), equalTo((byte)1)); @@ -328,7 +329,7 @@ public class BytesStreamsTests extends ESTestCase { namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - byte[] bytes = out.bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class); @@ -348,7 +349,7 @@ public class BytesStreamsTests extends ESTestCase { public void testNamedWriteableUnknownCategory() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(new TestNamedWriteable("test1", "test2")); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), new NamedWriteableRegistry()); + StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), new NamedWriteableRegistry()); //no named writeable registered with given name, can write but cannot read it back IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); assertThat(e.getMessage(), equalTo("unknown named writeable category [" + BaseNamedWriteable.class.getName() + "]")); @@ -368,7 +369,7 @@ public class BytesStreamsTests extends ESTestCase { public void writeTo(StreamOutput out) throws IOException { } }); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes().toBytes()), namedWriteableRegistry); + StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(BytesReference.toBytes(out.bytes())), namedWriteableRegistry); try { //no named writeable registered with given name under test category, can write but cannot read it back in.readNamedWriteable(BaseNamedWriteable.class); @@ -382,7 +383,7 @@ public class BytesStreamsTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2"); out.writeNamedWriteable(testNamedWriteable); - StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); try { in.readNamedWriteable(BaseNamedWriteable.class); fail("Expected UnsupportedOperationException"); @@ -397,7 +398,7 @@ public class BytesStreamsTests extends ESTestCase { namedWriteableRegistry.register(BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - byte[] bytes = out.bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); @@ -407,7 +408,7 @@ public class BytesStreamsTests extends ESTestCase { public void testOptionalWriteableReaderReturnsNull() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeOptionalWriteable(new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10))); - StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); IOException e = expectThrows(IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null)); assertThat(e.getMessage(), endsWith("] returned null which is not allowed and probably means it screwed up the stream.")); } @@ -423,7 +424,7 @@ public class BytesStreamsTests extends ESTestCase { }); TestNamedWriteable namedWriteableIn = new TestNamedWriteable(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); - byte[] bytes = out.bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); AssertionError e = expectThrows(AssertionError.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); @@ -442,7 +443,7 @@ public class BytesStreamsTests extends ESTestCase { final BytesStreamOutput out = new BytesStreamOutput(); out.writeStreamableList(expected); - final StreamInput in = StreamInput.wrap(out.bytes().toBytes()); + final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); List loaded = in.readStreamableList(TestStreamable::new); @@ -458,7 +459,7 @@ public class BytesStreamsTests extends ESTestCase { out.close(); } - private static abstract class BaseNamedWriteable implements NamedWriteable { + private abstract static class BaseNamedWriteable implements NamedWriteable { } @@ -537,7 +538,7 @@ public class BytesStreamsTests extends ESTestCase { // toByteArray() must fail try { - out.bytes().toBytes(); + BytesReference.toBytes(out.bytes()); fail("expected IllegalStateException: stream closed"); } catch (IllegalStateException iex1) { @@ -558,7 +559,7 @@ public class BytesStreamsTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble()); out.writeGenericValue(geoPoint); - StreamInput wrap = StreamInput.wrap(out.bytes()); + StreamInput wrap = out.bytes().streamInput(); GeoPoint point = (GeoPoint) wrap.readGenericValue(); assertEquals(point, geoPoint); } @@ -566,7 +567,7 @@ public class BytesStreamsTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); GeoPoint geoPoint = new GeoPoint(randomDouble(), randomDouble()); out.writeGeoPoint(geoPoint); - StreamInput wrap = StreamInput.wrap(out.bytes()); + StreamInput wrap = out.bytes().streamInput(); GeoPoint point = wrap.readGeoPoint(); assertEquals(point, geoPoint); } diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java index aa6016774b0..06d39398c8e 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/StreamTests.java @@ -19,13 +19,13 @@ package org.elasticsearch.common.io.stream; -import org.elasticsearch.common.bytes.ByteBufferBytesReference; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.test.ESTestCase; import java.io.ByteArrayInputStream; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -61,8 +61,8 @@ public class StreamTests extends ESTestCase { for (Tuple value : values) { BytesStreamOutput out = new BytesStreamOutput(); out.writeZLong(value.v1()); - assertArrayEquals(Long.toString(value.v1()), value.v2(), out.bytes().toBytes()); - ByteBufferBytesReference bytes = new ByteBufferBytesReference(ByteBuffer.wrap(value.v2())); + assertArrayEquals(Long.toString(value.v1()), value.v2(), BytesReference.toBytes(out.bytes())); + BytesReference bytes = new BytesArray(value.v2()); assertEquals(Arrays.toString(value.v2()), (long)value.v1(), bytes.streamInput().readZLong()); } } @@ -143,7 +143,7 @@ public class StreamTests extends ESTestCase { assertThat(targetArray, equalTo(sourceArray)); } - final static class WriteableString implements Writeable { + static final class WriteableString implements Writeable { final String string; public WriteableString(String string) { diff --git a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java index 9ae0beadf59..245520d65f3 100644 --- a/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java +++ b/core/src/test/java/org/elasticsearch/common/network/NetworkModuleTests.java @@ -95,7 +95,7 @@ public class NetworkModuleTests extends ModuleTestCase { static class FakeCatRestHandler extends AbstractCatAction { public FakeCatRestHandler() { - super(null, null); + super(null); } @Override protected void doRequest(RestRequest request, RestChannel channel, NodeClient client) {} diff --git a/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java b/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java index 45db5a33d21..1a3fa4db137 100644 --- a/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java +++ b/core/src/test/java/org/elasticsearch/common/transport/BoundTransportAddressTests.java @@ -51,7 +51,7 @@ public class BoundTransportAddressTests extends ESTestCase { // serialize BytesStreamOutput streamOutput = new BytesStreamOutput(); transportAddress.writeTo(streamOutput); - StreamInput in = ByteBufferStreamInput.wrap(streamOutput.bytes()); + StreamInput in = streamOutput.bytes().streamInput(); BoundTransportAddress serializedAddress; if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java index f9a4d3f22af..7c5463baed2 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/DistanceUnitTests.java @@ -82,7 +82,7 @@ public class DistanceUnitTests extends ESTestCase { for (DistanceUnit unit : DistanceUnit.values()) { try (BytesStreamOutput out = new BytesStreamOutput()) { unit.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat("Roundtrip serialisation failed.", DistanceUnit.readFromStream(in), equalTo(unit)); } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java index 2b5a7c00e5d..3f6f1848fd8 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/FuzzinessTests.java @@ -145,7 +145,7 @@ public class FuzzinessTests extends ESTestCase { private static Fuzziness doSerializeRoundtrip(Fuzziness in) throws IOException { BytesStreamOutput output = new BytesStreamOutput(); in.writeTo(output); - StreamInput streamInput = StreamInput.wrap(output.bytes()); + StreamInput streamInput = output.bytes().streamInput(); return new Fuzziness(streamInput); } } diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index 78afc9e514f..003d78ce42e 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -161,7 +161,7 @@ public class TimeValueTests extends ESTestCase { value.writeTo(out); assertEquals(expectedSize, out.size()); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); TimeValue inValue = new TimeValue(in); assertThat(inValue, equalTo(value)); diff --git a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java index 1a582d48f6b..d6797d4be26 100644 --- a/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java +++ b/core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java @@ -154,7 +154,7 @@ public class ThreadContextTests extends ESTestCase { assertNull(threadContext.getTransient("ctx.foo")); assertEquals("1", threadContext.getHeader("default")); - threadContext.readHeaders(StreamInput.wrap(out.bytes())); + threadContext.readHeaders(out.bytes().streamInput()); assertEquals("bar", threadContext.getHeader("foo")); assertNull(threadContext.getTransient("ctx.foo")); } @@ -179,14 +179,14 @@ public class ThreadContextTests extends ESTestCase { { Settings otherSettings = Settings.builder().put("request.headers.default", "5").build(); ThreadContext otherhreadContext = new ThreadContext(otherSettings); - otherhreadContext.readHeaders(StreamInput.wrap(out.bytes())); + otherhreadContext.readHeaders(out.bytes().streamInput()); assertEquals("bar", otherhreadContext.getHeader("foo")); assertNull(otherhreadContext.getTransient("ctx.foo")); assertEquals("1", otherhreadContext.getHeader("default")); } } - + public void testSerializeInDifferentContextNoDefaults() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); { @@ -202,7 +202,7 @@ public class ThreadContextTests extends ESTestCase { { Settings otherSettings = Settings.builder().put("request.headers.default", "5").build(); ThreadContext otherhreadContext = new ThreadContext(otherSettings); - otherhreadContext.readHeaders(StreamInput.wrap(out.bytes())); + otherhreadContext.readHeaders(out.bytes().streamInput()); assertEquals("bar", otherhreadContext.getHeader("foo")); assertNull(otherhreadContext.getTransient("ctx.foo")); diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java index 159d8a97be4..a8d26e87ecf 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/ObjectParserTests.java @@ -35,7 +35,7 @@ import org.elasticsearch.test.ESTestCase; public class ObjectParserTests extends ESTestCase { - private final static ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT; + private static final ParseFieldMatcherSupplier STRICT_PARSING = () -> ParseFieldMatcher.STRICT; public void testBasics() throws IOException { XContentParser parser = XContentType.JSON.xContent().createParser( diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java index 583234461b3..8319873878a 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/XContentFactoryTests.java @@ -57,11 +57,10 @@ public class XContentFactoryTests extends ESTestCase { builder.endObject(); assertThat(XContentFactory.xContentType(builder.bytes()), equalTo(type)); - BytesArray bytesArray = builder.bytes().toBytesArray(); - assertThat(XContentFactory.xContentType(StreamInput.wrap(bytesArray.array(), bytesArray.arrayOffset(), bytesArray.length())), equalTo(type)); + assertThat(XContentFactory.xContentType(builder.bytes().streamInput()), equalTo(type)); // CBOR is binary, cannot use String - if (type != XContentType.CBOR) { + if (type != XContentType.CBOR && type != XContentType.SMILE) { assertThat(XContentFactory.xContentType(builder.string()), equalTo(type)); } } diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java index 34944e713bd..fe69fc1f05d 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/builder/XContentBuilderTests.java @@ -94,7 +94,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.startObject(); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"foo\":{\"test\":\"value\"}}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"foo\":{\"test\":\"value\"}}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -102,7 +102,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"}}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -110,7 +110,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.field("test", "value"); xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"}}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -119,7 +119,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.rawField("foo", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.field("test1", "value1"); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"test1\":\"value1\"}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"test1\":\"value1\"}")); } { XContentBuilder xContentBuilder = XContentFactory.contentBuilder(XContentType.JSON); @@ -129,7 +129,7 @@ public class XContentBuilderTests extends ESTestCase { xContentBuilder.rawField("foo1", new BytesArray("{\"test\":\"value\"}")); xContentBuilder.field("test1", "value1"); xContentBuilder.endObject(); - assertThat(xContentBuilder.bytes().toUtf8(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"},\"test1\":\"value1\"}")); + assertThat(xContentBuilder.bytes().utf8ToString(), equalTo("{\"test\":\"value\",\"foo\":{\"test\":\"value\"},\"foo1\":{\"test\":\"value\"},\"test1\":\"value1\"}")); } } @@ -161,15 +161,14 @@ public class XContentBuilderTests extends ESTestCase { gen.writeEndObject(); gen.close(); - byte[] data = bos.bytes().toBytes(); - String sData = new String(data, "UTF8"); + String sData = bos.bytes().utf8ToString(); assertThat(sData, equalTo("{\"name\":\"something\", source : { test : \"value\" },\"name2\":\"something2\"}")); } public void testByteConversion() throws Exception { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); builder.startObject().field("test_name", (Byte)(byte)120).endObject(); - assertThat(builder.bytes().toUtf8(), equalTo("{\"test_name\":120}")); + assertThat(builder.bytes().utf8ToString(), equalTo("{\"test_name\":120}")); } public void testDateTypesConversion() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java index bf2dd442b64..efbca114aac 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/cbor/JsonVsCborTests.java @@ -63,7 +63,8 @@ public class JsonVsCborTests extends ESTestCase { xsonGen.close(); jsonGen.close(); - verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes().toBytes())); + verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()), + XContentFactory.xContent(XContentType.CBOR).createParser(xsonOs.bytes())); } private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java index 9e686fe78f1..63b19a63822 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/smile/JsonVsSmileTests.java @@ -63,7 +63,8 @@ public class JsonVsSmileTests extends ESTestCase { xsonGen.close(); jsonGen.close(); - verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes().toBytes()), XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes().toBytes())); + verifySameTokens(XContentFactory.xContent(XContentType.JSON).createParser(jsonOs.bytes()), + XContentFactory.xContent(XContentType.SMILE).createParser(xsonOs.bytes())); } private void verifySameTokens(XContentParser parser1, XContentParser parser2) throws IOException { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java index e3d8735e05e..b8b38a543f6 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/AbstractFilteringJsonGeneratorTestCase.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; -import java.io.ByteArrayInputStream; import java.io.IOException; import static org.hamcrest.CoreMatchers.is; @@ -45,7 +44,7 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase assertNotNull(expected); // Verify that the result is equal to the expected string - assertThat(builder.bytes().toUtf8(), is(expected.bytes().toUtf8())); + assertThat(builder.bytes().utf8ToString(), is(expected.bytes().utf8ToString())); } protected void assertBinary(XContentBuilder expected, XContentBuilder builder) { @@ -1166,15 +1165,15 @@ public abstract class AbstractFilteringJsonGeneratorTestCase extends ESTestCase // Test method: rawField(String fieldName, InputStream content) assertXContentBuilder(expectedRawField, - newXContentBuilder().startObject().field("foo", 0).rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + newXContentBuilder().startObject().field("foo", 0).rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("f*", true).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldFiltered, newXContentBuilder("r*", false).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("r*", true).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); assertXContentBuilder(expectedRawFieldNotFiltered, newXContentBuilder("f*", false).startObject().field("foo", 0) - .rawField("raw", new ByteArrayInputStream(raw.toBytes())).endObject()); + .rawField("raw", raw.streamInput()).endObject()); } public void testArrays() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java index dd2fe42eb8e..8dbefedb249 100644 --- a/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java +++ b/core/src/test/java/org/elasticsearch/common/xcontent/support/filtering/FilterPathGeneratorFilteringTests.java @@ -142,7 +142,7 @@ public class FilterPathGeneratorFilteringTests extends ESTestCase { } } } - assertThat(os.bytes().toUtf8(), equalTo(replaceQuotes(expected))); + assertThat(os.bytes().utf8ToString(), equalTo(replaceQuotes(expected))); } } diff --git a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java index 4efedd9154a..c25a0a6503b 100644 --- a/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java +++ b/core/src/test/java/org/elasticsearch/deps/jackson/JacksonLocationTests.java @@ -56,8 +56,7 @@ public class JacksonLocationTests extends ESTestCase { gen.close(); - byte[] data = os.bytes().toBytes(); - JsonParser parser = new JsonFactory().createParser(data); + JsonParser parser = new JsonFactory().createParser(os.bytes().streamInput()); assertThat(parser.nextToken(), equalTo(JsonToken.START_OBJECT)); assertThat(parser.nextToken(), equalTo(JsonToken.FIELD_NAME)); // "index" diff --git a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java index f6aac190c4b..bb38e329103 100644 --- a/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java @@ -40,7 +40,7 @@ import static org.hamcrest.Matchers.not; public class BlockingClusterStatePublishResponseHandlerTests extends ESTestCase { - static private class PublishResponder extends AbstractRunnable { + private static class PublishResponder extends AbstractRunnable { final boolean fail; final DiscoveryNode node; diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index 7f89acd169e..13e19e84978 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -170,7 +170,7 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { return nodes; } - final static Settings DEFAULT_SETTINGS = Settings.builder() + static final Settings DEFAULT_SETTINGS = Settings.builder() .put(FaultDetection.PING_TIMEOUT_SETTING.getKey(), "1s") // for hitting simulated network failures quickly .put(FaultDetection.PING_RETRIES_SETTING.getKey(), "1") // for hitting simulated network failures quickly .put("discovery.zen.join_timeout", "10s") // still long to induce failures but to long so test won't time out diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java index 135352343b6..15c8d312952 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/NodeJoinControllerTests.java @@ -581,7 +581,7 @@ public class NodeJoinControllerTests extends ESTestCase { } } - final static AtomicInteger joinId = new AtomicInteger(); + static final AtomicInteger joinId = new AtomicInteger(); private SimpleFuture joinNodeAsync(final DiscoveryNode node) throws InterruptedException { final SimpleFuture future = new SimpleFuture("join of " + node + " (id [" + joinId.incrementAndGet() + "]"); diff --git a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java index 4f65c5fafdd..1b62f5d330a 100644 --- a/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java +++ b/core/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.gateway; -import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -40,7 +39,7 @@ public class GatewayServiceTests extends ESTestCase { .put("http.enabled", "false") .put("discovery.type", "local") .put(settings.build()).build(), - null, clusterService, null, null, null, null, new NoopDiscovery(), null, null); + null, clusterService, null, null, null, new NoopDiscovery(), null, null); } public void testDefaultRecoverAfterTime() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java index d6e8d61a7a6..1e35bcdd469 100644 --- a/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/gateway/RecoverAfterNodesIT.java @@ -36,7 +36,7 @@ import static org.hamcrest.Matchers.hasItem; @ClusterScope(scope = Scope.TEST, numDataNodes = 0) public class RecoverAfterNodesIT extends ESIntegTestCase { - private final static TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); + private static final TimeValue BLOCK_WAIT_TIMEOUT = TimeValue.timeValueSeconds(10); public Set waitForNoBlocksOnNode(TimeValue timeout, Client nodeClient) throws InterruptedException { long start = System.currentTimeMillis(); diff --git a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java index 21bf69fd8d3..5413467c2a4 100644 --- a/core/src/test/java/org/elasticsearch/http/HttpServerTests.java +++ b/core/src/test/java/org/elasticsearch/http/HttpServerTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.http; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.breaker.CircuitBreaker; -import org.elasticsearch.common.bytes.ByteBufferBytesReference; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -205,7 +204,7 @@ public class HttpServerTests extends ESTestCase { private TestRestRequest(String path, String content) { this.path = path; - this.content = new ByteBufferBytesReference(ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8))); + this.content = new BytesArray(content); } @Override diff --git a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 2769534aee0..91e9bb2c016 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -193,7 +193,6 @@ public class IndexModuleTests extends ESTestCase { IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings); IndexModule module = new IndexModule(indexSettings, null, new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap())); - Consumer listener = (s) -> {}; module.addIndexEventListener(eventListener); IndexService indexService = module.newIndexService(nodeEnvironment, deleter, nodeServicesProvider, indicesQueryCache, mapperRegistry, new IndicesFieldDataCache(settings, this.listener)); diff --git a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java index c41051ec59c..f95e8408a87 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java +++ b/core/src/test/java/org/elasticsearch/index/IndexRequestBuilderIT.java @@ -22,6 +22,7 @@ package org.elasticsearch.index; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; @@ -43,7 +44,7 @@ public class IndexRequestBuilderIT extends ESIntegTestCase { client().prepareIndex("test", "test").setSource("{\"test_field\" : \"foobar\"}"), client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}")), - client().prepareIndex("test", "test").setSource(new BytesArray("{\"test_field\" : \"foobar\"}").toBytes()), + client().prepareIndex("test", "test").setSource(BytesReference.toBytes(new BytesArray("{\"test_field\" : \"foobar\"}"))), client().prepareIndex("test", "test").setSource(map) }; indexRandom(true, builders); diff --git a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java index 97258b12a3b..22324e1ff2b 100644 --- a/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/index/IndexServiceTests.java @@ -77,7 +77,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testFilteringAliases() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); add(indexService, "all", null); @@ -101,7 +100,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testAliasFilters() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); @@ -118,7 +116,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testRemovedAliasFilter() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); remove(indexService, "cats"); @@ -132,7 +129,6 @@ public class IndexServiceTests extends ESSingleNodeTestCase { public void testUnknownAliasFilter() throws Exception { IndexService indexService = createIndex("test", Settings.EMPTY); - IndexShard shard = indexService.getShard(0); add(indexService, "cats", filter(termQuery("animal", "cat"))); add(indexService, "dogs", filter(termQuery("animal", "dog"))); diff --git a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java index 000d2509ea8..8e2b8f68963 100644 --- a/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java +++ b/core/src/test/java/org/elasticsearch/index/SettingsListenerIT.java @@ -45,8 +45,6 @@ public class SettingsListenerIT extends ESIntegTestCase { public static class SettingsListenerPlugin extends Plugin { private final SettingsTestingService service = new SettingsTestingService(); - private static final Setting SETTING = Setting.intSetting("index.test.new.setting", 0, - Property.Dynamic, Property.IndexScope); @Override public List> getSettings() { diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index a0396b7abc6..0ecf8462651 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -539,7 +539,7 @@ public class InternalEngineTests extends ESTestCase { public void testCommitStats() { Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -716,7 +716,7 @@ public class InternalEngineTests extends ESTestCase { // create a document Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -729,7 +729,7 @@ public class InternalEngineTests extends ESTestCase { // but, we can still get it (in realtime) Engine.GetResult getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_1)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -755,7 +755,7 @@ public class InternalEngineTests extends ESTestCase { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); engine.index(new Engine.Index(newUid("1"), doc)); @@ -769,7 +769,7 @@ public class InternalEngineTests extends ESTestCase { // but, we can still get it (in realtime) getResult = engine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_2)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -808,7 +808,7 @@ public class InternalEngineTests extends ESTestCase { // add it back document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); engine.index(new Engine.Index(newUid("1"), doc, Versions.MATCH_DELETED)); diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index ef443d1e102..39112ed602e 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -500,7 +500,7 @@ public class ShadowEngineTests extends ESTestCase { public void testShadowEngineIgnoresWriteOperations() throws Exception { // create a document ParseContext.Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); try { replicaEngine.index(new Engine.Index(newUid("1"), doc)); @@ -538,7 +538,7 @@ public class ShadowEngineTests extends ESTestCase { // Now, add a document to the primary so we can test shadow engine deletes document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); primaryEngine.flush(); @@ -593,7 +593,7 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -612,7 +612,7 @@ public class ShadowEngineTests extends ESTestCase { // but, we can still get it (in realtime) Engine.GetResult getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_1.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_1)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -649,7 +649,7 @@ public class ShadowEngineTests extends ESTestCase { // now do an update document = testDocument(); document.add(new TextField("value", "test1", Field.Store.YES)); - document.add(new Field(SourceFieldMapper.NAME, B_2.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_2), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_2, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -663,7 +663,7 @@ public class ShadowEngineTests extends ESTestCase { // but, we can still get it (in realtime) getResult = primaryEngine.get(new Engine.Get(true, newUid("1"))); assertThat(getResult.exists(), equalTo(true)); - assertThat(getResult.source().source.toBytesArray(), equalTo(B_2.toBytesArray())); + assertThat(getResult.source().source, equalTo(B_2)); assertThat(getResult.docIdAndVersion(), nullValue()); getResult.release(); @@ -720,7 +720,7 @@ public class ShadowEngineTests extends ESTestCase { // add it back document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); primaryEngine.index(new Engine.Index(newUid("1"), doc)); @@ -971,7 +971,7 @@ public class ShadowEngineTests extends ESTestCase { // create a document ParseContext.Document document = testDocumentWithTextField(); - document.add(new Field(SourceFieldMapper.NAME, B_1.toBytes(), SourceFieldMapper.Defaults.FIELD_TYPE)); + document.add(new Field(SourceFieldMapper.NAME, BytesReference.toBytes(B_1), SourceFieldMapper.Defaults.FIELD_TYPE)); ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null); pEngine.index(new Engine.Index(newUid("1"), doc)); pEngine.flush(true, true); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java index 68e59527982..37d0436c9db 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/FieldTypeTestCase.java @@ -32,7 +32,7 @@ import java.util.List; public abstract class FieldTypeTestCase extends ESTestCase { /** Abstraction for mutating a property of a MappedFieldType */ - public static abstract class Modifier { + public abstract static class Modifier { /** The name of the property that is being modified. Used in test failure messages. */ public final String property; /** true if this modifier only makes types incompatible in strict mode, false otherwise */ diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 165b49d3145..817dc6e50df 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -292,17 +293,17 @@ public class SimpleAllMapperTests extends ESSingleNodeTestCase { } DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); - String mapping = mappingBuilder.endObject().endObject().bytes().toUtf8(); + String mapping = mappingBuilder.endObject().endObject().bytes().utf8ToString(); logger.info("Mapping: {}", mapping); DocumentMapper docMapper = parser.parse("test", new CompressedXContent(mapping)); String builtMapping = docMapper.mappingSource().string(); // reparse it DocumentMapper builtDocMapper = parser.parse("test", new CompressedXContent(builtMapping)); - byte[] json = jsonBuilder().startObject() + byte[] json = BytesReference.toBytes(jsonBuilder().startObject() .field("foo", "bar") .field("foobar", "foobar") - .endObject().bytes().toBytes(); + .endObject().bytes()); Document doc = builtDocMapper.parse("test", "test", "1", new BytesArray(json)).rootDoc(); IndexableField[] fields = doc.getFields("_all"); if (enabled) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java index fc8e2ba1872..4bf1d0c68f7 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/binary/BinaryMappingTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper.binary; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.compress.CompressorFactory; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -79,7 +80,7 @@ public class BinaryMappingTests extends ESSingleNodeTestCase { try (StreamOutput compressed = CompressorFactory.COMPRESSOR.streamOutput(out)) { new BytesArray(binaryValue1).writeTo(compressed); } - final byte[] binaryValue2 = out.bytes().toBytes(); + final byte[] binaryValue2 = BytesReference.toBytes(out.bytes()); assertTrue(CompressorFactory.isCompressed(new BytesArray(binaryValue2))); for (byte[] value : Arrays.asList(binaryValue1, binaryValue2)) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index 78da5abb746..8f38e2be576 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -297,7 +297,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { out.close(); BytesReference bytes = out.bytes(); - MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes)); + MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput()); assertThat(metaData, is(expected)); } @@ -314,7 +314,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { out.close(); BytesReference bytes = out.bytes(); - MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes)); + MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput()); assertThat(metaData, is(expected)); } @@ -331,7 +331,7 @@ public class TimestampMappingTests extends ESSingleNodeTestCase { out.close(); BytesReference bytes = out.bytes(); - MappingMetaData metaData = MappingMetaData.PROTO.readFrom(StreamInput.wrap(bytes)); + MappingMetaData metaData = MappingMetaData.PROTO.readFrom(bytes.streamInput()); assertThat(metaData, is(expected)); } diff --git a/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java b/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java index 695330c21e2..667efbc8bac 100644 --- a/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/CombineFunctionTests.java @@ -40,41 +40,41 @@ public class CombineFunctionTests extends ESTestCase { public void testWriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.MULTIPLY.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.REPLACE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.SUM.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(2)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.AVG.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(3)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.MIN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(4)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { CombineFunction.MAX.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(5)); } } @@ -83,37 +83,37 @@ public class CombineFunctionTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MULTIPLY)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.REPLACE)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(2); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.SUM)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(3); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.AVG)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(4); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MIN)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(5); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(CombineFunction.readFromStream(in), equalTo(CombineFunction.MAX)); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 6460d8505ee..090aa906456 100644 --- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -381,7 +381,7 @@ public class InnerHitBuilderTests extends ESTestCase { private static InnerHitBuilder serializedCopy(InnerHitBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new InnerHitBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java index 91a42d70809..3c5bfed86dd 100644 --- a/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/MoreLikeThisQueryBuilderTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.search.MoreLikeThisQuery; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -208,7 +207,7 @@ public class MoreLikeThisQueryBuilderTests extends AbstractQueryTestCase indexMapping = Collections.singletonMap("type", "{ \"type\": {} }"); - protected final static RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() { + private final Index index = new Index("test", "uuid"); + private final ShardId shardId = new ShardId(index, 0); + private final Map indexMapping = Collections.singletonMap("type", "{ \"type\": {} }"); + protected static final RecoveryTargetService.RecoveryListener recoveryListener = new RecoveryTargetService.RecoveryListener() { @Override public void onRecoveryDone(RecoveryState state) { diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 40a23ee66cf..862be713030 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -653,7 +653,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { if (randomBoolean() || true) { // try to serialize it to ensure values survive the serialization BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); stats = ShardStats.readShardStats(in); } XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -1442,7 +1442,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { DiscoveryNode localNode = new DiscoveryNode("foo", DummyTransportAddress.INSTANCE, emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("for testing", new RecoveryState(newShard.shardId(), routing.primary(), RecoveryState.Type.REPLICA, localNode, localNode)); List operations = new ArrayList<>(); - operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes())); + operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes()))); newShard.prepareForIndexRecovery(); newShard.recoveryState().getTranslog().totalOperations(operations.size()); newShard.skipTranslogRecovery(); @@ -1500,7 +1500,7 @@ public class IndexShardTests extends ESSingleNodeTestCase { // Shard is still inactive since we haven't started recovering yet assertFalse(newShard.isActive()); List operations = new ArrayList<>(); - operations.add(new Translog.Index("testtype", "1", jsonBuilder().startObject().field("foo", "bar").endObject().bytes().toBytes())); + operations.add(new Translog.Index("testtype", "1", BytesReference.toBytes(jsonBuilder().startObject().field("foo", "bar").endObject().bytes()))); newShard.prepareForIndexRecovery(); newShard.skipTranslogRecovery(); // Shard is still inactive since we haven't started recovering yet diff --git a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java index 70eacaafedb..abaebb88c5e 100644 --- a/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java +++ b/core/src/test/java/org/elasticsearch/index/snapshots/blobstore/FileInfoTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.snapshots.blobstore; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Version; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -52,7 +53,7 @@ public class FileInfoTests extends ESTestCase { BlobStoreIndexShardSnapshot.FileInfo info = new BlobStoreIndexShardSnapshot.FileInfo("_foobar", meta, size); XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); BlobStoreIndexShardSnapshot.FileInfo.toXContent(info, builder, ToXContent.EMPTY_PARAMS); - byte[] xcontent = shuffleXContent(builder).bytes().toBytes(); + byte[] xcontent = BytesReference.toBytes(shuffleXContent(builder).bytes()); final BlobStoreIndexShardSnapshot.FileInfo parsedInfo; try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(xcontent)) { @@ -111,7 +112,7 @@ public class FileInfoTests extends ESTestCase { builder.field(FileInfo.WRITTEN_BY, Version.LATEST.toString()); builder.field(FileInfo.CHECKSUM, "666"); builder.endObject(); - byte[] xContent = builder.bytes().toBytes(); + byte[] xContent = BytesReference.toBytes(builder.bytes()); if (failure == null) { // No failures should read as usual diff --git a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java index 6508336d9f8..84d50c6620f 100644 --- a/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java +++ b/core/src/test/java/org/elasticsearch/index/store/CorruptedFileIT.java @@ -22,6 +22,7 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse; @@ -210,7 +211,7 @@ public class CorruptedFileIT extends ESIntegTestCase { out.flush(); CheckIndex.Status status = checkIndex.checkIndex(); if (!status.clean) { - logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); + logger.warn("check index [failure]\n{}", os.bytes().utf8ToString()); throw new IOException("index check failure"); } } @@ -346,7 +347,7 @@ public class CorruptedFileIT extends ESIntegTestCase { public void sendRequest(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { if (corrupt.get() && action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; - byte[] array = req.content().array(); + byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes; int i = randomIntBetween(0, req.content().length() - 1); array[i] = (byte) ~array[i]; // flip one byte in the content hasCorrupted.countDown(); @@ -419,10 +420,12 @@ public class CorruptedFileIT extends ESIntegTestCase { if (action.equals(RecoveryTargetService.Actions.FILE_CHUNK)) { RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request; if (truncate && req.length() > 1) { - BytesArray array = new BytesArray(req.content().array(), req.content().arrayOffset(), (int) req.length() - 1); + BytesRef bytesRef = req.content().toBytesRef(); + BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1); request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos()); } else { - byte[] array = req.content().array(); + assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!"; + final byte[] array = req.content().toBytesRef().bytes; int i = randomIntBetween(0, req.content().length() - 1); array[i] = (byte) ~array[i]; // flip one byte in the content } diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index c72a6ecef27..01eead9c96b 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -60,9 +60,7 @@ import java.nio.charset.Charset; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.InvalidPathException; -import java.nio.file.OpenOption; import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collection; @@ -217,24 +215,24 @@ public class TranslogTests extends ESTestCase { Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2})); assertThat(loc2, greaterThan(loc1)); assertThat(translog.getLastWriteLocation(), greaterThan(loc2)); - assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1}))); - assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2}))); + assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1}))); + assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2}))); Translog.Location lastLocBeforeSync = translog.getLastWriteLocation(); translog.sync(); assertEquals(lastLocBeforeSync, translog.getLastWriteLocation()); - assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1}))); - assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2}))); + assertThat(translog.read(loc1).getSource().source, equalTo(new BytesArray(new byte[]{1}))); + assertThat(translog.read(loc2).getSource().source, equalTo(new BytesArray(new byte[]{2}))); Translog.Location loc3 = translog.add(new Translog.Index("test", "2", new byte[]{3})); assertThat(loc3, greaterThan(loc2)); assertThat(translog.getLastWriteLocation(), greaterThan(loc3)); - assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3}))); lastLocBeforeSync = translog.getLastWriteLocation(); translog.sync(); assertEquals(lastLocBeforeSync, translog.getLastWriteLocation()); - assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3}))); translog.prepareCommit(); /* * The commit adds to the lastWriteLocation even though is isn't really a write. This is just an implementation artifact but it can @@ -242,7 +240,7 @@ public class TranslogTests extends ESTestCase { * and less than the location of the next write operation. */ assertThat(translog.getLastWriteLocation(), greaterThan(lastLocBeforeSync)); - assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + assertThat(translog.read(loc3).getSource().source, equalTo(new BytesArray(new byte[]{3}))); translog.commit(); assertNull(translog.read(loc1)); assertNull(translog.read(loc2)); @@ -274,7 +272,7 @@ public class TranslogTests extends ESTestCase { Translog.Index index = (Translog.Index) snapshot.next(); assertThat(index != null, equalTo(true)); - assertThat(index.source().toBytes(), equalTo(new byte[]{1})); + assertThat(BytesReference.toBytes(index.source()), equalTo(new byte[]{1})); Translog.Delete delete = (Translog.Delete) snapshot.next(); assertThat(delete != null, equalTo(true)); @@ -303,7 +301,7 @@ public class TranslogTests extends ESTestCase { if (randomBoolean()) { BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); stats = new TranslogStats(); stats.readFrom(in); } @@ -350,7 +348,7 @@ public class TranslogTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); total.writeTo(out); TranslogStats copy = new TranslogStats(); - copy.readFrom(StreamInput.wrap(out.bytes())); + copy.readFrom(out.bytes().streamInput()); assertEquals(6, copy.estimatedNumberOfOperations()); assertEquals(431, copy.getTranslogSizeInBytes()); @@ -827,7 +825,7 @@ public class TranslogTests extends ESTestCase { assertEquals(max.generation, translog.currentFileGeneration()); final Translog.Operation read = translog.read(max); - assertEquals(read.getSource().source.toUtf8(), Integer.toString(count)); + assertEquals(read.getSource().source.utf8ToString(), Integer.toString(count)); } public static Translog.Location max(Translog.Location a, Translog.Location b) { @@ -859,7 +857,7 @@ public class TranslogTests extends ESTestCase { Translog.Location location = locations.get(op); if (op <= lastSynced) { final Translog.Operation read = reader.read(location); - assertEquals(Integer.toString(op), read.getSource().source.toUtf8()); + assertEquals(Integer.toString(op), read.getSource().source.utf8ToString()); } else { try { reader.read(location); @@ -995,7 +993,7 @@ public class TranslogTests extends ESTestCase { assertEquals("expected operation" + i + " to be in the previous translog but wasn't", translog.currentFileGeneration() - 1, locations.get(i).generation); Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals(i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1030,7 +1028,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } if (randomBoolean()) { // recover twice @@ -1043,7 +1041,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1084,7 +1082,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } @@ -1099,7 +1097,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1143,7 +1141,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < upTo; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null synced: " + sync, next); - assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch, synced: " + sync, i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1157,7 +1155,7 @@ public class TranslogTests extends ESTestCase { ops.add(test); } Translog.writeOperations(out, ops); - final List readOperations = Translog.readOperations(StreamInput.wrap(out.bytes())); + final List readOperations = Translog.readOperations(out.bytes().streamInput()); assertEquals(ops.size(), readOperations.size()); assertEquals(ops, readOperations); } @@ -1218,7 +1216,7 @@ public class TranslogTests extends ESTestCase { for (int i = firstUncommitted; i < translogOperations; i++) { Translog.Operation next = snapshot.next(); assertNotNull("" + i, next); - assertEquals(Integer.parseInt(next.getSource().source.toUtf8()), i); + assertEquals(Integer.parseInt(next.getSource().source.utf8ToString()), i); } assertNull(snapshot.next()); } @@ -1392,7 +1390,7 @@ public class TranslogTests extends ESTestCase { assertEquals("expected operation" + i + " to be in the previous translog but wasn't", tlog.currentFileGeneration() - 1, locations.get(i).generation); Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals(i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals(i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1716,7 +1714,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < 1; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } @@ -1727,7 +1725,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < 2; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } } } @@ -1771,7 +1769,7 @@ public class TranslogTests extends ESTestCase { for (int i = 0; i < 1; i++) { Translog.Operation next = snapshot.next(); assertNotNull("operation " + i + " must be non-null", next); - assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.toUtf8())); + assertEquals("payload missmatch", i, Integer.parseInt(next.getSource().source.utf8ToString())); } tlog.add(new Translog.Index("test", "" + 1, Integer.toString(1).getBytes(Charset.forName("UTF-8")))); } @@ -1870,7 +1868,7 @@ public class TranslogTests extends ESTestCase { assertEquals(syncedDocs.size(), snapshot.totalOperations()); for (int i = 0; i < syncedDocs.size(); i++) { Translog.Operation next = snapshot.next(); - assertEquals(syncedDocs.get(i), next.getSource().source.toUtf8()); + assertEquals(syncedDocs.get(i), next.getSource().source.utf8ToString()); assertNotNull("operation " + i + " must be non-null", next); } } diff --git a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index 1f1b758f349..995beb1742c 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -70,8 +70,7 @@ public class IndexingMemoryControllerTests extends ESSingleNodeTestCase { super(Settings.builder() .put("indices.memory.interval", "200h") // disable it .put(settings) - .build(), - null, null, 100 * 1024 * 1024); // fix jvm mem size to 100mb + .build(), null, null); } public void deleteShard(IndexShard shard) { diff --git a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java index 1cca3bb7215..d43217d9785 100644 --- a/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java +++ b/core/src/test/java/org/elasticsearch/indices/IndicesRequestCacheTests.java @@ -62,7 +62,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // initial cache TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -72,7 +72,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // cache hit entity = new TestEntity(requestCacheStats, reader, indexShard, 0); value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -117,7 +117,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // initial cache TestEntity entity = new TestEntity(requestCacheStats, reader, indexShard, 0); BytesReference value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(1, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -130,7 +130,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // cache the second TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value).readString()); + assertEquals("bar", value.streamInput().readString()); assertEquals(0, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -141,7 +141,7 @@ public class IndicesRequestCacheTests extends ESTestCase { secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); value = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value).readString()); + assertEquals("bar", value.streamInput().readString()); assertEquals(1, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -150,7 +150,7 @@ public class IndicesRequestCacheTests extends ESTestCase { entity = new TestEntity(requestCacheStats, reader, indexShard, 0); value = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value).readString()); + assertEquals("foo", value.streamInput().readString()); assertEquals(2, requestCacheStats.stats().getHitCount()); assertEquals(2, requestCacheStats.stats().getMissCount()); assertEquals(0, requestCacheStats.stats().getEvictions()); @@ -209,9 +209,9 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity secondEntity = new TestEntity(requestCacheStats, secondReader, indexShard, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value1).readString()); + assertEquals("foo", value1.streamInput().readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value2).readString()); + assertEquals("bar", value2.streamInput().readString()); size = requestCacheStats.stats().getMemorySize(); IOUtils.close(reader, secondReader, writer, dir, cache); } @@ -240,12 +240,12 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, indexShard, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value1).readString()); + assertEquals("foo", value1.streamInput().readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value2).readString()); + assertEquals("bar", value2.streamInput().readString()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); - assertEquals("baz", StreamInput.wrap(value3).readString()); + assertEquals("baz", value3.streamInput().readString()); assertEquals(2, cache.count()); assertEquals(1, requestCacheStats.stats().getEvictions()); IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache); @@ -277,12 +277,12 @@ public class IndicesRequestCacheTests extends ESTestCase { TestEntity thirddEntity = new TestEntity(requestCacheStats, thirdReader, differentIdentity, 0); BytesReference value1 = cache.getOrCompute(entity, reader, termQuery.buildAsBytes()); - assertEquals("foo", StreamInput.wrap(value1).readString()); + assertEquals("foo", value1.streamInput().readString()); BytesReference value2 = cache.getOrCompute(secondEntity, secondReader, termQuery.buildAsBytes()); - assertEquals("bar", StreamInput.wrap(value2).readString()); + assertEquals("bar", value2.streamInput().readString()); logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize()); BytesReference value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); - assertEquals("baz", StreamInput.wrap(value3).readString()); + assertEquals("baz", value3.streamInput().readString()); assertEquals(3, cache.count()); final long hitCount = requestCacheStats.stats().getHitCount(); // clear all for the indexShard Idendity even though is't still open @@ -292,7 +292,7 @@ public class IndicesRequestCacheTests extends ESTestCase { // third has not been validated since it's a different identity value3 = cache.getOrCompute(thirddEntity, thirdReader, termQuery.buildAsBytes()); assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount()); - assertEquals("baz", StreamInput.wrap(value3).readString()); + assertEquals("baz", value3.streamInput().readString()); IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache); diff --git a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java index 59d86ddce67..fea69133377 100644 --- a/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java +++ b/core/src/test/java/org/elasticsearch/indices/TermsLookupTests.java @@ -70,7 +70,7 @@ public class TermsLookupTests extends ESTestCase { TermsLookup termsLookup = randomTermsLookup(); try (BytesStreamOutput output = new BytesStreamOutput()) { termsLookup.writeTo(output); - try (StreamInput in = StreamInput.wrap(output.bytes())) { + try (StreamInput in = output.bytes().streamInput()) { TermsLookup deserializedLookup = new TermsLookup(in); assertEquals(deserializedLookup, termsLookup); assertEquals(deserializedLookup.hashCode(), termsLookup.hashCode()); diff --git a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java index 69bee510710..4bf12bd9138 100644 --- a/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java +++ b/core/src/test/java/org/elasticsearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java @@ -181,7 +181,8 @@ public abstract class AbstractIndicesClusterStateServiceTestCase extends ESTestC } @Override - public @Nullable MockIndexService indexService(Index index) { + @Nullable + public MockIndexService indexService(Index index) { return indices.get(index.getUUID()); } diff --git a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java index 8552db2d376..bcd614121b6 100644 --- a/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java +++ b/core/src/test/java/org/elasticsearch/indices/recovery/RecoveryTargetTests.java @@ -57,8 +57,8 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; public class RecoveryTargetTests extends ESTestCase { abstract class Streamer extends Thread { private T lastRead; - final private AtomicBoolean shouldStop; - final private T source; + private final AtomicBoolean shouldStop; + private final T source; final AtomicReference error = new AtomicReference<>(); final Version streamVersion; @@ -84,7 +84,7 @@ public class RecoveryTargetTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); source.writeTo(out); out.close(); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); T obj = deserialize(in); lastRead = obj; return obj; diff --git a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java index 6bce95af184..8e88aff523c 100644 --- a/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -647,7 +647,7 @@ public class IndexStatsIT extends ESIntegTestCase { flags.writeTo(out); out.close(); BytesReference bytes = out.bytes(); - CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes)); + CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(bytes.streamInput()); for (Flag flag : values) { assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag))); } @@ -661,7 +661,7 @@ public class IndexStatsIT extends ESIntegTestCase { flags.writeTo(out); out.close(); BytesReference bytes = out.bytes(); - CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(StreamInput.wrap(bytes)); + CommonStatsFlags readStats = CommonStatsFlags.readCommonStatsFlags(bytes.streamInput()); for (Flag flag : values) { assertThat(flags.isSet(flag), equalTo(readStats.isSet(flag))); } diff --git a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java index e558f0f2a12..96af4ef3671 100644 --- a/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java +++ b/core/src/test/java/org/elasticsearch/indices/store/IndicesStoreTests.java @@ -57,7 +57,7 @@ import static org.elasticsearch.test.VersionUtils.randomVersion; /** */ public class IndicesStoreTests extends ESTestCase { - private final static ShardRoutingState[] NOT_STARTED_STATES; + private static final ShardRoutingState[] NOT_STARTED_STATES; static { Set set = new HashSet<>(); diff --git a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java index 119e94580ad..9974dd568a8 100644 --- a/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/IngestStatsTests.java @@ -62,7 +62,7 @@ public class IngestStatsTests extends ESTestCase { private IngestStats serialize(IngestStats stats) throws IOException { BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); return new IngestStats(in); } } diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java index 8bf6f77a026..fcc6e04c6c1 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineExecutionServiceTests.java @@ -134,8 +134,25 @@ public class PipelineExecutionServiceTests extends ESTestCase { verify(completionHandler, times(1)).accept(true); } + public void testExecuteEmptyPipeline() throws Exception { + CompoundProcessor processor = mock(CompoundProcessor.class); + when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", processor)); + when(processor.getProcessors()).thenReturn(Collections.emptyList()); + + IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); + @SuppressWarnings("unchecked") + Consumer failureHandler = mock(Consumer.class); + @SuppressWarnings("unchecked") + Consumer completionHandler = mock(Consumer.class); + executionService.executeIndexRequest(indexRequest, failureHandler, completionHandler); + verify(processor, never()).execute(any()); + verify(failureHandler, never()).accept(any()); + verify(completionHandler, times(1)).accept(true); + } + public void testExecutePropagateAllMetaDataUpdates() throws Exception { CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class))); doAnswer((InvocationOnMock invocationOnMock) -> { IngestDocument ingestDocument = (IngestDocument) invocationOnMock.getArguments()[0]; for (IngestDocument.MetaData metaData : IngestDocument.MetaData.values()) { @@ -171,6 +188,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { public void testExecuteFailure() throws Exception { CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class))); when(store.get("_id")).thenReturn(new Pipeline("_id", "_description", processor)); IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(Collections.emptyMap()).setPipeline("_id"); doThrow(new RuntimeException()).when(processor).execute(eqID("_index", "_type", "_id", Collections.emptyMap())); @@ -313,6 +331,7 @@ public class PipelineExecutionServiceTests extends ESTestCase { } CompoundProcessor processor = mock(CompoundProcessor.class); + when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class))); Exception error = new RuntimeException(); doThrow(error).when(processor).execute(any()); when(store.get(pipelineId)).thenReturn(new Pipeline(pipelineId, null, processor)); @@ -356,8 +375,8 @@ public class PipelineExecutionServiceTests extends ESTestCase { assertThat(ingestStats.getTotalStats().getIngestFailedCount(), equalTo(0L)); assertThat(ingestStats.getTotalStats().getIngestTimeInMillis(), equalTo(0L)); - when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor())); - when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor())); + when(store.get("_id1")).thenReturn(new Pipeline("_id1", null, new CompoundProcessor(mock(Processor.class)))); + when(store.get("_id2")).thenReturn(new Pipeline("_id2", null, new CompoundProcessor(mock(Processor.class)))); Map configurationMap = new HashMap<>(); configurationMap.put("_id1", new PipelineConfiguration("_id1", new BytesArray("{}"))); diff --git a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java index f601d11c878..b09d772729c 100644 --- a/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/PipelineFactoryTests.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; @@ -69,6 +70,17 @@ public class PipelineFactoryTests extends ESTestCase { } } + public void testCreateWithEmptyProcessorsField() throws Exception { + Map pipelineConfig = new HashMap<>(); + pipelineConfig.put(Pipeline.DESCRIPTION_KEY, "_description"); + pipelineConfig.put(Pipeline.PROCESSORS_KEY, Collections.emptyList()); + Pipeline.Factory factory = new Pipeline.Factory(); + Pipeline pipeline = factory.create("_id", pipelineConfig, null); + assertThat(pipeline.getId(), equalTo("_id")); + assertThat(pipeline.getDescription(), equalTo("_description")); + assertThat(pipeline.getProcessors(), is(empty())); + } + public void testCreateWithPipelineOnFailure() throws Exception { Map processorConfig = new HashMap<>(); Map pipelineConfig = new HashMap<>(); diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java index 0d07bcf0981..739dcd8b2c6 100644 --- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java +++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java @@ -70,7 +70,7 @@ public class NodeInfoStreamingTests extends ESTestCase { out.setVersion(version); nodeInfo.writeTo(out); out.close(); - StreamInput in = StreamInput.wrap(out.bytes()); + StreamInput in = out.bytes().streamInput(); in.setVersion(version); NodeInfo readNodeInfo = NodeInfo.readNodeInfo(in); assertExpectedUnchanged(nodeInfo, readNodeInfo); @@ -81,11 +81,6 @@ public class NodeInfoStreamingTests extends ESTestCase { assertThat(nodeInfo.getBuild().toString(), equalTo(readNodeInfo.getBuild().toString())); assertThat(nodeInfo.getHostname(), equalTo(readNodeInfo.getHostname())); assertThat(nodeInfo.getVersion(), equalTo(readNodeInfo.getVersion())); - assertThat(nodeInfo.getServiceAttributes().size(), equalTo(readNodeInfo.getServiceAttributes().size())); - for (Map.Entry entry : nodeInfo.getServiceAttributes().entrySet()) { - assertNotNull(readNodeInfo.getServiceAttributes().get(entry.getKey())); - assertThat(readNodeInfo.getServiceAttributes().get(entry.getKey()), equalTo(entry.getValue())); - } compareJsonOutput(nodeInfo.getHttp(), readNodeInfo.getHttp()); compareJsonOutput(nodeInfo.getJvm(), readNodeInfo.getJvm()); compareJsonOutput(nodeInfo.getProcess(), readNodeInfo.getProcess()); @@ -149,6 +144,7 @@ public class NodeInfoStreamingTests extends ESTestCase { // pick a random long that sometimes exceeds an int: indexingBuffer = new ByteSizeValue(random().nextLong() & ((1L<<40)-1)); } - return new NodeInfo(VersionUtils.randomVersion(random()), build, node, serviceAttributes, settings, osInfo, process, jvm, threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer); + return new NodeInfo(VersionUtils.randomVersion(random()), build, node, settings, osInfo, process, jvm, + threadPoolInfo, transport, htttpInfo, plugins, ingestInfo, indexingBuffer); } } diff --git a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java index 2564b31488b..740a027aecb 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java +++ b/core/src/test/java/org/elasticsearch/recovery/RecoveriesCollectionTests.java @@ -47,7 +47,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThan; public class RecoveriesCollectionTests extends ESSingleNodeTestCase { - final static RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() { + static final RecoveryTargetService.RecoveryListener listener = new RecoveryTargetService.RecoveryListener() { @Override public void onRecoveryDone(RecoveryState state) { diff --git a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java index 620dfeb94c2..fd5da198ed2 100644 --- a/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java +++ b/core/src/test/java/org/elasticsearch/recovery/RelocationIT.java @@ -22,6 +22,7 @@ package org.elasticsearch.recovery; import com.carrotsearch.hppc.IntHashSet; import com.carrotsearch.hppc.procedures.IntProcedure; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.English; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -93,8 +94,6 @@ import static org.hamcrest.Matchers.startsWith; public class RelocationIT extends ESIntegTestCase { private final TimeValue ACCEPTABLE_RELOCATION_TIME = new TimeValue(5, TimeUnit.MINUTES); - - @Override protected Collection> nodePlugins() { return pluginList(MockTransportService.TestPlugin.class, MockIndexEventListener.TestPlugin.class); @@ -505,7 +504,8 @@ public class RelocationIT extends ESIntegTestCase { if (chunkRequest.name().startsWith(IndexFileNames.SEGMENTS)) { // corrupting the segments_N files in order to make sure future recovery re-send files logger.debug("corrupting [{}] to {}. file name: [{}]", action, node, chunkRequest.name()); - byte[] array = chunkRequest.content().array(); + assert chunkRequest.content().toBytesRef().bytes == chunkRequest.content().toBytesRef().bytes : "no internal reference!!"; + byte[] array = chunkRequest.content().toBytesRef().bytes; array[0] = (byte) ~array[0]; // flip one byte in the content corruptionCount.countDown(); } diff --git a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java index aa3b11e6250..6bb1716cb0f 100644 --- a/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java +++ b/core/src/test/java/org/elasticsearch/rest/BytesRestResponseTests.java @@ -36,7 +36,6 @@ import java.io.IOException; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Mockito.mock; @@ -64,7 +63,7 @@ public class BytesRestResponseTests extends ESTestCase { Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("ElasticsearchException[an error occurred reading data]")); assertThat(text, not(containsString("FileNotFoundException"))); assertThat(text, not(containsString("/foo/bar"))); @@ -77,7 +76,7 @@ public class BytesRestResponseTests extends ESTestCase { Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}")); assertThat(text, containsString("{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}")); } @@ -88,7 +87,7 @@ public class BytesRestResponseTests extends ESTestCase { Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, not(containsString("Throwable[an error occurred reading data]"))); assertThat(text, not(containsString("FileNotFoundException[/foo/bar]"))); assertThat(text, not(containsString("error_trace"))); @@ -102,7 +101,7 @@ public class BytesRestResponseTests extends ESTestCase { Throwable t = new Throwable("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("\"type\":\"throwable\",\"reason\":\"an error occurred reading data\"")); assertThat(text, containsString("{\"type\":\"file_not_found_exception\"")); assertThat(text, containsString("\"stack_trace\":\"[an error occurred reading data]")); @@ -114,13 +113,13 @@ public class BytesRestResponseTests extends ESTestCase { { Throwable t = new ElasticsearchException("an error occurred reading data", new FileNotFoundException("/foo/bar")); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("{\"root_cause\":[{\"type\":\"exception\",\"reason\":\"an error occurred reading data\"}]")); } { Throwable t = new FileNotFoundException("/foo/bar"); BytesRestResponse response = new BytesRestResponse(channel, t); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("{\"root_cause\":[{\"type\":\"file_not_found_exception\",\"reason\":\"/foo/bar\"}]")); } } @@ -130,7 +129,7 @@ public class BytesRestResponseTests extends ESTestCase { RestChannel channel = new SimpleExceptionRestChannel(request); BytesRestResponse response = new BytesRestResponse(channel, null); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); assertThat(text, containsString("\"error\":\"unknown\"")); assertThat(text, not(containsString("error_trace"))); } @@ -144,7 +143,7 @@ public class BytesRestResponseTests extends ESTestCase { new SearchShardTarget("node_1", new Index("foo", "_na_"), 2)); SearchPhaseExecutionException ex = new SearchPhaseExecutionException("search", "all shards failed", new ShardSearchFailure[] {failure, failure1}); BytesRestResponse response = new BytesRestResponse(channel, new RemoteTransportException("foo", ex)); - String text = response.content().toUtf8(); + String text = response.content().utf8ToString(); String expected = "{\"error\":{\"root_cause\":[{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}],\"type\":\"search_phase_execution_exception\",\"reason\":\"all shards failed\",\"phase\":\"search\",\"grouped\":true,\"failed_shards\":[{\"shard\":1,\"index\":\"foo\",\"node\":\"node_1\",\"reason\":{\"type\":\"parsing_exception\",\"reason\":\"foobar\",\"line\":1,\"col\":2}}]},\"status\":400}"; assertEquals(expected.trim(), text.trim()); String stackTrace = ExceptionsHelper.stackTrace(ex); @@ -160,7 +159,7 @@ public class BytesRestResponseTests extends ESTestCase { // if we try to decode the path, this will throw an IllegalArgumentException again final BytesRestResponse response = new BytesRestResponse(channel, e); assertNotNull(response.content()); - final String content = response.content().toUtf8(); + final String content = response.content().utf8ToString(); assertThat(content, containsString("\"type\":\"illegal_argument_exception\"")); assertThat(content, containsString("\"reason\":\"partial escape sequence at end of string: %a\"")); assertThat(content, containsString("\"status\":" + 400)); @@ -171,7 +170,7 @@ public class BytesRestResponseTests extends ESTestCase { final RestChannel channel = new DetailedExceptionRestChannel(request); final BytesRestResponse response = new BytesRestResponse(channel, new ElasticsearchException("simulated")); assertNotNull(response.content()); - final String content = response.content().toUtf8(); + final String content = response.content().utf8ToString(); assertThat(content, containsString("\"type\":\"exception\"")); assertThat(content, containsString("\"reason\":\"simulated\"")); assertThat(content, containsString("\"status\":" + 500)); diff --git a/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java new file mode 100644 index 00000000000..db2148b9f61 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.cat; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsTests; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.common.Table; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.cache.query.QueryCacheStats; +import org.elasticsearch.index.cache.request.RequestCacheStats; +import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.fielddata.FieldDataStats; +import org.elasticsearch.index.flush.FlushStats; +import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.merge.MergeStats; +import org.elasticsearch.index.refresh.RefreshStats; +import org.elasticsearch.index.search.stats.SearchStats; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.IndexingStats; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.index.warmer.WarmerStats; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.search.suggest.completion.CompletionStats; +import org.elasticsearch.test.ESTestCase; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.hamcrest.Matchers.equalTo; + +/** + * Tests for {@link RestIndicesAction} + */ +public class RestIndicesActionTests extends ESTestCase { + + public void testBuildTable() { + final Settings settings = Settings.EMPTY; + final RestController restController = new RestController(settings); + final RestIndicesAction action = new RestIndicesAction(settings, restController, new IndexNameExpressionResolver(settings)); + + // build a (semi-)random table + final int numIndices = randomIntBetween(0, 5); + Index[] indices = new Index[numIndices]; + for (int i = 0; i < numIndices; i++) { + indices[i] = new Index(randomAsciiOfLength(5), UUIDs.randomBase64UUID()); + } + + final MetaData.Builder metaDataBuilder = MetaData.builder(); + for (final Index index : indices) { + metaDataBuilder.put(IndexMetaData.builder(index.getName()) + .settings(Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetaData.SETTING_INDEX_UUID, index.getUUID())) + .creationDate(System.currentTimeMillis()) + .numberOfShards(1) + .numberOfReplicas(1) + .state(IndexMetaData.State.OPEN)); + } + final MetaData metaData = metaDataBuilder.build(); + + final ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData) + .build(); + final String[] indicesStr = new String[indices.length]; + for (int i = 0; i < indices.length; i++) { + indicesStr[i] = indices[i].getName(); + } + final ClusterHealthResponse clusterHealth = new ClusterHealthResponse( + clusterState.getClusterName().value(), indicesStr, clusterState, 0, 0, 0, TimeValue.timeValueMillis(1000L) + ); + + final Table table = action.buildTable(null, indices, clusterHealth, randomIndicesStatsResponse(indices), metaData); + + // now, verify the table is correct + int count = 0; + List headers = table.getHeaders(); + assertThat(headers.get(count++).value, equalTo("health")); + assertThat(headers.get(count++).value, equalTo("status")); + assertThat(headers.get(count++).value, equalTo("index")); + assertThat(headers.get(count++).value, equalTo("uuid")); + + List> rows = table.getRows(); + assertThat(rows.size(), equalTo(indices.length)); + // TODO: more to verify (e.g. randomize cluster health, num primaries, num replicas, etc) + for (int i = 0; i < rows.size(); i++) { + count = 0; + final List row = rows.get(i); + assertThat(row.get(count++).value, equalTo("red*")); // all are red because cluster state doesn't have routing entries + assertThat(row.get(count++).value, equalTo("open")); // all are OPEN for now + assertThat(row.get(count++).value, equalTo(indices[i].getName())); + assertThat(row.get(count++).value, equalTo(indices[i].getUUID())); + } + } + + private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) { + List shardStats = new ArrayList<>(); + for (final Index index : indices) { + for (int i = 0; i < 2; i++) { + ShardId shardId = new ShardId(index, i); + Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i)); + ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, null, i == 0, + new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE); + shardRouting = shardRouting.moveToStarted(); + CommonStats stats = new CommonStats(); + stats.fieldData = new FieldDataStats(); + stats.queryCache = new QueryCacheStats(); + stats.docs = new DocsStats(); + stats.store = new StoreStats(); + stats.indexing = new IndexingStats(); + stats.search = new SearchStats(); + stats.segments = new SegmentsStats(); + stats.merge = new MergeStats(); + stats.refresh = new RefreshStats(); + stats.completion = new CompletionStats(); + stats.requestCache = new RequestCacheStats(); + stats.get = new GetStats(); + stats.flush = new FlushStats(); + stats.warmer = new WarmerStats(); + shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null)); + } + } + return IndicesStatsTests.newIndicesStatsResponse( + shardStats.toArray(new ShardStats[shardStats.size()]), shardStats.size(), shardStats.size(), 0, emptyList() + ); + } +} diff --git a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java index ebb7dd255aa..ffefa074df7 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/main/RestMainActionTests.java @@ -95,6 +95,6 @@ public class RestMainActionTests extends ESTestCase { } mainResponse.toXContent(responseBuilder, ToXContent.EMPTY_PARAMS); BytesReference xcontentBytes = responseBuilder.bytes(); - assertTrue(BytesReference.Helper.bytesEqual(xcontentBytes, response.content())); + assertEquals(xcontentBytes, response.content()); } } diff --git a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java index a7e17785d48..3dfae8cc4f8 100644 --- a/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java +++ b/core/src/test/java/org/elasticsearch/rest/action/support/RestTableTests.java @@ -169,7 +169,7 @@ public class RestTableTests extends ESTestCase { private void assertResponse(Map headers, String mediaType, String body) throws Exception { RestResponse response = assertResponseContentType(headers, mediaType); - assertThat(response.content().toUtf8(), equalTo(body)); + assertThat(response.content().utf8ToString(), equalTo(body)); } private List getHeaderNames(List headers) { diff --git a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java index d8d6b0f5409..658a3bf5658 100644 --- a/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java +++ b/core/src/test/java/org/elasticsearch/script/StoredScriptsIT.java @@ -29,8 +29,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke public class StoredScriptsIT extends ESIntegTestCase { - private final static int SCRIPT_MAX_SIZE_IN_BYTES = 64; - private final static String LANG = MockScriptEngine.NAME; + private static final int SCRIPT_MAX_SIZE_IN_BYTES = 64; + private static final String LANG = MockScriptEngine.NAME; @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index 2ca255ea1a3..192f40d4b2b 100644 --- a/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/core/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -43,13 +43,13 @@ public class DocValueFormatTests extends ESTestCase { BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.BOOLEAN); - StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.BOOLEAN, in.readNamedWriteable(DocValueFormat.class)); DocValueFormat.Decimal decimalFormat = new DocValueFormat.Decimal("###.##"); out = new BytesStreamOutput(); out.writeNamedWriteable(decimalFormat); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); DocValueFormat vf = in.readNamedWriteable(DocValueFormat.class); assertEquals(DocValueFormat.Decimal.class, vf.getClass()); assertEquals("###.##", ((DocValueFormat.Decimal) vf).pattern); @@ -57,7 +57,7 @@ public class DocValueFormatTests extends ESTestCase { DocValueFormat.DateTime dateFormat = new DocValueFormat.DateTime(Joda.forPattern("epoch_second"), DateTimeZone.forOffsetHours(1)); out = new BytesStreamOutput(); out.writeNamedWriteable(dateFormat); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); vf = in.readNamedWriteable(DocValueFormat.class); assertEquals(DocValueFormat.DateTime.class, vf.getClass()); assertEquals("epoch_second", ((DocValueFormat.DateTime) vf).formatter.format()); @@ -65,17 +65,17 @@ public class DocValueFormatTests extends ESTestCase { out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.GEOHASH); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.GEOHASH, in.readNamedWriteable(DocValueFormat.class)); out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.IP); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.IP, in.readNamedWriteable(DocValueFormat.class)); out = new BytesStreamOutput(); out.writeNamedWriteable(DocValueFormat.RAW); - in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class)); } diff --git a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java index a4837e382ac..5caba0fb441 100644 --- a/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java +++ b/core/src/test/java/org/elasticsearch/search/MultiValueModeTests.java @@ -753,35 +753,35 @@ public class MultiValueModeTests extends ESTestCase { public void testWriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.SUM.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.AVG.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.MEDIAN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(2)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.MIN.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(3)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { MultiValueMode.MAX.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(4)); } } @@ -790,35 +790,35 @@ public class MultiValueModeTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.SUM)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.AVG)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(2); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MEDIAN)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(3); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MIN)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(4); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(MultiValueMode.readMultiValueModeFrom(in), equalTo(MultiValueMode.MAX)); } } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java index a4103e7ee56..ddde1fd9eb6 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/BaseAggregationTestCase.java @@ -215,7 +215,7 @@ public abstract class BaseAggregationTestCase categoryToControl = new HashMap<>(); + private static final Map categoryToControl = new HashMap<>(); @Override public void setupSuiteScopeCluster() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java index 36c4caae12d..97d5cf1f9ee 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/percentiles/PercentilesMethodTests.java @@ -38,14 +38,14 @@ public class PercentilesMethodTests extends ESTestCase { public void testwriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { PercentilesMethod.TDIGEST.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { PercentilesMethod.HDR.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } @@ -54,13 +54,13 @@ public class PercentilesMethodTests extends ESTestCase { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(PercentilesMethod.readFromStream(in), equalTo(PercentilesMethod.TDIGEST)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(PercentilesMethod.readFromStream(in), equalTo(PercentilesMethod.HDR)); } } @@ -69,7 +69,7 @@ public class PercentilesMethodTests extends ESTestCase { public void testInvalidReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { PercentilesMethod.readFromStream(in); fail("Expected IOException"); } catch(IOException e) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java index e1441b0dc54..390501d2002 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/pipeline/bucketmetrics/ExtendedStatsBucketTests.java @@ -22,7 +22,6 @@ package org.elasticsearch.search.aggregations.pipeline.bucketmetrics; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; -import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.bucketmetrics.stats.extended.ExtendedStatsBucketPipelineAggregationBuilder; import static org.hamcrest.Matchers.equalTo; @@ -51,7 +50,7 @@ public class ExtendedStatsBucketTests extends AbstractBucketMetricsTestCase CONTEXT_FACTORY = new ContextFactory() { diff --git a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java index 14d0fc959c3..4ba1b902fef 100644 --- a/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java +++ b/core/src/test/java/org/elasticsearch/search/functionscore/QueryRescorerIT.java @@ -355,7 +355,7 @@ public class QueryRescorerIT extends ESIntegTestCase { // and shard id are equal during merging shard results. // This comparator uses a custom tie in case the scores are equal, so that both regular hits and rescored hits // are sorted equally. This is fine since tests only care about the fact the scores should be equal, not ordering. - private final static Comparator searchHitsComparator = new Comparator() { + private static final Comparator searchHitsComparator = new Comparator() { @Override public int compare(SearchHit hit1, SearchHit hit2) { int cmp = Float.compare(hit2.getScore(), hit1.getScore()); diff --git a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java index 930a7b220e0..1e3c5453fd7 100644 --- a/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/highlight/HighlightBuilderTests.java @@ -484,14 +484,14 @@ public class HighlightBuilderTests extends ESTestCase { public void testOrderSerialization() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { Order.NONE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { Order.SCORE.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat(in.readVInt(), equalTo(1)); } } @@ -738,7 +738,7 @@ public class HighlightBuilderTests extends ESTestCase { private static HighlightBuilder serializedCopy(HighlightBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new HighlightBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java index 77fc2f0e6a9..dedd47d3e43 100644 --- a/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java +++ b/core/src/test/java/org/elasticsearch/search/internal/InternalSearchHitTests.java @@ -67,7 +67,7 @@ public class InternalSearchHitTests extends ESTestCase { context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM); BytesStreamOutput output = new BytesStreamOutput(); hits.writeTo(output, context); - InputStream input = new ByteArrayInputStream(output.bytes().toBytes()); + InputStream input = output.bytes().streamInput(); context = new InternalSearchHits.StreamContext(); context.streamShardTarget(InternalSearchHits.StreamContext.ShardTargetType.STREAM); InternalSearchHits results = InternalSearchHits.readSearchHits(new InputStreamStreamInput(input), context); diff --git a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java index ef7a4ecc7ce..300c4f141b0 100644 --- a/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java +++ b/core/src/test/java/org/elasticsearch/search/matchedqueries/MatchedQueriesIT.java @@ -362,8 +362,8 @@ public class MatchedQueriesIT extends ESIntegTestCase { refresh(); QueryBuilder[] queries = new QueryBuilder[]{ - wrapperQuery(matchQuery("content", "amet").queryName("abc").buildAsBytes().toUtf8()), - constantScoreQuery(wrapperQuery(termQuery("content", "amet").queryName("abc").buildAsBytes().toUtf8())) + wrapperQuery(matchQuery("content", "amet").queryName("abc").buildAsBytes().utf8ToString()), + constantScoreQuery(wrapperQuery(termQuery("content", "amet").queryName("abc").buildAsBytes().utf8ToString())) }; for (QueryBuilder query : queries) { SearchResponse searchResponse = client().prepareSearch() diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index 7c3690dcf35..28eb56bdcaf 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -340,7 +340,7 @@ public class QueryRescoreBuilderTests extends ESTestCase { private static RescoreBuilder serializedCopy(RescoreBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.writeNamedWriteable(original); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return in.readNamedWriteable(RescoreBuilder.class); } } diff --git a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java index 3c675926328..7f9b9761feb 100644 --- a/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/searchafter/SearchAfterBuilderTests.java @@ -164,7 +164,7 @@ public class SearchAfterBuilderTests extends ESTestCase { private static SearchAfterBuilder serializedCopy(SearchAfterBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new SearchAfterBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 217f97ace0a..a0ce8f02ea3 100644 --- a/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -99,7 +99,7 @@ public class SliceBuilderTests extends ESTestCase { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); try (StreamInput in = - new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return new SliceBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index b494fa4d1e6..8d46372aa42 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -272,7 +272,7 @@ public abstract class AbstractSortTestCase> extends EST private T copyItem(T original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return (T) namedWriteableRegistry.getReader(SortBuilder.class, original.getWriteableName()).read(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java index 2de48decbd8..208b4ed1b53 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SortOrderTests.java @@ -37,7 +37,7 @@ public class SortOrderTests extends ESTestCase { for (SortOrder unit : SortOrder.values()) { try (BytesStreamOutput out = new BytesStreamOutput()) { unit.writeTo(out); - try (StreamInput in = StreamInput.wrap(out.bytes())) { + try (StreamInput in = out.bytes().streamInput()) { assertThat("Roundtrip serialisation failed.", SortOrder.readFromStream(in), equalTo(unit)); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index cd6c34497f7..b67036e1152 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -215,7 +215,7 @@ public abstract class AbstractSuggestionBuilderTestCase extends ESTestCase { private M copyModel(M original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), provideNamedWritableRegistry())) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), provideNamedWritableRegistry())) { return readFrom(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java index 3fd3850b98a..846d3193f6d 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/DirectCandidateGeneratorTests.java @@ -206,7 +206,7 @@ public class DirectCandidateGeneratorTests extends ESTestCase{ private static DirectCandidateGeneratorBuilder serializedCopy(DirectCandidateGeneratorBuilder original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = StreamInput.wrap(output.bytes())) { + try (StreamInput in = output.bytes().streamInput()) { return new DirectCandidateGeneratorBuilder(in); } } diff --git a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java index f167eefa43d..c7b883b583b 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/phrase/SmoothingModelTestCase.java @@ -180,7 +180,7 @@ public abstract class SmoothingModelTestCase extends ESTestCase { static SmoothingModel copyModel(SmoothingModel original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { return namedWriteableRegistry.getReader(SmoothingModel.class, original.getWriteableName()).read(in); } } diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 19c508e2bb1..15623825887 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -646,8 +646,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest @Override public void sendResponse(RestResponse response) { try { - assertThat(response.content().toUtf8(), containsString("notsecretusername")); - assertThat(response.content().toUtf8(), not(containsString("verysecretpassword"))); + assertThat(response.content().utf8ToString(), containsString("notsecretusername")); + assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword"))); } catch (AssertionError ex) { getRepoError.set(ex); } @@ -667,8 +667,8 @@ public class DedicatedClusterSnapshotRestoreIT extends AbstractSnapshotIntegTest @Override public void sendResponse(RestResponse response) { try { - assertThat(response.content().toUtf8(), containsString("notsecretusername")); - assertThat(response.content().toUtf8(), not(containsString("verysecretpassword"))); + assertThat(response.content().utf8ToString(), containsString("notsecretusername")); + assertThat(response.content().utf8ToString(), not(containsString("verysecretpassword"))); } catch (AssertionError ex) { clusterStateError.set(ex); } diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java index 38d858c49aa..c178b2a6f83 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotRequestsTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.snapshots; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.test.ESTestCase; @@ -75,7 +76,7 @@ public class SnapshotRequestsTests extends ESTestCase { builder.endArray(); } - byte[] bytes = builder.endObject().bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(builder.endObject().bytes()); request.source(bytes); @@ -134,7 +135,7 @@ public class SnapshotRequestsTests extends ESTestCase { builder.endArray(); } - byte[] bytes = builder.endObject().bytes().toBytes(); + byte[] bytes = BytesReference.toBytes(builder.endObject().bytes()); request.source(bytes); diff --git a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java index cb297785e4b..41cfa3d4141 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java +++ b/core/src/test/java/org/elasticsearch/snapshots/SnapshotTests.java @@ -20,12 +20,10 @@ package org.elasticsearch.snapshots; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import java.nio.ByteBuffer; import static org.hamcrest.CoreMatchers.equalTo; @@ -50,8 +48,7 @@ public class SnapshotTests extends ESTestCase { final Snapshot original = new Snapshot(randomAsciiOfLength(randomIntBetween(2, 8)), snapshotId); final BytesStreamOutput out = new BytesStreamOutput(); original.writeTo(out); - final ByteBufferStreamInput in = new ByteBufferStreamInput(ByteBuffer.wrap(out.bytes().toBytes())); - assertThat(new Snapshot(in), equalTo(original)); + assertThat(new Snapshot(out.bytes().streamInput()), equalTo(original)); } } diff --git a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java index bfbb2dff4c7..5b507436129 100644 --- a/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java +++ b/core/src/test/java/org/elasticsearch/tasks/PersistedTaskInfoTests.java @@ -37,7 +37,7 @@ import java.util.Map; import java.util.TreeMap; /** - * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. + * Round trip tests for {@link PersistedTaskInfo} and those classes that it includes like {@link TaskInfo} and {@link RawTaskStatus}. */ public class PersistedTaskInfoTests extends ESTestCase { public void testBinaryRoundTrip() throws IOException { @@ -47,7 +47,7 @@ public class PersistedTaskInfoTests extends ESTestCase { PersistedTaskInfo read; try (BytesStreamOutput out = new BytesStreamOutput()) { result.writeTo(out); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(out.bytes()), registry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry)) { read = new PersistedTaskInfo(in); } } catch (IOException e) { diff --git a/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java b/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java index b13de26b976..f7990cfacb7 100644 --- a/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java +++ b/core/src/test/java/org/elasticsearch/tasks/TaskIdTests.java @@ -57,7 +57,7 @@ public class TaskIdTests extends ESTestCase { taskId.writeTo(out); BytesReference bytes = out.bytes(); assertEquals(expectedSize, bytes.length()); - try (StreamInput in = StreamInput.wrap(bytes)) { + try (StreamInput in = bytes.streamInput()) { return TaskId.readFromStream(in); } } diff --git a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java index 9e4a881b25b..8f10ccd6537 100644 --- a/core/src/test/java/org/elasticsearch/test/MockLogAppender.java +++ b/core/src/test/java/org/elasticsearch/test/MockLogAppender.java @@ -34,7 +34,7 @@ import static org.hamcrest.MatcherAssert.assertThat; */ public class MockLogAppender extends AppenderSkeleton { - private final static String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch."); + private static final String COMMON_PREFIX = System.getProperty("es.logger.prefix", "org.elasticsearch."); private List expectations; @@ -75,7 +75,7 @@ public class MockLogAppender extends AppenderSkeleton { void assertMatched(); } - public static abstract class AbstractEventExpectation implements LoggingExpectation { + public abstract static class AbstractEventExpectation implements LoggingExpectation { protected final String name; protected final String logger; protected final Level level; diff --git a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java index 486b0635c64..14cf10b8f31 100644 --- a/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/threadpool/ThreadPoolSerializationTests.java @@ -58,7 +58,7 @@ public class ThreadPoolSerializationTests extends ESTestCase { output.setVersion(Version.CURRENT); info.writeTo(output); - StreamInput input = StreamInput.wrap(output.bytes()); + StreamInput input = output.bytes().streamInput(); ThreadPool.Info newInfo = new ThreadPool.Info(); newInfo.readFrom(input); @@ -70,7 +70,7 @@ public class ThreadPoolSerializationTests extends ESTestCase { output.setVersion(Version.CURRENT); info.writeTo(output); - StreamInput input = StreamInput.wrap(output.bytes()); + StreamInput input = output.bytes().streamInput(); ThreadPool.Info newInfo = new ThreadPool.Info(); newInfo.readFrom(input); @@ -125,7 +125,7 @@ public class ThreadPoolSerializationTests extends ESTestCase { output.setVersion(Version.CURRENT); info.writeTo(output); - StreamInput input = StreamInput.wrap(output.bytes()); + StreamInput input = output.bytes().streamInput(); ThreadPool.Info newInfo = new ThreadPool.Info(); newInfo.readFrom(input); diff --git a/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java index a284f6ea911..bc10458378f 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/ChannelBufferBytesReferenceTests.java @@ -18,11 +18,10 @@ */ package org.elasticsearch.transport.netty; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; -import org.elasticsearch.transport.netty.NettyUtils; import org.jboss.netty.buffer.ChannelBuffer; import org.jboss.netty.buffer.ChannelBuffers; @@ -38,17 +37,16 @@ public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTest assertEquals(out.size(), length); BytesReference ref = out.bytes(); assertEquals(ref.length(), length); - BytesArray bytesArray = ref.toBytesArray(); - return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(), - bytesArray.length())); + BytesRef bytesRef = ref.toBytesRef(); + final ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, bytesRef.length); + return NettyUtils.toBytesReference(channelBuffer); } public void testSliceOnAdvancedBuffer() throws IOException { BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); - BytesArray bytesArray = bytesReference.toBytesArray(); - - ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(), - bytesArray.length()); + BytesRef bytesRef = bytesReference.toBytesRef(); + ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); int numBytesToRead = randomIntBetween(1, 5); for (int i = 0; i < numBytesToRead; i++) { channelBuffer.readByte(); @@ -56,7 +54,25 @@ public class ChannelBufferBytesReferenceTests extends AbstractBytesReferenceTest BytesReference other = NettyUtils.toBytesReference(channelBuffer); BytesReference slice = bytesReference.slice(numBytesToRead, bytesReference.length() - numBytesToRead); assertEquals(other, slice); - assertEquals(other.slice(3, 1), slice.slice(3, 1)); } + + public void testImmutable() throws IOException { + BytesReference bytesReference = newBytesReference(randomIntBetween(10, 3 * PAGE_SIZE)); + BytesRef bytesRef = BytesRef.deepCopyOf(bytesReference.toBytesRef()); + ChannelBuffer channelBuffer = ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length); + ChannelBufferBytesReference channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, bytesRef.length); + assertEquals(channelBufferBytesReference, bytesReference); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(channelBufferBytesReference, bytesReference); + assertEquals(bytesRef, channelBufferBytesReference.toBytesRef()); + + BytesRef unicodeBytes = new BytesRef(randomUnicodeOfCodepointLength(100)); + channelBuffer = ChannelBuffers.wrappedBuffer(unicodeBytes.bytes, unicodeBytes.offset, unicodeBytes.length); + channelBufferBytesReference = new ChannelBufferBytesReference(channelBuffer, unicodeBytes.length); + String utf8ToString = channelBufferBytesReference.utf8ToString(); + channelBuffer.readInt(); // this advances the index of the channel buffer + assertEquals(utf8ToString, channelBufferBytesReference.utf8ToString()); + } } diff --git a/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java index fa8f30249bb..aa1db4a44c7 100644 --- a/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/transport/netty/NettyUtilsTests.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.transport.netty; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; @@ -47,7 +49,7 @@ public class NettyUtilsTests extends ESTestCase { BytesReference slice = ref.slice(sliceOffset, sliceLength); ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(slice); BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer); - assertArrayEquals(slice.toBytes(), bytesReference.toBytes()); + assertArrayEquals(BytesReference.toBytes(slice), BytesReference.toBytes(bytesReference)); } public void testToChannelBufferWithSliceAfter() throws IOException { @@ -56,7 +58,8 @@ public class NettyUtilsTests extends ESTestCase { int sliceLength = randomIntBetween(ref.length() - sliceOffset, ref.length() - sliceOffset); ChannelBuffer channelBuffer = NettyUtils.toChannelBuffer(ref); BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer); - assertArrayEquals(ref.slice(sliceOffset, sliceLength).toBytes(), bytesReference.slice(sliceOffset, sliceLength).toBytes()); + assertArrayEquals(BytesReference.toBytes(ref.slice(sliceOffset, sliceLength)), + BytesReference.toBytes(bytesReference.slice(sliceOffset, sliceLength))); } public void testToChannelBuffer() throws IOException { @@ -65,10 +68,10 @@ public class NettyUtilsTests extends ESTestCase { BytesReference bytesReference = NettyUtils.toBytesReference(channelBuffer); if (ref instanceof ChannelBufferBytesReference) { assertEquals(channelBuffer, ((ChannelBufferBytesReference) ref).toChannelBuffer()); - } else if (ref.hasArray() == false) { // we gather the buffers into a channel buffer + } else if (AbstractBytesReferenceTestCase.getNumPages(ref) > 1) { // we gather the buffers into a channel buffer assertTrue(channelBuffer instanceof CompositeChannelBuffer); } - assertArrayEquals(ref.toBytes(), bytesReference.toBytes()); + assertArrayEquals(BytesReference.toBytes(ref), BytesReference.toBytes(bytesReference)); } private BytesReference getRandomizedBytesReference(int length) throws IOException { @@ -81,13 +84,14 @@ public class NettyUtilsTests extends ESTestCase { BytesReference ref = out.bytes(); assertEquals(ref.length(), length); if (randomBoolean()) { - return ref.toBytesArray(); + return new BytesArray(ref.toBytesRef()); } else if (randomBoolean()) { - BytesArray bytesArray = ref.toBytesArray(); - return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesArray.array(), bytesArray.arrayOffset(), - bytesArray.length())); + BytesRef bytesRef = ref.toBytesRef(); + return NettyUtils.toBytesReference(ChannelBuffers.wrappedBuffer(bytesRef.bytes, bytesRef.offset, + bytesRef.length)); } else { return ref; } } + } diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index 9e08ecde6fa..583d8a0288d 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -59,7 +59,7 @@ import static org.hamcrest.Matchers.nullValue; @ClusterScope(scope= Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1) public class SimpleTTLIT extends ESIntegTestCase { - static private final long PURGE_INTERVAL = 200; + private static final long PURGE_INTERVAL = 200; @Override protected int numberOfShards() { diff --git a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java index b4b5eefc832..f14d91465f6 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateNoopIT.java @@ -240,12 +240,12 @@ public class UpdateNoopIT extends ESIntegTestCase { private void updateAndCheckSource(long expectedVersion, Boolean detectNoop, XContentBuilder xContentBuilder) { UpdateResponse updateResponse = update(detectNoop, expectedVersion, xContentBuilder); - assertEquals(updateResponse.getGetResult().sourceRef().toUtf8(), xContentBuilder.bytes().toUtf8()); + assertEquals(updateResponse.getGetResult().sourceRef().utf8ToString(), xContentBuilder.bytes().utf8ToString()); } private UpdateResponse update(Boolean detectNoop, long expectedVersion, XContentBuilder xContentBuilder) { UpdateRequestBuilder updateRequest = client().prepareUpdate("test", "type1", "1") - .setDoc(xContentBuilder.bytes().toUtf8()) + .setDoc(xContentBuilder.bytes().utf8ToString()) .setDocAsUpsert(true) .setFields("_source"); if (detectNoop != null) { diff --git a/docs/plugins/ingest-useragent.asciidoc b/docs/plugins/ingest-useragent.asciidoc new file mode 100644 index 00000000000..8868a4cf711 --- /dev/null +++ b/docs/plugins/ingest-useragent.asciidoc @@ -0,0 +1,74 @@ +[[ingest-useragent]] +=== Ingest Useragent Processor Plugin + +The Useragent processor extracts details from the user agent string a browser sends with its web requests. +This processor adds this information by default under the `useragent` field. + +The ingest-useragent plugin ships by default with the regexes.yaml made available by uap-java with an Apache 2.0 license. For more details see https://github.com/ua-parser/uap-core. + +[[ingest-useragent-install]] +[float] +==== Installation + +This plugin can be installed using the plugin manager: + +[source,sh] +---------------------------------------------------------------- +sudo bin/elasticsearch-plugin install ingest-useragent +---------------------------------------------------------------- + +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +[[ingest-useragent-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/elasticsearch-plugin remove ingest-useragent +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[using-ingest-useragent]] +==== Using the Useragent Processor in a Pipeline + +[[ingest-useragent-options]] +.Useragent options +[options="header"] +|====== +| Name | Required | Default | Description +| `field` | yes | - | The field containing the user agent string. +| `target_field` | no | useragent | The field that will be filled with the user agent details. +| `regex_file` | no | - | The name of the file in the `config/ingest-useragent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-useragent will use the regexes.yaml from uap-core it ships with (see below). +| `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`. +|====== + +Here is an example that adds the user agent details to the `useragent` field based on the `agent` field: + +[source,js] +-------------------------------------------------- +{ + "description" : "...", + "processors" : [ + { + "useragent" : { + "field" : "agent" + } + } + ] +} +-------------------------------------------------- + +===== Using a custom regex file +To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-useragent` directory and +has to have a `.yaml` filename extension. The file has to be present at node startup, any changes to it or any new files added +while the node is running will not have any effect. + +In practice, it will make most sense for any custom regex file to be a variant of the default file, either a more recent version +or a customised version. + +The default file included in `ingest-useragent` is the `regexes.yaml` from uap-core: https://github.com/ua-parser/uap-core/blob/master/regexes.yaml diff --git a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc index 3c1f4ae860a..959b93611d8 100644 --- a/docs/reference/aggregations/bucket/terms-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/terms-aggregation.asciidoc @@ -9,8 +9,8 @@ Example: -------------------------------------------------- { "aggs" : { - "genders" : { - "terms" : { "field" : "gender" } + "genres" : { + "terms" : { "field" : "genre" } } } } @@ -24,16 +24,20 @@ Response: ... "aggregations" : { - "genders" : { + "genres" : { "doc_count_error_upper_bound": 0, <1> "sum_other_doc_count": 0, <2> "buckets" : [ <3> { - "key" : "male", + "key" : "jazz", "doc_count" : 10 }, { - "key" : "female", + "key" : "rock", + "doc_count" : 10 + }, + { + "key" : "electronic", "doc_count" : 10 }, ] @@ -247,9 +251,9 @@ Ordering the buckets by their `doc_count` in an ascending manner: -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", + "field" : "genre", "order" : { "_count" : "asc" } } } @@ -263,9 +267,9 @@ Ordering the buckets alphabetically by their terms in an ascending manner: -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", + "field" : "genre", "order" : { "_term" : "asc" } } } @@ -280,13 +284,13 @@ Ordering the buckets by single value metrics sub-aggregation (identified by the -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", - "order" : { "avg_height" : "desc" } + "field" : "genre", + "order" : { "avg_play_count" : "desc" } }, "aggs" : { - "avg_height" : { "avg" : { "field" : "height" } } + "avg_play_count" : { "avg" : { "field" : "play_count" } } } } } @@ -299,13 +303,13 @@ Ordering the buckets by multi value metrics sub-aggregation (identified by the a -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", - "order" : { "height_stats.avg" : "desc" } + "field" : "genre", + "order" : { "playback_stats.avg" : "desc" } }, "aggs" : { - "height_stats" : { "stats" : { "field" : "height" } } + "playback_stats" : { "stats" : { "field" : "play_count" } } } } } @@ -343,14 +347,14 @@ PATH := []*[height_stats.avg" : "desc" } + "field" : "artist.country", + "order" : { "rock>playback_stats.avg" : "desc" } }, "aggs" : { - "females" : { - "filter" : { "term" : { "gender" : "female" }}, + "rock" : { + "filter" : { "term" : { "genre" : "rock" }}, "aggs" : { - "height_stats" : { "stats" : { "field" : "height" }} + "playback_stats" : { "stats" : { "field" : "play_count" }} } } } @@ -359,7 +363,7 @@ PATH := []*[height_stats.avg" : "desc" }, { "_count" : "desc" } ] + "field" : "artist.country", + "order" : [ { "rock>playback_stats.avg" : "desc" }, { "_count" : "desc" } ] }, "aggs" : { - "females" : { - "filter" : { "term" : { "gender" : { "female" }}}, + "rock" : { + "filter" : { "term" : { "genre" : { "rock" }}}, "aggs" : { - "height_stats" : { "stats" : { "field" : "height" }} + "playback_stats" : { "stats" : { "field" : "play_count" }} } } } @@ -385,7 +389,7 @@ Multiple criteria can be used to order the buckets by providing an array of orde } -------------------------------------------------- -The above will sort the countries buckets based on the average height among the female population and then by +The above will sort the artist's countries buckets based on the average play count among the rock songs and then by their `doc_count` in descending order. NOTE: In the event that two buckets share the same values for all order criteria the bucket's term value is used as a @@ -439,10 +443,10 @@ Generating the terms using a script: -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { "script" : { - "inline": "doc['gender'].value" + "inline": "doc['genre'].value" "lang": "painless" } } @@ -457,12 +461,12 @@ This will interpret the `script` parameter as an `inline` script with the defaul -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { "script" : { "file": "my_script", "params": { - "field": "gender" + "field": "genre" } } } @@ -480,11 +484,11 @@ TIP: for indexed scripts replace the `file` parameter with an `id` parameter. -------------------------------------------------- { "aggs" : { - "genders" : { + "genres" : { "terms" : { - "field" : "gender", + "field" : "gendre", "script" : { - "inline" : "'Gender: ' +_value" + "inline" : "'Genre: ' +_value" "lang" : "painless" } } diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index b03ed641de7..ec4f9c30e66 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -46,7 +46,6 @@ PUT _ingest/pipeline/my-pipeline-id "value": "bar" } } - // other processors ] } -------------------------------------------------- @@ -83,7 +82,6 @@ Example response: "value": "bar" } } - // other processors ] } } ] diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java index edef75389c8..20be7e72888 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/InternalMatrixStats.java @@ -36,8 +36,8 @@ import java.util.Map; */ public class InternalMatrixStats extends InternalMetricsAggregation implements MatrixStats { - public final static Type TYPE = new Type("matrix_stats"); - public final static AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { + public static final Type TYPE = new Type("matrix_stats"); + public static final AggregationStreams.Stream STREAM = new AggregationStreams.Stream() { @Override public InternalMatrixStats readResult(StreamInput in) throws IOException { InternalMatrixStats result = new InternalMatrixStats(); diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java index 96b7b74ab4e..1ae29e65761 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/matrix/stats/MatrixStatsResults.java @@ -34,9 +34,9 @@ import java.util.Map; */ class MatrixStatsResults implements Writeable { /** object holding results - computes results in place */ - final protected RunningStats results; + protected final RunningStats results; /** pearson product correlation coefficients */ - final protected Map> correlation; + protected final Map> correlation; /** Base ctor */ public MatrixStatsResults() { diff --git a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java index 51e5ce1cf27..e3aa171fe3d 100644 --- a/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java +++ b/modules/aggs-matrix-stats/src/main/java/org/elasticsearch/search/aggregations/support/MultiValuesSourceAggregationBuilder.java @@ -52,7 +52,7 @@ public abstract class MultiValuesSourceAggregationBuilder> + public abstract static class LeafOnly> extends MultiValuesSourceAggregationBuilder { protected LeafOnly(String name, Type type, ValuesSourceType valuesSourceType, ValueType targetValueType) { diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java index b1296bb1146..2e4fa4313bd 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/stats/BaseMatrixStatsTestCase.java @@ -34,8 +34,8 @@ public abstract class BaseMatrixStatsTestCase extends ESTestCase { protected final ArrayList fieldA = new ArrayList<>(numObs); protected final ArrayList fieldB = new ArrayList<>(numObs); protected final MultiPassStats actualStats = new MultiPassStats(); - protected final static String fieldAKey = "fieldA"; - protected final static String fieldBKey = "fieldB"; + protected static final String fieldAKey = "fieldA"; + protected static final String fieldBKey = "fieldB"; @Before public void setup() { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index d99bcb550a9..079ff73846a 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -53,7 +53,7 @@ abstract class AbstractStringProcessor extends AbstractProcessor { protected abstract String process(String value); - static abstract class Factory implements Processor.Factory { + abstract static class Factory implements Processor.Factory { protected final String processorType; protected Factory(String processorType) { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index fe607e12b6e..44528bdac82 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -114,7 +114,7 @@ public final class GrokProcessor extends AbstractProcessor { return combinedPattern; } - public final static class Factory implements Processor.Factory { + public static final class Factory implements Processor.Factory { private final Map builtinPatterns; diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java index 7a8916cd439..6b4327f726d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java @@ -70,7 +70,7 @@ public final class JoinProcessor extends AbstractProcessor { return TYPE; } - public final static class Factory implements Processor.Factory { + public static final class Factory implements Processor.Factory { @Override public JoinProcessor create(Map registry, String processorTag, Map config) throws Exception { diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java index e7a8f3f3e6a..a0ae8e13158 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/LowercaseProcessor.java @@ -44,7 +44,7 @@ public final class LowercaseProcessor extends AbstractStringProcessor { return TYPE; } - public final static class Factory extends AbstractStringProcessor.Factory { + public static final class Factory extends AbstractStringProcessor.Factory { public Factory() { super(TYPE); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java index 0eb3843f666..411b22adef0 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java @@ -111,7 +111,7 @@ public final class SortProcessor extends AbstractProcessor { return TYPE; } - public final static class Factory implements Processor.Factory { + public static final class Factory implements Processor.Factory { @Override public SortProcessor create(Map registry, String processorTag, diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java index 28894c5c1cc..d3871e90510 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java @@ -465,7 +465,7 @@ public class SearchFieldsTests extends ESIntegTestCase { String dateTime = Joda.forPattern("dateOptionalTime").printer().print(new DateTime(2012, 3, 22, 0, 0, DateTimeZone.UTC)); assertThat(searchResponse.getHits().getAt(0).fields().get("date_field").value(), equalTo((Object) dateTime)); assertThat(searchResponse.getHits().getAt(0).fields().get("boolean_field").value(), equalTo((Object) Boolean.TRUE)); - assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()).toBytesArray(), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8")))); + assertThat(((BytesReference) searchResponse.getHits().getAt(0).fields().get("binary_field").value()), equalTo((BytesReference) new BytesArray("testing text".getBytes("UTF8")))); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java index 131443da887..e12137ee3c5 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/rest/action/search/template/RestSearchTemplateAction.java @@ -73,7 +73,7 @@ public class RestSearchTemplateAction extends BaseRestHandler { request.setScriptType(ScriptService.ScriptType.INLINE); if (parser.currentToken() == XContentParser.Token.START_OBJECT) { try (XContentBuilder builder = XContentFactory.contentBuilder(parser.contentType())) { - request.setScript(builder.copyCurrentStructure(parser).bytes().toUtf8()); + request.setScript(builder.copyCurrentStructure(parser).bytes().utf8ToString()); } catch (IOException e) { throw new ParsingException(parser.getTokenLocation(), "Could not parse inline template", e); } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java index ceb187bcc63..8419730dc1c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java @@ -92,7 +92,7 @@ public class CustomMustacheFactory extends DefaultMustacheFactory { /** * Base class for custom Mustache functions */ - static abstract class CustomCode extends IterableCode { + abstract static class CustomCode extends IterableCode { private final String code; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java index 45d3d8c182d..dd3055ba8e8 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java @@ -49,7 +49,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler { } } - final static class ArrayMap extends AbstractMap implements Iterable { + static final class ArrayMap extends AbstractMap implements Iterable { private final Object array; private final int length; @@ -109,7 +109,7 @@ final class CustomReflectionObjectHandler extends ReflectionObjectHandler { } - final static class CollectionMap extends AbstractMap implements Iterable { + static final class CollectionMap extends AbstractMap implements Iterable { private final Collection col; diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java index 9c28225c081..03d04e518dc 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/action/search/template/MultiSearchTemplateIT.java @@ -146,14 +146,14 @@ public class MultiSearchTemplateIT extends ESIntegTestCase { SearchTemplateResponse searchTemplateResponse1 = response1.getResponse(); assertThat(searchTemplateResponse1.hasResponse(), is(true)); assertHitCount(searchTemplateResponse1.getResponse(), (numDocs / 2) + (numDocs % 2)); - assertThat(searchTemplateResponse1.getSource().toUtf8(), + assertThat(searchTemplateResponse1.getSource().utf8ToString(), equalTo("{\"query\":{\"match\":{\"odd\":\"true\"}}}")); MultiSearchTemplateResponse.Item response2 = response.getResponses()[1]; assertThat(response2.isFailure(), is(false)); SearchTemplateResponse searchTemplateResponse2 = response2.getResponse(); assertThat(searchTemplateResponse2.hasResponse(), is(false)); - assertThat(searchTemplateResponse2.getSource().toUtf8(), + assertThat(searchTemplateResponse2.getSource().utf8ToString(), equalTo("{\"query\":{\"match_phrase_prefix\":{\"message\":\"quick brown f\"}}}")); MultiSearchTemplateResponse.Item response3 = response.getResponses()[2]; @@ -161,7 +161,7 @@ public class MultiSearchTemplateIT extends ESIntegTestCase { SearchTemplateResponse searchTemplateResponse3 = response3.getResponse(); assertThat(searchTemplateResponse3.hasResponse(), is(true)); assertHitCount(searchTemplateResponse3.getResponse(), (numDocs / 2)); - assertThat(searchTemplateResponse3.getSource().toUtf8(), + assertThat(searchTemplateResponse3.getSource().utf8ToString(), equalTo("{\"query\":{\"term\":{\"odd\":\"false\"}}}")); MultiSearchTemplateResponse.Item response4 = response.getResponses()[3]; @@ -173,7 +173,7 @@ public class MultiSearchTemplateIT extends ESIntegTestCase { assertThat(response5.isFailure(), is(false)); SearchTemplateResponse searchTemplateResponse5 = response5.getResponse(); assertThat(searchTemplateResponse5.hasResponse(), is(false)); - assertThat(searchTemplateResponse5.getSource().toUtf8(), + assertThat(searchTemplateResponse5.getSource().utf8ToString(), equalTo("{\"query\":{\"terms\":{\"group\":[1,2,3,]}}}")); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java index 054268ef681..bdba0346a85 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheScriptEngineTests.java @@ -28,7 +28,6 @@ import org.junit.Before; import java.io.IOException; import java.io.StringWriter; -import java.nio.charset.Charset; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -58,7 +57,7 @@ public class MustacheScriptEngineTests extends ESTestCase { BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.3 } }}", - new String(o.toBytes(), Charset.forName("UTF-8"))); + o.utf8ToString()); } { String template = "GET _search {\"query\": " + "{\"boosting\": {" + "\"positive\": {\"match\": {\"body\": \"gift\"}}," @@ -69,7 +68,7 @@ public class MustacheScriptEngineTests extends ESTestCase { BytesReference o = (BytesReference) qe.executable(new CompiledScript(ScriptService.ScriptType.INLINE, "", "mustache", qe.compile(null, template, compileParams)), vars).run(); assertEquals("GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"\\\"quick brown\\\"\"}}}, \"negative_boost\": 0.3 } }}", - new String(o.toBytes(), Charset.forName("UTF-8"))); + o.utf8ToString()); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java index 8b6d0e69573..91098eb1c88 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java @@ -72,7 +72,7 @@ public class MustacheTests extends ESTestCase { "Mustache templating broken", "GET _search {\"query\": {\"boosting\": {\"positive\": {\"match\": {\"body\": \"gift\"}}," + "\"negative\": {\"term\": {\"body\": {\"value\": \"solr\"}}}, \"negative_boost\": 0.2 } }}", - ((BytesReference) result.run()).toUtf8() + ((BytesReference) result.run()).utf8ToString() ); } @@ -88,7 +88,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); BytesReference bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), equalTo("foo bar")); + assertThat(bytes.utf8ToString(), equalTo("foo bar")); // Sets can come out in any order Set setData = new HashSet<>(); @@ -99,7 +99,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), both(containsString("foo")).and(containsString("bar"))); + assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar"))); } public void testArrayInArrayAccess() throws Exception { @@ -116,7 +116,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); BytesReference bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), equalTo("foo bar")); + assertThat(bytes.utf8ToString(), equalTo("foo bar")); } public void testMapInArrayAccess() throws Exception { @@ -131,7 +131,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); BytesReference bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), equalTo("foo bar")); + assertThat(bytes.utf8ToString(), equalTo("foo bar")); // HashSet iteration order isn't fixed Set setData = new HashSet<>(); @@ -142,7 +142,7 @@ public class MustacheTests extends ESTestCase { assertThat(output, notNullValue()); assertThat(output, instanceOf(BytesReference.class)); bytes = (BytesReference) output; - assertThat(bytes.toUtf8(), both(containsString("foo")).and(containsString("bar"))); + assertThat(bytes.utf8ToString(), both(containsString("foo")).and(containsString("bar"))); } public void testEscaping() { @@ -152,7 +152,7 @@ public class MustacheTests extends ESTestCase { CompiledScript compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache); ExecutableScript executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\"")); BytesReference rawResult = (BytesReference) executableScript.run(); - String result = rawResult.toUtf8(); + String result = rawResult.utf8ToString(); assertThat(result, equalTo("{ \"field1\": \"a \\\"value\\\"\"}")); // json string escaping disabled: @@ -160,7 +160,7 @@ public class MustacheTests extends ESTestCase { compiledScript = new CompiledScript(INLINE, "name", "mustache", mustache); executableScript = engine.executable(compiledScript, Collections.singletonMap("value", "a \"value\"")); rawResult = (BytesReference) executableScript.run(); - result = rawResult.toUtf8(); + result = rawResult.utf8ToString(); assertThat(result, equalTo("{ \"field1\": \"a \"value\"\"}")); } @@ -182,7 +182,7 @@ public class MustacheTests extends ESTestCase { BytesReference bytes = (BytesReference) output; String expectedString = String.format(Locale.ROOT, "%s %s", randomArrayValues.length, randomList.size()); - assertThat(bytes.toUtf8(), equalTo(expectedString)); + assertThat(bytes.utf8ToString(), equalTo(expectedString)); } public void testPrimitiveToJSON() throws Exception { @@ -378,7 +378,7 @@ public class MustacheTests extends ESTestCase { Object result = engine.executable(new CompiledScript(INLINE, "inline", "mustache", compile(script)), vars).run(); assertThat(result, notNullValue()); assertThat(result, instanceOf(BytesReference.class)); - assertThat(((BytesReference) result).toUtf8(), matcher); + assertThat(((BytesReference) result).utf8ToString(), matcher); } private Object compile(String script) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index ca549522018..c546207b1ee 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -43,37 +43,37 @@ import java.util.regex.Pattern; */ public final class WriterConstants { - public final static int CLASS_VERSION = Opcodes.V1_8; - public final static int ASM_VERSION = Opcodes.ASM5; - public final static String BASE_CLASS_NAME = Executable.class.getName(); - public final static Type BASE_CLASS_TYPE = Type.getType(Executable.class); + public static final int CLASS_VERSION = Opcodes.V1_8; + public static final int ASM_VERSION = Opcodes.ASM5; + public static final String BASE_CLASS_NAME = Executable.class.getName(); + public static final Type BASE_CLASS_TYPE = Type.getType(Executable.class); - public final static String CLASS_NAME = BASE_CLASS_NAME + "$Script"; - public final static Type CLASS_TYPE = Type.getObjectType(CLASS_NAME.replace('.', '/')); + public static final String CLASS_NAME = BASE_CLASS_NAME + "$Script"; + public static final Type CLASS_TYPE = Type.getObjectType(CLASS_NAME.replace('.', '/')); - public final static Method CONSTRUCTOR = getAsmMethod(void.class, "", String.class, String.class, BitSet.class); - public final static Method CLINIT = getAsmMethod(void.class, ""); - public final static Method EXECUTE = + public static final Method CONSTRUCTOR = getAsmMethod(void.class, "", String.class, String.class, BitSet.class); + public static final Method CLINIT = getAsmMethod(void.class, ""); + public static final Method EXECUTE = getAsmMethod(Object.class, "execute", Map.class, Scorer.class, LeafDocLookup.class, Object.class); - public final static Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class); + public static final Type PAINLESS_ERROR_TYPE = Type.getType(PainlessError.class); - public final static Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class); - public final static Type SCORER_TYPE = Type.getType(Scorer.class); - public final static Method SCORER_SCORE = getAsmMethod(float.class, "score"); + public static final Type NEEDS_SCORE_TYPE = Type.getType(NeedsScore.class); + public static final Type SCORER_TYPE = Type.getType(Scorer.class); + public static final Method SCORER_SCORE = getAsmMethod(float.class, "score"); - public final static Type MAP_TYPE = Type.getType(Map.class); - public final static Method MAP_GET = getAsmMethod(Object.class, "get", Object.class); + public static final Type MAP_TYPE = Type.getType(Map.class); + public static final Method MAP_GET = getAsmMethod(Object.class, "get", Object.class); - public final static Type ITERATOR_TYPE = Type.getType(Iterator.class); - public final static Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext"); - public final static Method ITERATOR_NEXT = getAsmMethod(Object.class, "next"); + public static final Type ITERATOR_TYPE = Type.getType(Iterator.class); + public static final Method ITERATOR_HASNEXT = getAsmMethod(boolean.class, "hasNext"); + public static final Method ITERATOR_NEXT = getAsmMethod(Object.class, "next"); - public final static Type UTILITY_TYPE = Type.getType(Utility.class); - public final static Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class); - public final static Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class); + public static final Type UTILITY_TYPE = Type.getType(Utility.class); + public static final Method STRING_TO_CHAR = getAsmMethod(char.class, "StringTochar", String.class); + public static final Method CHAR_TO_STRING = getAsmMethod(String.class, "charToString", char.class); - public final static Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class); + public static final Type METHOD_HANDLE_TYPE = Type.getType(MethodHandle.class); public static final Type AUGMENTATION_TYPE = Type.getType(Augmentation.class); @@ -83,46 +83,46 @@ public final class WriterConstants { * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex * per time it is run. */ - public final static Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class); - public final static Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class); - public final static Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches"); - public final static Method MATCHER_FIND = getAsmMethod(boolean.class, "find"); + public static final Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class); + public static final Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class); + public static final Method MATCHER_MATCHES = getAsmMethod(boolean.class, "matches"); + public static final Method MATCHER_FIND = getAsmMethod(boolean.class, "find"); /** dynamic callsite bootstrap signature */ - final static MethodType DEF_BOOTSTRAP_TYPE = + static final MethodType DEF_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class); - final static Handle DEF_BOOTSTRAP_HANDLE = + static final Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(DefBootstrap.class), "bootstrap", DEF_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); - public final static Type DEF_UTIL_TYPE = Type.getType(Def.class); - public final static Method DEF_TO_BOOLEAN = getAsmMethod(boolean.class, "DefToboolean" , Object.class); - public final static Method DEF_TO_BYTE_IMPLICIT = getAsmMethod(byte.class , "DefTobyteImplicit" , Object.class); - public final static Method DEF_TO_SHORT_IMPLICIT = getAsmMethod(short.class , "DefToshortImplicit" , Object.class); - public final static Method DEF_TO_CHAR_IMPLICIT = getAsmMethod(char.class , "DefTocharImplicit" , Object.class); - public final static Method DEF_TO_INT_IMPLICIT = getAsmMethod(int.class , "DefTointImplicit" , Object.class); - public final static Method DEF_TO_LONG_IMPLICIT = getAsmMethod(long.class , "DefTolongImplicit" , Object.class); - public final static Method DEF_TO_FLOAT_IMPLICIT = getAsmMethod(float.class , "DefTofloatImplicit" , Object.class); - public final static Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class); - public final static Method DEF_TO_BYTE_EXPLICIT = getAsmMethod(byte.class , "DefTobyteExplicit" , Object.class); - public final static Method DEF_TO_SHORT_EXPLICIT = getAsmMethod(short.class , "DefToshortExplicit" , Object.class); - public final static Method DEF_TO_CHAR_EXPLICIT = getAsmMethod(char.class , "DefTocharExplicit" , Object.class); - public final static Method DEF_TO_INT_EXPLICIT = getAsmMethod(int.class , "DefTointExplicit" , Object.class); - public final static Method DEF_TO_LONG_EXPLICIT = getAsmMethod(long.class , "DefTolongExplicit" , Object.class); - public final static Method DEF_TO_FLOAT_EXPLICIT = getAsmMethod(float.class , "DefTofloatExplicit" , Object.class); - public final static Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class); + public static final Type DEF_UTIL_TYPE = Type.getType(Def.class); + public static final Method DEF_TO_BOOLEAN = getAsmMethod(boolean.class, "DefToboolean" , Object.class); + public static final Method DEF_TO_BYTE_IMPLICIT = getAsmMethod(byte.class , "DefTobyteImplicit" , Object.class); + public static final Method DEF_TO_SHORT_IMPLICIT = getAsmMethod(short.class , "DefToshortImplicit" , Object.class); + public static final Method DEF_TO_CHAR_IMPLICIT = getAsmMethod(char.class , "DefTocharImplicit" , Object.class); + public static final Method DEF_TO_INT_IMPLICIT = getAsmMethod(int.class , "DefTointImplicit" , Object.class); + public static final Method DEF_TO_LONG_IMPLICIT = getAsmMethod(long.class , "DefTolongImplicit" , Object.class); + public static final Method DEF_TO_FLOAT_IMPLICIT = getAsmMethod(float.class , "DefTofloatImplicit" , Object.class); + public static final Method DEF_TO_DOUBLE_IMPLICIT = getAsmMethod(double.class , "DefTodoubleImplicit", Object.class); + public static final Method DEF_TO_BYTE_EXPLICIT = getAsmMethod(byte.class , "DefTobyteExplicit" , Object.class); + public static final Method DEF_TO_SHORT_EXPLICIT = getAsmMethod(short.class , "DefToshortExplicit" , Object.class); + public static final Method DEF_TO_CHAR_EXPLICIT = getAsmMethod(char.class , "DefTocharExplicit" , Object.class); + public static final Method DEF_TO_INT_EXPLICIT = getAsmMethod(int.class , "DefTointExplicit" , Object.class); + public static final Method DEF_TO_LONG_EXPLICIT = getAsmMethod(long.class , "DefTolongExplicit" , Object.class); + public static final Method DEF_TO_FLOAT_EXPLICIT = getAsmMethod(float.class , "DefTofloatExplicit" , Object.class); + public static final Method DEF_TO_DOUBLE_EXPLICIT = getAsmMethod(double.class , "DefTodoubleExplicit", Object.class); /** invokedynamic bootstrap for lambda expression/method references */ - public final static MethodType LAMBDA_BOOTSTRAP_TYPE = + public static final MethodType LAMBDA_BOOTSTRAP_TYPE = MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class, Object[].class); - public final static Handle LAMBDA_BOOTSTRAP_HANDLE = + public static final Handle LAMBDA_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaMetafactory.class), "altMetafactory", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false); /** dynamic invokedynamic bootstrap for indy string concats (Java 9+) */ - public final static Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; + public static final Handle INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; static { Handle bs; try { @@ -139,24 +139,24 @@ public final class WriterConstants { INDY_STRING_CONCAT_BOOTSTRAP_HANDLE = bs; } - public final static int MAX_INDY_STRING_CONCAT_ARGS = 200; + public static final int MAX_INDY_STRING_CONCAT_ARGS = 200; - public final static Type STRING_TYPE = Type.getType(String.class); - public final static Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class); + public static final Type STRING_TYPE = Type.getType(String.class); + public static final Type STRINGBUILDER_TYPE = Type.getType(StringBuilder.class); - public final static Method STRINGBUILDER_CONSTRUCTOR = getAsmMethod(void.class, ""); - public final static Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class); - public final static Method STRINGBUILDER_APPEND_CHAR = getAsmMethod(StringBuilder.class, "append", char.class); - public final static Method STRINGBUILDER_APPEND_INT = getAsmMethod(StringBuilder.class, "append", int.class); - public final static Method STRINGBUILDER_APPEND_LONG = getAsmMethod(StringBuilder.class, "append", long.class); - public final static Method STRINGBUILDER_APPEND_FLOAT = getAsmMethod(StringBuilder.class, "append", float.class); - public final static Method STRINGBUILDER_APPEND_DOUBLE = getAsmMethod(StringBuilder.class, "append", double.class); - public final static Method STRINGBUILDER_APPEND_STRING = getAsmMethod(StringBuilder.class, "append", String.class); - public final static Method STRINGBUILDER_APPEND_OBJECT = getAsmMethod(StringBuilder.class, "append", Object.class); - public final static Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString"); + public static final Method STRINGBUILDER_CONSTRUCTOR = getAsmMethod(void.class, ""); + public static final Method STRINGBUILDER_APPEND_BOOLEAN = getAsmMethod(StringBuilder.class, "append", boolean.class); + public static final Method STRINGBUILDER_APPEND_CHAR = getAsmMethod(StringBuilder.class, "append", char.class); + public static final Method STRINGBUILDER_APPEND_INT = getAsmMethod(StringBuilder.class, "append", int.class); + public static final Method STRINGBUILDER_APPEND_LONG = getAsmMethod(StringBuilder.class, "append", long.class); + public static final Method STRINGBUILDER_APPEND_FLOAT = getAsmMethod(StringBuilder.class, "append", float.class); + public static final Method STRINGBUILDER_APPEND_DOUBLE = getAsmMethod(StringBuilder.class, "append", double.class); + public static final Method STRINGBUILDER_APPEND_STRING = getAsmMethod(StringBuilder.class, "append", String.class); + public static final Method STRINGBUILDER_APPEND_OBJECT = getAsmMethod(StringBuilder.class, "append", Object.class); + public static final Method STRINGBUILDER_TOSTRING = getAsmMethod(String.class, "toString"); - public final static Type OBJECTS_TYPE = Type.getType(Objects.class); - public final static Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class); + public static final Type OBJECTS_TYPE = Type.getType(Objects.class); + public static final Method EQUALS = getAsmMethod(boolean.class, "equals", Object.class, Object.class); private static Method getAsmMethod(final Class rtype, final String name, final Class... ptypes) { return new Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString()); diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java index 311b612a18d..76bc136656c 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQuery.java @@ -292,19 +292,13 @@ public final class PercolateQuery extends Query implements Accountable { @Override public String toString(String s) { - return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.toUtf8() + + return "PercolateQuery{document_type={" + documentType + "},document_source={" + documentSource.utf8ToString() + "},inner={" + percolatorQueriesQuery.toString(s) + "}}"; } @Override public long ramBytesUsed() { - long sizeInBytes = 0; - if (documentSource.hasArray()) { - sizeInBytes += documentSource.array().length; - } else { - sizeInBytes += documentSource.length(); - } - return sizeInBytes; + return documentSource.ramBytesUsed(); } @FunctionalInterface @@ -321,7 +315,7 @@ public final class PercolateQuery extends Query implements Accountable { } - static abstract class BaseScorer extends Scorer { + abstract static class BaseScorer extends Scorer { final Scorer approximation; final QueryStore.Leaf percolatorQueries; diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 3ce3726aa5e..a5db24a71ef 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -541,7 +541,7 @@ public class PercolateQueryBuilder extends AbstractQueryBuilder INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = + public static final XContentType QUERY_BUILDER_CONTENT_TYPE = XContentType.SMILE; + public static final Setting INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING = Setting.boolSetting("index.percolator.map_unmapped_fields_as_string", false, Setting.Property.IndexScope); public static final String CONTENT_TYPE = "percolator"; private static final PercolatorFieldType FIELD_TYPE = new PercolatorFieldType(); @@ -216,7 +217,7 @@ public class PercolatorFieldMapper extends FieldMapper { try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) { queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap())); builder.flush(); - byte[] queryBuilderAsBytes = builder.bytes().toBytes(); + byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes()); context.doc().add(new Field(queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType())); } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java index b9486a89f49..a050c8eb420 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/ExtractQueryTermsServiceTests.java @@ -71,9 +71,9 @@ import static org.hamcrest.Matchers.sameInstance; public class ExtractQueryTermsServiceTests extends ESTestCase { - public final static String QUERY_TERMS_FIELD = "extracted_terms"; - public final static String EXTRACTION_RESULT_FIELD = "extraction_result"; - public final static FieldType QUERY_TERMS_FIELD_TYPE = new FieldType(); + public static final String QUERY_TERMS_FIELD = "extracted_terms"; + public static final String EXTRACTION_RESULT_FIELD = "extraction_result"; + public static final FieldType QUERY_TERMS_FIELD_TYPE = new FieldType(); static { QUERY_TERMS_FIELD_TYPE.setTokenized(false); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java index 15d531467d4..0a359376f7b 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/MultiPercolatorIT.java @@ -53,8 +53,8 @@ import static org.hamcrest.Matchers.nullValue; public class MultiPercolatorIT extends ESIntegTestCase { - private final static String INDEX_NAME = "queries"; - private final static String TYPE_NAME = "query"; + private static final String INDEX_NAME = "queries"; + private static final String TYPE_NAME = "query"; @Override protected Collection> nodePlugins() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java index 79904f422bb..c2c2a641a71 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryTests.java @@ -82,9 +82,9 @@ import static org.hamcrest.Matchers.is; public class PercolateQueryTests extends ESTestCase { - public final static String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; - public final static String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; - public final static FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType(); + public static final String EXTRACTED_TERMS_FIELD_NAME = "extracted_terms"; + public static final String UNKNOWN_QUERY_FIELD_NAME = "unknown_query"; + public static final FieldType EXTRACTED_TERMS_FIELD_TYPE = new FieldType(); static { EXTRACTED_TERMS_FIELD_TYPE.setTokenized(false); @@ -433,7 +433,7 @@ public class PercolateQueryTests extends ESTestCase { } } - private final static class CustomQuery extends Query { + private static final class CustomQuery extends Query { private final Term term; @@ -462,7 +462,7 @@ public class PercolateQueryTests extends ESTestCase { } } - private final static class ControlQuery extends Query { + private static final class ControlQuery extends Query { private final MemoryIndex memoryIndex; private final PercolateQuery.QueryStore queryStore; diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java index 75d4f408774..f4a436e7c40 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorAggregationsIT.java @@ -50,8 +50,8 @@ import static org.hamcrest.Matchers.notNullValue; public class PercolatorAggregationsIT extends ESIntegTestCase { - private final static String INDEX_NAME = "queries"; - private final static String TYPE_NAME = "query"; + private static final String INDEX_NAME = "queries"; + private static final String TYPE_NAME = "query"; @Override protected Collection> nodePlugins() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java index 701fd1b660f..7a51d8a7ab2 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorBackwardsCompatibilityTests.java @@ -48,7 +48,7 @@ import static org.hamcrest.Matchers.notNullValue; // Can'r run as IT as the test cluster is immutable and this test adds nodes during the test public class PercolatorBackwardsCompatibilityTests extends ESIntegTestCase { - private final static String INDEX_NAME = "percolator_index"; + private static final String INDEX_NAME = "percolator_index"; @Override protected Collection> nodePlugins() { diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java index ff9bf6634cd..e4e379c8f60 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java @@ -91,8 +91,8 @@ import static org.hamcrest.Matchers.nullValue; public class PercolatorIT extends ESIntegTestCase { - private final static String INDEX_NAME = "queries"; - private final static String TYPE_NAME = "query"; + private static final String INDEX_NAME = "queries"; + private static final String TYPE_NAME = "query"; @Override protected Collection> nodePlugins() { diff --git a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java index 728a9354d97..e8b49a50edb 100644 --- a/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java +++ b/plugins/analysis-phonetic/src/main/java/org/elasticsearch/index/analysis/phonetic/HaasePhonetik.java @@ -37,9 +37,9 @@ package org.elasticsearch.index.analysis.phonetic; */ public class HaasePhonetik extends KoelnerPhonetik { - private final static String[] HAASE_VARIATIONS_PATTERNS = {"OWN", "RB", "WSK", "A$", "O$", "SCH", + private static final String[] HAASE_VARIATIONS_PATTERNS = {"OWN", "RB", "WSK", "A$", "O$", "SCH", "GLI", "EAU$", "^CH", "AUX", "EUX", "ILLE"}; - private final static String[] HAASE_VARIATIONS_REPLACEMENTS = {"AUN", "RW", "RSK", "AR", "OW", "CH", + private static final String[] HAASE_VARIATIONS_REPLACEMENTS = {"AUN", "RW", "RSK", "AR", "OW", "CH", "LI", "O", "SCH", "O", "O", "I"}; @Override diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java index aa6e91fabdf..a49124749fe 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/RetryHttpInitializerWrapperTests.java @@ -44,7 +44,7 @@ import static org.hamcrest.Matchers.lessThan; public class RetryHttpInitializerWrapperTests extends ESTestCase { - static private class FailThenSuccessBackoffTransport extends MockHttpTransport { + private static class FailThenSuccessBackoffTransport extends MockHttpTransport { public int lowLevelExecCalls; int errorStatusCode; diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 6f35ea98ff0..5923e3b690e 100644 --- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -272,7 +272,7 @@ public final class GeoIpProcessor extends AbstractProcessor { // Geoip2's AddressNotFoundException is checked and due to the fact that we need run their code // inside a PrivilegedAction code block, we are forced to catch any checked exception and rethrow // it with an unchecked exception. - private final static class AddressNotFoundRuntimeException extends RuntimeException { + private static final class AddressNotFoundRuntimeException extends RuntimeException { public AddressNotFoundRuntimeException(Throwable cause) { super(cause); diff --git a/plugins/ingest-useragent/build.gradle b/plugins/ingest-useragent/build.gradle new file mode 100644 index 00000000000..86fbf243497 --- /dev/null +++ b/plugins/ingest-useragent/build.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'Ingest processor that extracts information from a user agent' + classname 'org.elasticsearch.ingest.useragent.IngestUserAgentPlugin' +} + +integTest { + cluster { + extraConfigFile 'ingest-useragent/test-regexes.yaml', 'test/test-regexes.yaml' + } +} \ No newline at end of file diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java new file mode 100644 index 00000000000..e86a8443d7e --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/IngestUserAgentPlugin.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.node.NodeModule; +import org.elasticsearch.plugins.Plugin; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +public class IngestUserAgentPlugin extends Plugin { + + private final Setting CACHE_SIZE_SETTING = Setting.longSetting("ingest.useragent.cache_size", 1000, 0, + Setting.Property.NodeScope); + + static final String DEFAULT_PARSER_NAME = "_default_"; + + public void onModule(NodeModule nodeModule) throws IOException { + Path userAgentConfigDirectory = nodeModule.getNode().getEnvironment().configFile().resolve("ingest-useragent"); + + if (Files.exists(userAgentConfigDirectory) == false && Files.isDirectory(userAgentConfigDirectory)) { + throw new IllegalStateException( + "the user agent directory [" + userAgentConfigDirectory + "] containing the regex file doesn't exist"); + } + + long cacheSize = CACHE_SIZE_SETTING.get(nodeModule.getNode().settings()); + + UserAgentCache cache = new UserAgentCache(cacheSize); + + Map userAgentParsers = createUserAgentParsers(userAgentConfigDirectory, cache); + + nodeModule.registerProcessor(UserAgentProcessor.TYPE, (registry) -> new UserAgentProcessor.Factory(userAgentParsers)); + } + + static Map createUserAgentParsers(Path userAgentConfigDirectory, UserAgentCache cache) throws IOException { + Map userAgentParsers = new HashMap<>(); + + UserAgentParser defaultParser = new UserAgentParser(DEFAULT_PARSER_NAME, + IngestUserAgentPlugin.class.getResourceAsStream("/regexes.yaml"), cache); + userAgentParsers.put(DEFAULT_PARSER_NAME, defaultParser); + + if (Files.exists(userAgentConfigDirectory) && Files.isDirectory(userAgentConfigDirectory)) { + PathMatcher pathMatcher = userAgentConfigDirectory.getFileSystem().getPathMatcher("glob:**.yaml"); + + try (Stream regexFiles = Files.find(userAgentConfigDirectory, 1, + (path, attr) -> attr.isRegularFile() && pathMatcher.matches(path))) { + Iterable iterable = regexFiles::iterator; + for (Path path : iterable) { + String parserName = path.getFileName().toString(); + try (InputStream regexStream = Files.newInputStream(path, StandardOpenOption.READ)) { + userAgentParsers.put(parserName, new UserAgentParser(parserName, regexStream, cache)); + } + } + } + } + + return Collections.unmodifiableMap(userAgentParsers); + } + +} diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java new file mode 100644 index 00000000000..d1002f2df06 --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentCache.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.common.cache.Cache; +import org.elasticsearch.common.cache.CacheBuilder; +import org.elasticsearch.ingest.useragent.UserAgentParser.Details; + +import java.util.Objects; + +class UserAgentCache { + private final Cache cache; + + UserAgentCache(long cacheSize) { + cache = CacheBuilder.builder().setMaximumWeight(cacheSize).build(); + } + + public Details get(String parserName, String userAgent) { + return cache.get(new CompositeCacheKey(parserName, userAgent)); + } + + public void put(String parserName, String userAgent, Details details) { + cache.put(new CompositeCacheKey(parserName, userAgent), details); + } + + private static final class CompositeCacheKey { + private final String parserName; + private final String userAgent; + + CompositeCacheKey(String parserName, String userAgent) { + this.parserName = parserName; + this.userAgent = userAgent; + } + + @Override + public boolean equals(Object obj) { + if(obj != null && obj instanceof CompositeCacheKey) { + CompositeCacheKey s = (CompositeCacheKey)obj; + return parserName.equals(s.parserName) && userAgent.equals(s.userAgent); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(parserName, userAgent); + } + } +} diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java new file mode 100644 index 00000000000..af764d5baf2 --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentParser.java @@ -0,0 +1,280 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final class UserAgentParser { + + private final UserAgentCache cache; + private final List uaPatterns = new ArrayList<>(); + private final List osPatterns = new ArrayList<>(); + private final List devicePatterns = new ArrayList<>(); + private final String name; + + public UserAgentParser(String name, InputStream regexStream, UserAgentCache cache) { + this.name = name; + this.cache = cache; + + try { + init(regexStream); + } catch (IOException e) { + throw new ElasticsearchParseException("error parsing regular expression file", e); + } + } + + private void init(InputStream regexStream) throws IOException { + XContentParser yamlParser = XContentFactory.xContent(XContentType.YAML).createParser(regexStream); + + XContentParser.Token token = yamlParser.nextToken(); + + if (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + for (; token != null; token = yamlParser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("user_agent_parsers")) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + uaPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), + map.get("family_replacement"), map.get("v1_replacement"), map.get("v2_replacement"), + map.get("v3_replacement"), map.get("v4_replacement"))); + } + } + else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("os_parsers")) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + osPatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), + map.get("os_replacement"), map.get("os_v1_replacement"), map.get("os_v2_replacement"), + map.get("os_v3_replacement"), map.get("os_v4_replacement"))); + } + } + else if (token == XContentParser.Token.FIELD_NAME && yamlParser.currentName().equals("device_parsers")) { + List> parserConfigurations = readParserConfigurations(yamlParser); + + for (Map map : parserConfigurations) { + devicePatterns.add(new UserAgentSubpattern(compilePattern(map.get("regex"), map.get("regex_flag")), + map.get("device_replacement"), null, null, null, null)); + } + } + } + } + + if (uaPatterns.isEmpty() && osPatterns.isEmpty() && devicePatterns.isEmpty()) { + throw new ElasticsearchParseException("not a valid regular expression file"); + } + } + + private Pattern compilePattern(String regex, String regex_flag) { + // Only flag present in the current default regexes.yaml + if (regex_flag != null && regex_flag.equals("i")) { + return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); + } else { + return Pattern.compile(regex); + } + } + + private List> readParserConfigurations(XContentParser yamlParser) throws IOException { + List > patternList = new ArrayList<>(); + + XContentParser.Token token = yamlParser.nextToken(); + if (token != XContentParser.Token.START_ARRAY) { + throw new ElasticsearchParseException("malformed regular expression file, should continue with 'array' after 'object'"); + } + + token = yamlParser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("malformed regular expression file, expecting 'object'"); + } + + while (token == XContentParser.Token.START_OBJECT) { + token = yamlParser.nextToken(); + + if (token != XContentParser.Token.FIELD_NAME) { + throw new ElasticsearchParseException("malformed regular expression file, should continue with 'field_name' after 'array'"); + } + + Map regexMap = new HashMap<>(); + for (; token == XContentParser.Token.FIELD_NAME; token = yamlParser.nextToken()) { + String fieldName = yamlParser.currentName(); + + token = yamlParser.nextToken(); + String fieldValue = yamlParser.text(); + regexMap.put(fieldName, fieldValue); + } + + patternList.add(regexMap); + + token = yamlParser.nextToken(); + } + + return patternList; + } + + List getUaPatterns() { + return uaPatterns; + } + + List getOsPatterns() { + return osPatterns; + } + + List getDevicePatterns() { + return devicePatterns; + } + + String getName() { + return name; + } + + public Details parse(String agentString) { + Details details = cache.get(name, agentString);; + + if (details == null) { + VersionedName userAgent = findMatch(uaPatterns, agentString); + VersionedName operatingSystem = findMatch(osPatterns, agentString); + VersionedName device = findMatch(devicePatterns, agentString); + + details = new Details(userAgent, operatingSystem, device); + + cache.put(name, agentString, details); + } + + return details; + } + + private VersionedName findMatch(List possiblePatterns, String agentString) { + VersionedName name; + for (UserAgentSubpattern pattern : possiblePatterns) { + name = pattern.match(agentString); + + if (name != null) { + return name; + } + } + + return null; + } + + static final class Details { + public final VersionedName userAgent; + public final VersionedName operatingSystem; + public final VersionedName device; + + public Details(VersionedName userAgent, VersionedName operatingSystem, VersionedName device) { + this.userAgent = userAgent; + this.operatingSystem = operatingSystem; + this.device = device; + } + } + + static final class VersionedName { + public final String name; + public final String major; + public final String minor; + public final String patch; + public final String build; + + public VersionedName(String name, String major, String minor, String patch, String build) { + this.name = name; + this.major = major; + this.minor = minor; + this.patch = patch; + this.build = build; + } + } + + /** + * One of: user agent, operating system, device + */ + static final class UserAgentSubpattern { + private final Pattern pattern; + private final String nameReplacement, v1Replacement, v2Replacement, v3Replacement, v4Replacement; + + public UserAgentSubpattern(Pattern pattern, String nameReplacement, + String v1Replacement, String v2Replacement, String v3Replacement, String v4Replacement) { + this.pattern = pattern; + this.nameReplacement = nameReplacement; + this.v1Replacement = v1Replacement; + this.v2Replacement = v2Replacement; + this.v3Replacement = v3Replacement; + this.v4Replacement = v4Replacement; + } + + public VersionedName match(String agentString) { + String name = null, major = null, minor = null, patch = null, build = null; + Matcher matcher = pattern.matcher(agentString); + + if (!matcher.find()) { + return null; + } + + int groupCount = matcher.groupCount(); + + if (nameReplacement != null) { + if (nameReplacement.contains("$1") && groupCount >= 1 && matcher.group(1) != null) { + name = nameReplacement.replaceFirst("\\$1", Matcher.quoteReplacement(matcher.group(1))); + } else { + name = nameReplacement; + } + } else if (groupCount >= 1) { + name = matcher.group(1); + } + + if (v1Replacement != null) { + major = v1Replacement; + } else if (groupCount >= 2) { + major = matcher.group(2); + } + + if (v2Replacement != null) { + minor = v2Replacement; + } else if (groupCount >= 3) { + minor = matcher.group(3); + } + + if (v3Replacement != null) { + patch = v3Replacement; + } else if (groupCount >= 4) { + patch = matcher.group(4); + } + + if (v4Replacement != null) { + build = v4Replacement; + } else if (groupCount >= 5) { + build = matcher.group(5); + } + + return name == null ? null : new VersionedName(name, major, minor, patch, build); + } + } +} diff --git a/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java new file mode 100644 index 00000000000..7cb829e36e2 --- /dev/null +++ b/plugins/ingest-useragent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -0,0 +1,241 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.useragent.UserAgentParser.Details; +import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; + +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException; +import static org.elasticsearch.ingest.ConfigurationUtils.readOptionalList; +import static org.elasticsearch.ingest.ConfigurationUtils.readStringProperty; + +public class UserAgentProcessor extends AbstractProcessor { + + public static final String TYPE = "useragent"; + + private final String field; + private final String targetField; + private final Set properties; + + private final UserAgentParser parser; + + public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set properties) { + super(tag); + this.field = field; + this.targetField = targetField; + this.parser = parser; + this.properties = properties; + } + + @Override + public void execute(IngestDocument ingestDocument) throws Exception { + String userAgent = ingestDocument.getFieldValue(field, String.class); + + Details uaClient = parser.parse(userAgent); + + Map uaDetails = new HashMap<>(); + for (Property property : this.properties) { + switch (property) { + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } + else { + uaDetails.put("name", "Other"); + } + break; + case MAJOR: + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + uaDetails.put("major", uaClient.userAgent.major); + } + break; + case MINOR: + if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { + uaDetails.put("minor", uaClient.userAgent.minor); + } + break; + case PATCH: + if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { + uaDetails.put("patch", uaClient.userAgent.patch); + } + break; + case BUILD: + if (uaClient.userAgent != null && uaClient.userAgent.build != null) { + uaDetails.put("build", uaClient.userAgent.build); + } + break; + case OS: + if (uaClient.operatingSystem != null) { + uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); + } + else { + uaDetails.put("os", "Other"); + } + + break; + case OS_NAME: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { + uaDetails.put("os_name", uaClient.operatingSystem.name); + } + else { + uaDetails.put("os_name", "Other"); + } + break; + case OS_MAJOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { + uaDetails.put("os_major", uaClient.operatingSystem.major); + } + break; + case OS_MINOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { + uaDetails.put("os_minor", uaClient.operatingSystem.minor); + } + break; + case DEVICE: + if (uaClient.device != null && uaClient.device.name != null) { + uaDetails.put("device", uaClient.device.name); + } + else { + uaDetails.put("device", "Other"); + } + break; + } + } + + ingestDocument.setFieldValue(targetField, uaDetails); + } + + /** To maintain compatibility with logstash-filter-useragent */ + private String buildFullOSName(VersionedName operatingSystem) { + if (operatingSystem == null || operatingSystem.name == null) { + return null; + } + + StringBuilder sb = new StringBuilder(operatingSystem.name); + + if (operatingSystem.major != null) { + sb.append(" "); + sb.append(operatingSystem.major); + + if (operatingSystem.minor != null) { + sb.append("."); + sb.append(operatingSystem.minor); + + if (operatingSystem.patch != null) { + sb.append("."); + sb.append(operatingSystem.patch); + + if (operatingSystem.build != null) { + sb.append("."); + sb.append(operatingSystem.build); + } + } + } + } + + return sb.toString(); + } + + @Override + public String getType() { + return TYPE; + } + + String getField() { + return field; + } + + String getTargetField() { + return targetField; + } + + Set getProperties() { + return properties; + } + + UserAgentParser getUaParser() { + return parser; + } + + public static final class Factory implements Processor.Factory { + + private final Map userAgentParsers; + + public Factory(Map userAgentParsers) { + this.userAgentParsers = userAgentParsers; + } + + @Override + public UserAgentProcessor create(String processorTag, Map config) throws Exception { + String field = readStringProperty(TYPE, processorTag, config, "field"); + String targetField = readStringProperty(TYPE, processorTag, config, "target_field", "useragent"); + String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); + List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); + + UserAgentParser parser = userAgentParsers.get(regexFilename); + if (parser == null) { + throw newConfigurationException(TYPE, processorTag, + "regex_file", "regex file [" + regexFilename + "] doesn't exist (has to exist at node startup)"); + } + + final Set properties; + if (propertyNames != null) { + properties = EnumSet.noneOf(Property.class); + for (String fieldName : propertyNames) { + try { + properties.add(Property.parseProperty(fieldName)); + } catch (IllegalArgumentException e) { + throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage()); + } + } + } else { + properties = EnumSet.allOf(Property.class); + } + + return new UserAgentProcessor(processorTag, field, targetField, parser, properties); + } + } + + enum Property { + + NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; + + public static Property parseProperty(String propertyName) { + try { + return valueOf(propertyName.toUpperCase(Locale.ROOT)); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + + Arrays.toString(EnumSet.allOf(Property.class).toArray())); + } + } + } +} diff --git a/plugins/ingest-useragent/src/main/resources/regexes.yaml b/plugins/ingest-useragent/src/main/resources/regexes.yaml new file mode 100644 index 00000000000..cc28842308c --- /dev/null +++ b/plugins/ingest-useragent/src/main/resources/regexes.yaml @@ -0,0 +1,4815 @@ +# Apache License, Version 2.0 +# =========================== +# +# Copyright 2009 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +user_agent_parsers: + #### SPECIAL CASES TOP #### + + # @note: iOS / OSX Applications + - regex: '(CFNetwork)(?:/(\d+)\.(\d+)\.?(\d+)?)?' + family_replacement: 'CFNetwork' + + # Pingdom + - regex: '(Pingdom.com_bot_version_)(\d+)\.(\d+)' + family_replacement: 'PingdomBot' + + # Facebook + - regex: '(facebookexternalhit)/(\d+)\.(\d+)' + family_replacement: 'FacebookBot' + + # Google Plus + - regex: 'Google.*/\+/web/snippet' + family_replacement: 'GooglePlusBot' + + # Twitter + - regex: '(Twitterbot)/(\d+)\.(\d+)' + family_replacement: 'TwitterBot' + + # Bots Pattern '/name-0.0' + - regex: '/((?:Ant-)?Nutch|[A-z]+[Bb]ot|[A-z]+[Ss]pider|Axtaris|fetchurl|Isara|ShopSalad|Tailsweep)[ \-](\d+)(?:\.(\d+)(?:\.(\d+))?)?' + # Bots Pattern 'name/0.0' + - regex: '(008|Altresium|Argus|BaiduMobaider|BoardReader|DNSGroup|DataparkSearch|EDI|Goodzer|Grub|INGRID|Infohelfer|LinkedInBot|LOOQ|Nutch|PathDefender|Peew|PostPost|Steeler|Twitterbot|VSE|WebCrunch|WebZIP|Y!J-BR[A-Z]|YahooSeeker|envolk|sproose|wminer)/(\d+)(?:\.(\d+)(?:\.(\d+))?)?' + + # MSIECrawler + - regex: '(MSIE) (\d+)\.(\d+)([a-z]\d?)?;.* MSIECrawler' + family_replacement: 'MSIECrawler' + + # Downloader ... + - regex: '(Google-HTTP-Java-Client|Apache-HttpClient|http%20client|Python-urllib|HttpMonitor|TLSProber|WinHTTP|JNLP)(?:[ /](\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + + # Bots + - regex: '(1470\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]+-Agent|AdsBot-Google(?:-[a-z]+)?|altavista|AppEngine-Google|archive.*?\.org_bot|archiver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]+)*|bingbot|BingPreview|blitzbot|BlogBridge|BoardReader(?: [A-Za-z]+)*|boitho.com-dc|BotSeer|\b\w*favicon\w*\b|\bYeti(?:-[a-z]+)?|Catchpoint bot|[Cc]harlotte|Checklinks|clumboot|Comodo HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConvera|Daumoa(?:-feedfetcher)?|Feed Seeker Bot|findlinks|Flamingo_SearchEngine|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]+-)?Googlebot(?:-[a-zA-Z]+)?|Google SketchUp|grub-client|gsa-crawler|heritrix|HiddenMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobile)?|IconSurf|IlTrovatore(?:-Setaccio)?|InfuzApp|Innovazion Crawler|InternetArchive|IP2[a-z]+Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Linguee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masidani_bot|Mediapartners-Google|Microsoft .*? Bot|mogimogi|mozDex|MJ12bot|msnbot(?:-media *)?|msrbot|netresearch|Netvibes|NewsGator[^/]*|^NING|Nutch[^/]*|Nymesis|ObjectsSearch|Orbiter|OOZBOT|PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobot|Pompos|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scooter|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|Simpy|SimplePie|SEOstats|SimpleRSS|SiteCon|Slurp|snappy|Speedy Spider|Squrl Java|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|TwitterBot|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voyager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]+|WhatWeb|WIRE|WordPress|Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s)? Link Sleuth|Xerka [A-z]+Bot|yacy(?:bot)?|Yahoo[a-z]*Seeker|Yahoo! Slurp|Yandex\w+|YodaoBot(?:-[A-z]+)?|YottaaMonitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg)(?:[ /]v?(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + + # Bots General matcher 'name/0.0' + - regex: '(?:\/[A-Za-z0-9\.]+)? *([A-Za-z0-9 \-_\!\[\]:]*(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]*))/(\d+)(?:\.(\d+)(?:\.(\d+))?)?' + # Bots General matcher 'name 0.0' + - regex: '(?:\/[A-Za-z0-9\.]+)? *([A-Za-z0-9 _\!\[\]:]*(?:[Aa]rchiver|[Ii]ndexer|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]*)) (\d+)(?:\.(\d+)(?:\.(\d+))?)?' + # Bots containing spider|scrape|bot(but not CUBOT)|Crawl + - regex: '((?:[A-z0-9]+|[A-z\-]+ ?)?(?: the )?(?:[Ss][Pp][Ii][Dd][Ee][Rr]|[Ss]crape|[A-Za-z0-9-]*(?:[^C][^Uu])[Bb]ot|[Cc][Rr][Aa][Ww][Ll])[A-z0-9]*)(?:(?:[ /]| v)(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + + # HbbTV standard defines what features the browser should understand. + # but it's like targeting "HTML5 browsers", effective browser support depends on the model + # See os_parsers if you want to target a specific TV + - regex: '(HbbTV)/(\d+)\.(\d+)\.(\d+) \(' + + # must go before Firefox to catch Chimera/SeaMonkey/Camino + - regex: '(Chimera|SeaMonkey|Camino)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)?' + + # Social Networks + # Facebook + - regex: '\[FB.*;(FBAV)/(\d+)(?:\.(\d+)(?:\.(\d)+)?)?' + family_replacement: 'Facebook' + # Pinterest + - regex: '\[(Pinterest)/[^\]]+\]' + - regex: '(Pinterest)(?: for Android(?: Tablet)?)?/(\d+)(?:\.(\d+)(?:\.(\d)+)?)?' + + # Firefox + - regex: '(Pale[Mm]oon)/(\d+)\.(\d+)\.?(\d+)?' + family_replacement: 'Pale Moon (Firefox Variant)' + - regex: '(Fennec)/(\d+)\.(\d+)\.?([ab]?\d+[a-z]*)' + family_replacement: 'Firefox Mobile' + - regex: '(Fennec)/(\d+)\.(\d+)(pre)' + family_replacement: 'Firefox Mobile' + - regex: '(Fennec)/(\d+)\.(\d+)' + family_replacement: 'Firefox Mobile' + - regex: '(?:Mobile|Tablet);.*(Firefox)/(\d+)\.(\d+)' + family_replacement: 'Firefox Mobile' + - regex: '(Namoroka|Shiretoko|Minefield)/(\d+)\.(\d+)\.(\d+(?:pre)?)' + family_replacement: 'Firefox ($1)' + - regex: '(Firefox)/(\d+)\.(\d+)(a\d+[a-z]*)' + family_replacement: 'Firefox Alpha' + - regex: '(Firefox)/(\d+)\.(\d+)(b\d+[a-z]*)' + family_replacement: 'Firefox Beta' + - regex: '(Firefox)-(?:\d+\.\d+)?/(\d+)\.(\d+)(a\d+[a-z]*)' + family_replacement: 'Firefox Alpha' + - regex: '(Firefox)-(?:\d+\.\d+)?/(\d+)\.(\d+)(b\d+[a-z]*)' + family_replacement: 'Firefox Beta' + - regex: '(Namoroka|Shiretoko|Minefield)/(\d+)\.(\d+)([ab]\d+[a-z]*)?' + family_replacement: 'Firefox ($1)' + - regex: '(Firefox).*Tablet browser (\d+)\.(\d+)\.(\d+)' + family_replacement: 'MicroB' + - regex: '(MozillaDeveloperPreview)/(\d+)\.(\d+)([ab]\d+[a-z]*)?' + - regex: '(FxiOS)/(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?' + family_replacement: 'Firefox iOS' + + # e.g.: Flock/2.0b2 + - regex: '(Flock)/(\d+)\.(\d+)(b\d+?)' + + # RockMelt + - regex: '(RockMelt)/(\d+)\.(\d+)\.(\d+)' + + # e.g.: Fennec/0.9pre + - regex: '(Navigator)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Netscape' + + - regex: '(Navigator)/(\d+)\.(\d+)([ab]\d+)' + family_replacement: 'Netscape' + + - regex: '(Netscape6)/(\d+)\.(\d+)\.?([ab]?\d+)?' + family_replacement: 'Netscape' + + - regex: '(MyIBrow)/(\d+)\.(\d+)' + family_replacement: 'My Internet Browser' + + # Opera will stop at 9.80 and hide the real version in the Version string. + # see: http://dev.opera.com/articles/view/opera-ua-string-changes/ + - regex: '(Opera Tablet).*Version/(\d+)\.(\d+)(?:\.(\d+))?' + - regex: '(Opera Mini)(?:/att)?/?(\d+)?(?:\.(\d+))?(?:\.(\d+))?' + - regex: '(Opera)/.+Opera Mobi.+Version/(\d+)\.(\d+)' + family_replacement: 'Opera Mobile' + - regex: '(Opera)/(\d+)\.(\d+).+Opera Mobi' + family_replacement: 'Opera Mobile' + - regex: 'Opera Mobi.+(Opera)(?:/|\s+)(\d+)\.(\d+)' + family_replacement: 'Opera Mobile' + - regex: 'Opera Mobi' + family_replacement: 'Opera Mobile' + - regex: '(Opera)/9.80.*Version/(\d+)\.(\d+)(?:\.(\d+))?' + + # Opera 14 for Android uses a WebKit render engine. + - regex: '(?:Mobile Safari).*(OPR)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Opera Mobile' + + # Opera >=15 for Desktop is similar to Chrome but includes an "OPR" Version string. + - regex: '(?:Chrome).*(OPR)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Opera' + + # Opera Coast + - regex: '(Coast)/(\d+).(\d+).(\d+)' + family_replacement: 'Opera Coast' + + # Opera Mini for iOS (from version 8.0.0) + - regex: '(OPiOS)/(\d+).(\d+).(\d+)' + family_replacement: 'Opera Mini' + + # Palm WebOS looks a lot like Safari. + - regex: '(hpw|web)OS/(\d+)\.(\d+)(?:\.(\d+))?' + family_replacement: 'webOS Browser' + + # LuaKit has no version info. + # http://luakit.org/projects/luakit/ + - regex: '(luakit)' + family_replacement: 'LuaKit' + + # Snowshoe + - regex: '(Snowshoe)/(\d+)\.(\d+).(\d+)' + + # Lightning (for Thunderbird) + # http://www.mozilla.org/projects/calendar/lightning/ + - regex: '(Lightning)/(\d+)\.(\d+)\.?((?:[ab]?\d+[a-z]*)|(?:\d*))' + + # Swiftfox + - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+(?:pre)?) \(Swiftfox\)' + family_replacement: 'Swiftfox' + - regex: '(Firefox)/(\d+)\.(\d+)([ab]\d+[a-z]*)? \(Swiftfox\)' + family_replacement: 'Swiftfox' + + # Rekonq + - regex: '(rekonq)/(\d+)\.(\d+)\.?(\d+)? Safari' + family_replacement: 'Rekonq' + - regex: 'rekonq' + family_replacement: 'Rekonq' + + # Conkeror lowercase/uppercase + # http://conkeror.org/ + - regex: '(conkeror|Conkeror)/(\d+)\.(\d+)\.?(\d+)?' + family_replacement: 'Conkeror' + + # catches lower case konqueror + - regex: '(konqueror)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Konqueror' + + - regex: '(WeTab)-Browser' + + - regex: '(Comodo_Dragon)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Comodo Dragon' + + - regex: '(Symphony) (\d+).(\d+)' + + - regex: '(Minimo)' + + - regex: 'PLAYSTATION 3.+WebKit' + family_replacement: 'NetFront NX' + - regex: 'PLAYSTATION 3' + family_replacement: 'NetFront' + - regex: '(PlayStation Portable)' + family_replacement: 'NetFront' + - regex: '(PlayStation Vita)' + family_replacement: 'NetFront NX' + + - regex: 'AppleWebKit.+ (NX)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'NetFront NX' + - regex: '(Nintendo 3DS)' + family_replacement: 'NetFront NX' + + # Amazon Silk, should go before Safari and Chrome Mobile + - regex: '(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+))?' + family_replacement: 'Amazon Silk' + + + # @ref: http://www.puffinbrowser.com + - regex: '(Puffin)/(\d+)\.(\d+)(?:\.(\d+))?' + + # Edge Mobile + - regex: 'Windows Phone .*(Edge)/(\d+)\.(\d+)' + family_replacement: 'Edge Mobile' + + # Samsung Internet (based on Chrome, but lacking some features) + - regex: '(SamsungBrowser)/(\d+)\.(\d+)' + family_replacement: 'Samsung Internet' + + # Chrome Mobile + - regex: '(CrMo)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Chrome Mobile' + - regex: '(CriOS)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Chrome Mobile iOS' + - regex: '(Chrome)/(\d+)\.(\d+)\.(\d+)\.(\d+) Mobile' + family_replacement: 'Chrome Mobile' + + # Chrome Frame must come before MSIE. + - regex: '(chromeframe)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Chrome Frame' + + # UC Browser + - regex: '(UCBrowser)[ /](\d+)\.(\d+)\.(\d+)' + family_replacement: 'UC Browser' + - regex: '(UC Browser)[ /](\d+)\.(\d+)\.(\d+)' + - regex: '(UC Browser|UCBrowser|UCWEB)(\d+)\.(\d+)\.(\d+)' + family_replacement: 'UC Browser' + + # Tizen Browser (second case included in browser/major.minor regex) + - regex: '(SLP Browser)/(\d+)\.(\d+)' + family_replacement: 'Tizen Browser' + + # Sogou Explorer 2.X + - regex: '(SE 2\.X) MetaSr (\d+)\.(\d+)' + family_replacement: 'Sogou Explorer' + + # Baidu Browsers (desktop spoofs chrome & IE, explorer is mobile) + - regex: '(baidubrowser)[/\s](\d+)' + family_replacement: 'Baidu Browser' + - regex: '(FlyFlow)/(\d+)\.(\d+)' + family_replacement: 'Baidu Explorer' + + # QQ Browsers + - regex: '(MQQBrowser/Mini)(?:(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + family_replacement: 'QQ Browser Mini' + - regex: '(MQQBrowser)(?:/(\d+)(?:\.(\d+)(?:\.(\d+))?)?)?' + family_replacement: 'QQ Browser Mobile' + - regex: '(QQBrowser)(?:/(\d+)(?:\.(\d+)\.(\d+)(?:\.(\d+))?)?)?' + family_replacement: 'QQ Browser' + + # Rackspace Monitoring + - regex: '(Rackspace Monitoring)/(\d+)\.(\d+)' + family_replacement: 'RackspaceBot' + + # PyAMF + - regex: '(PyAMF)/(\d+)\.(\d+)\.(\d+)' + + # Yandex Browser + - regex: '(YaBrowser)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Yandex Browser' + + # Mail.ru Amigo/Internet Browser (Chromium-based) + - regex: '(Chrome)/(\d+)\.(\d+)\.(\d+).* MRCHROME' + family_replacement: 'Mail.ru Chromium Browser' + + # AOL Browser (IE-based) + - regex: '(AOL) (\d+)\.(\d+); AOLBuild (\d+)' + + #### END SPECIAL CASES TOP #### + + #### MAIN CASES - this catches > 50% of all browsers #### + + # Browser/major_version.minor_version.beta_version + - regex: '(AdobeAIR|FireWeb|Jasmine|ANTGalio|Midori|Fresco|Lobo|PaleMoon|Maxthon|Lynx|OmniWeb|Dillo|Camino|Demeter|Fluid|Fennec|Epiphany|Shiira|Sunrise|Spotify|Flock|Netscape|Lunascape|WebPilot|NetFront|Netfront|Konqueror|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|Opera Mini|iCab|NetNewsWire|ThunderBrowse|Iris|UP\.Browser|Bunjalloo|Google Earth|Raven for Mac|Openwave)/(\d+)\.(\d+)\.(\d+)' + + # Outlook 2007 + - regex: 'Microsoft Office Outlook 12\.\d+\.\d+|MSOffice 12' + family_replacement: 'Outlook' + v1_replacement: '2007' + + # Outlook 2010 + - regex: 'Microsoft Outlook 14\.\d+\.\d+|MSOffice 14' + family_replacement: 'Outlook' + v1_replacement: '2010' + + # Outlook 2013 + - regex: 'Microsoft Outlook 15\.\d+\.\d+' + family_replacement: 'Outlook' + v1_replacement: '2013' + + # Outlook 2016 + - regex: 'Microsoft Outlook (?:Mail )?16\.\d+\.\d+' + family_replacement: 'Outlook' + v1_replacement: '2016' + + # Windows Live Mail + - regex: 'Outlook-Express\/7\.0.*' + family_replacement: 'Windows Live Mail' + + # Apple Air Mail + - regex: '(Airmail) (\d+)\.(\d+)(?:\.(\d+))?' + + # Thunderbird + - regex: '(Thunderbird)/(\d+)\.(\d+)\.(\d+(?:pre)?)' + family_replacement: 'Thunderbird' + + # Vivaldi uses "Vivaldi" + - regex: '(Vivaldi)/(\d+)\.(\d+)\.(\d+)' + + # Edge/major_version.minor_version + - regex: '(Edge)/(\d+)\.(\d+)' + + # Brave Browser https://brave.com/ + - regex: '(brave)/(\d+)\.(\d+)\.(\d+) Chrome' + family_replacement: 'Brave' + + # Chrome/Chromium/major_version.minor_version.beta_version + - regex: '(Chromium|Chrome)/(\d+)\.(\d+)\.(\d+)' + + # Dolphin Browser + # @ref: http://www.dolphin.com + - regex: '\b(Dolphin)(?: |HDCN/|/INT\-)(\d+)\.(\d+)\.?(\d+)?' + + # Browser/major_version.minor_version + - regex: '(bingbot|Bolt|Jasmine|IceCat|Skyfire|Midori|Maxthon|Lynx|Arora|IBrowse|Dillo|Camino|Shiira|Fennec|Phoenix|Chrome|Flock|Netscape|Lunascape|Epiphany|WebPilot|Opera Mini|Opera|NetFront|Netfront|Konqueror|Googlebot|SeaMonkey|Kazehakase|Vienna|Iceape|Iceweasel|IceWeasel|Iron|K-Meleon|Sleipnir|Galeon|GranParadiso|iCab|iTunes|MacAppStore|NetNewsWire|Space Bison|Stainless|Orca|Dolfin|BOLT|Minimo|Tizen Browser|Polaris|Abrowser|Planetweb|ICE Browser|mDolphin|qutebrowser|Otter|QupZilla)/(\d+)\.(\d+)\.?(\d+)?' + + # Chrome/Chromium/major_version.minor_version + - regex: '(Chromium|Chrome)/(\d+)\.(\d+)' + + ########## + # IE Mobile needs to happen before Android to catch cases such as: + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; ANZ821)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Orange)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Vodafone)... + ########## + + # IE Mobile + - regex: '(IEMobile)[ /](\d+)\.(\d+)' + family_replacement: 'IE Mobile' + + # Browser major_version.minor_version.beta_version (space instead of slash) + - regex: '(iRider|Crazy Browser|SkipStone|iCab|Lunascape|Sleipnir|Maemo Browser) (\d+)\.(\d+)\.(\d+)' + # Browser major_version.minor_version (space instead of slash) + - regex: '(iCab|Lunascape|Opera|Android|Jasmine|Polaris) (\d+)\.(\d+)\.?(\d+)?' + + # Kindle WebKit + - regex: '(Kindle)/(\d+)\.(\d+)' + + # weird android UAs + - regex: '(Android) Donut' + v1_replacement: '1' + v2_replacement: '2' + + - regex: '(Android) Eclair' + v1_replacement: '2' + v2_replacement: '1' + + - regex: '(Android) Froyo' + v1_replacement: '2' + v2_replacement: '2' + + - regex: '(Android) Gingerbread' + v1_replacement: '2' + v2_replacement: '3' + + - regex: '(Android) Honeycomb' + v1_replacement: '3' + + # desktop mode + # http://www.anandtech.com/show/3982/windows-phone-7-review + - regex: '(MSIE) (\d+)\.(\d+).*XBLWP7' + family_replacement: 'IE Large Screen' + + #### END MAIN CASES #### + + #### SPECIAL CASES #### + - regex: '(Obigo)InternetBrowser' + - regex: '(Obigo)\-Browser' + - regex: '(Obigo|OBIGO)[^\d]*(\d+)(?:.(\d+))?' + family_replacement: 'Obigo' + + - regex: '(MAXTHON|Maxthon) (\d+)\.(\d+)' + family_replacement: 'Maxthon' + - regex: '(Maxthon|MyIE2|Uzbl|Shiira)' + v1_replacement: '0' + + - regex: '(BrowseX) \((\d+)\.(\d+)\.(\d+)' + + - regex: '(NCSA_Mosaic)/(\d+)\.(\d+)' + family_replacement: 'NCSA Mosaic' + + # Polaris/d.d is above + - regex: '(POLARIS)/(\d+)\.(\d+)' + family_replacement: 'Polaris' + - regex: '(Embider)/(\d+)\.(\d+)' + family_replacement: 'Polaris' + + - regex: '(BonEcho)/(\d+)\.(\d+)\.?([ab]?\d+)?' + family_replacement: 'Bon Echo' + + # @note: iOS / OSX Applications + - regex: '(iPod|iPhone|iPad).+Version/(\d+)\.(\d+)(?:\.(\d+))?.* Safari' + family_replacement: 'Mobile Safari' + - regex: '(iPod|iPhone|iPad).+Version/(\d+)\.(\d+)(?:\.(\d+))?' + family_replacement: 'Mobile Safari UI/WKWebView' + - regex: '(iPod|iPhone|iPad);.*CPU.*OS (\d+)_(\d+)(?:_(\d+))?.*Mobile.* Safari' + family_replacement: 'Mobile Safari' + - regex: '(iPod|iPhone|iPad);.*CPU.*OS (\d+)_(\d+)(?:_(\d+))?.*Mobile' + family_replacement: 'Mobile Safari UI/WKWebView' + - regex: '(iPod|iPhone|iPad).* Safari' + family_replacement: 'Mobile Safari' + - regex: '(iPod|iPhone|iPad)' + family_replacement: 'Mobile Safari UI/WKWebView' + + - regex: '(AvantGo) (\d+).(\d+)' + + - regex: '(OneBrowser)/(\d+).(\d+)' + family_replacement: 'ONE Browser' + + - regex: '(Avant)' + v1_replacement: '1' + + # This is the Tesla Model S (see similar entry in device parsers) + - regex: '(QtCarBrowser)' + v1_replacement: '1' + + - regex: '^(iBrowser/Mini)(\d+).(\d+)' + family_replacement: 'iBrowser Mini' + - regex: '^(iBrowser|iRAPP)/(\d+).(\d+)' + + # nokia browsers + # based on: http://www.developer.nokia.com/Community/Wiki/User-Agent_headers_for_Nokia_devices + - regex: '^(Nokia)' + family_replacement: 'Nokia Services (WAP) Browser' + - regex: '(NokiaBrowser)/(\d+)\.(\d+).(\d+)\.(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(NokiaBrowser)/(\d+)\.(\d+).(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(NokiaBrowser)/(\d+)\.(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(BrowserNG)/(\d+)\.(\d+).(\d+)' + family_replacement: 'Nokia Browser' + - regex: '(Series60)/5\.0' + family_replacement: 'Nokia Browser' + v1_replacement: '7' + v2_replacement: '0' + - regex: '(Series60)/(\d+)\.(\d+)' + family_replacement: 'Nokia OSS Browser' + - regex: '(S40OviBrowser)/(\d+)\.(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Ovi Browser' + - regex: '(Nokia)[EN]?(\d+)' + + # BlackBerry devices + - regex: '(PlayBook).+RIM Tablet OS (\d+)\.(\d+)\.(\d+)' + family_replacement: 'BlackBerry WebKit' + - regex: '(Black[bB]erry|BB10).+Version/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'BlackBerry WebKit' + - regex: '(Black[bB]erry)\s?(\d+)' + family_replacement: 'BlackBerry' + + - regex: '(OmniWeb)/v(\d+)\.(\d+)' + + - regex: '(Blazer)/(\d+)\.(\d+)' + family_replacement: 'Palm Blazer' + + - regex: '(Pre)/(\d+)\.(\d+)' + family_replacement: 'Palm Pre' + + # fork of Links + - regex: '(ELinks)/(\d+)\.(\d+)' + - regex: '(ELinks) \((\d+)\.(\d+)' + - regex: '(Links) \((\d+)\.(\d+)' + + - regex: '(QtWeb) Internet Browser/(\d+)\.(\d+)' + + #- regex: '\(iPad;.+(Version)/(\d+)\.(\d+)(?:\.(\d+))?.*Safari/' + # family_replacement: 'iPad' + + # Phantomjs, should go before Safari + - regex: '(PhantomJS)/(\d+)\.(\d+)\.(\d+)' + + # WebKit Nightly + - regex: '(AppleWebKit)/(\d+)\.?(\d+)?\+ .* Safari' + family_replacement: 'WebKit Nightly' + + # Safari + - regex: '(Version)/(\d+)\.(\d+)(?:\.(\d+))?.*Safari/' + family_replacement: 'Safari' + # Safari didn't provide "Version/d.d.d" prior to 3.0 + - regex: '(Safari)/\d+' + + - regex: '(OLPC)/Update(\d+)\.(\d+)' + + - regex: '(OLPC)/Update()\.(\d+)' + v1_replacement: '0' + + - regex: '(SEMC\-Browser)/(\d+)\.(\d+)' + + - regex: '(Teleca)' + family_replacement: 'Teleca Browser' + + - regex: '(Phantom)/V(\d+)\.(\d+)' + family_replacement: 'Phantom Browser' + + - regex: 'Trident(.*)rv.(\d+)\.(\d+)' + family_replacement: 'IE' + + # Espial + - regex: '(Espial)/(\d+)(?:\.(\d+))?(?:\.(\d+))?' + + # Apple Mail + + # apple mail - not directly detectable, have it after Safari stuff + - regex: '(AppleWebKit)/(\d+)\.(\d+)\.(\d+)' + family_replacement: 'Apple Mail' + + # AFTER THE EDGE CASES ABOVE! + # AFTER IE11 + # BEFORE all other IE + - regex: '(Firefox)/(\d+)\.(\d+)\.(\d+)' + - regex: '(Firefox)/(\d+)\.(\d+)(pre|[ab]\d+[a-z]*)?' + + - regex: '([MS]?IE) (\d+)\.(\d+)' + family_replacement: 'IE' + + - regex: '(python-requests)/(\d+)\.(\d+)' + family_replacement: 'Python Requests' + + - regex: '(Java)[/ ]{0,1}\d+\.(\d+)\.(\d+)[_-]*([a-zA-Z0-9]+)*' + + # Roku Digital-Video-Players https://www.roku.com/ + - regex: '^(Roku)/DVP-(\d+)\.(\d+)' + +os_parsers: + ########## + # HbbTV vendors + ########## + + # starts with the easy one : Panasonic seems consistent across years, hope it will continue + #HbbTV/1.1.1 (;Panasonic;VIERA 2011;f.532;0071-0802 2000-0000;) + #HbbTV/1.1.1 (;Panasonic;VIERA 2012;1.261;0071-3103 2000-0000;) + #HbbTV/1.2.1 (;Panasonic;VIERA 2013;3.672;4101-0003 0002-0000;) + #- regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Panasonic);VIERA ([0-9]{4});' + + # Sony is consistent too but do not place year like the other + # Opera/9.80 (Linux armv7l; HbbTV/1.1.1 (; Sony; KDL32W650A; PKG3.211EUA; 2013;); ) Presto/2.12.362 Version/12.11 + # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Sony; KDL40HX751; PKG1.902EUA; 2012;);; en) Presto/2.10.250 Version/11.60 + # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Sony; KDL22EX320; PKG4.017EUA; 2011;);; en) Presto/2.7.61 Version/11.00 + #- regex: 'HbbTV/\d+\.\d+\.\d+ \(; (Sony);.*;.*; ([0-9]{4});\)' + + + # LG is consistent too, but we need to add manually the year model + #Mozilla/5.0 (Unknown; Linux armv7l) AppleWebKit/537.1+ (KHTML, like Gecko) Safari/537.1+ HbbTV/1.1.1 ( ;LGE ;NetCast 4.0 ;03.20.30 ;1.0M ;) + #Mozilla/5.0 (DirectFB; Linux armv7l) AppleWebKit/534.26+ (KHTML, like Gecko) Version/5.0 Safari/534.26+ HbbTV/1.1.1 ( ;LGE ;NetCast 3.0 ;1.0 ;1.0M ;) + - regex: 'HbbTV/\d+\.\d+\.\d+ \( ;(LG)E ;NetCast 4.0' + os_v1_replacement: '2013' + - regex: 'HbbTV/\d+\.\d+\.\d+ \( ;(LG)E ;NetCast 3.0' + os_v1_replacement: '2012' + + # Samsung is on its way of normalizing their user-agent + # HbbTV/1.1.1 (;Samsung;SmartTV2013;T-FXPDEUC-1102.2;;) WebKit + # HbbTV/1.1.1 (;Samsung;SmartTV2013;T-MST12DEUC-1102.1;;) WebKit + # HbbTV/1.1.1 (;Samsung;SmartTV2012;;;) WebKit + # HbbTV/1.1.1 (;;;;;) Maple_2011 + - regex: 'HbbTV/1.1.1 \(;;;;;\) Maple_2011' + os_replacement: 'Samsung' + os_v1_replacement: '2011' + # manage the two models of 2013 + - regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});.*FXPDEUC' + os_v2_replacement: 'UE40F7000' + - regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});.*MST12DEUC' + os_v2_replacement: 'UE32F4500' + # generic Samsung (works starting in 2012) + #- regex: 'HbbTV/\d+\.\d+\.\d+ \(;(Samsung);SmartTV([0-9]{4});' + + # Philips : not found any other way than a manual mapping + # Opera/9.80 (Linux mips; U; HbbTV/1.1.1 (; Philips; ; ; ; ) CE-HTML/1.0 NETTV/4.1.3 PHILIPSTV/1.1.1; en) Presto/2.10.250 Version/11.60 + # Opera/9.80 (Linux mips ; U; HbbTV/1.1.1 (; Philips; ; ; ; ) CE-HTML/1.0 NETTV/3.2.1; en) Presto/2.6.33 Version/10.70 + - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/4' + os_v1_replacement: '2013' + - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/3' + os_v1_replacement: '2012' + - regex: 'HbbTV/1.1.1 \(; (Philips);.*NETTV/2' + os_v1_replacement: '2011' + + # the HbbTV emulator developers use HbbTV/1.1.1 (;;;;;) firetv-firefox-plugin 1.1.20 + - regex: 'HbbTV/\d+\.\d+\.\d+.*(firetv)-firefox-plugin (\d+).(\d+).(\d+)' + os_replacement: 'FireHbbTV' + + # generic HbbTV, hoping to catch manufacturer name (always after 2nd comma) and the first string that looks like a 2011-2019 year + - regex: 'HbbTV/\d+\.\d+\.\d+ \(.*; ?([a-zA-Z]+) ?;.*(201[1-9]).*\)' + + ########## + # @note: Windows Phone needs to come before Windows NT 6.1 *and* before Android to catch cases such as: + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; ANZ821)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Orange)... + # Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 920; Vodafone)... + ########## + + - regex: '(Windows Phone) (?:OS[ /])?(\d+)\.(\d+)' + + ########## + # Android + # can actually detect rooted android os. do we care? + ########## + - regex: '(Android)[ \-/](\d+)\.(\d+)(?:[.\-]([a-z0-9]+))?' + + - regex: '(Android) Donut' + os_v1_replacement: '1' + os_v2_replacement: '2' + + - regex: '(Android) Eclair' + os_v1_replacement: '2' + os_v2_replacement: '1' + + - regex: '(Android) Froyo' + os_v1_replacement: '2' + os_v2_replacement: '2' + + - regex: '(Android) Gingerbread' + os_v1_replacement: '2' + os_v2_replacement: '3' + + - regex: '(Android) Honeycomb' + os_v1_replacement: '3' + + # UCWEB + - regex: '^UCWEB.*; (Adr) (\d+)\.(\d+)(?:[.\-]([a-z0-9]+))?;' + os_replacement: 'Android' + - regex: '^UCWEB.*; (iPad OS|iPh OS) (\d+)_(\d+)(?:_(\d+))?;' + os_replacement: 'iOS' + - regex: '^UCWEB.*; (wds) (\d+)\.(\d+)(?:\.(\d+))?;' + os_replacement: 'Windows Phone' + # JUC + - regex: '^(JUC).*; ?U; ?(?:Android)?(\d+)\.(\d+)(?:[\.\-]([a-z0-9]+))?' + os_replacement: 'Android' + + ########## + # Kindle Android + ########## + - regex: '(Silk-Accelerated=[a-z]{4,5})' + os_replacement: 'Android' + + ########## + # Windows + # http://en.wikipedia.org/wiki/Windows_NT#Releases + # possibility of false positive when different marketing names share same NT kernel + # e.g. windows server 2003 and windows xp + # lots of ua strings have Windows NT 4.1 !?!?!?!? !?!? !? !????!?! !!! ??? !?!?! ? + # (very) roughly ordered in terms of frequency of occurence of regex (win xp currently most frequent, etc) + ########## + + # ie mobile desktop mode + # spoofs nt 6.1. must come before windows 7 + - regex: '(XBLWP7)' + os_replacement: 'Windows Phone' + + # @note: This needs to come before Windows NT 6.1 + - regex: '(Windows ?Mobile)' + os_replacement: 'Windows Mobile' + + - regex: '(Windows (?:NT 5\.2|NT 5\.1))' + os_replacement: 'Windows XP' + + - regex: '(Windows NT 6\.1)' + os_replacement: 'Windows 7' + + - regex: '(Windows NT 6\.0)' + os_replacement: 'Windows Vista' + + - regex: '(Win 9x 4\.90)' + os_replacement: 'Windows ME' + + - regex: '(Windows 98|Windows XP|Windows ME|Windows 95|Windows CE|Windows 7|Windows NT 4\.0|Windows Vista|Windows 2000|Windows 3.1)' + + - regex: '(Windows NT 6\.2; ARM;)' + os_replacement: 'Windows RT' + - regex: '(Windows NT 6\.2)' + os_replacement: 'Windows 8' + + - regex: '(Windows NT 6\.3; ARM;)' + os_replacement: 'Windows RT 8.1' + - regex: '(Windows NT 6\.3)' + os_replacement: 'Windows 8.1' + + - regex: '(Windows NT 6\.4)' + os_replacement: 'Windows 10' + - regex: '(Windows NT 10\.0)' + os_replacement: 'Windows 10' + + - regex: '(Windows NT 5\.0)' + os_replacement: 'Windows 2000' + + - regex: '(WinNT4.0)' + os_replacement: 'Windows NT 4.0' + + - regex: '(Windows ?CE)' + os_replacement: 'Windows CE' + + - regex: 'Win ?(95|98|3.1|NT|ME|2000)' + os_replacement: 'Windows $1' + + - regex: 'Win16' + os_replacement: 'Windows 3.1' + + - regex: 'Win32' + os_replacement: 'Windows 95' + + ########## + # Tizen OS from Samsung + # spoofs Android so pushing it above + ########## + - regex: '(Tizen)/(\d+)\.(\d+)' + + ########## + # Mac OS + # @ref: http://en.wikipedia.org/wiki/Mac_OS_X#Versions + # @ref: http://www.puredarwin.org/curious/versions + ########## + - regex: '((?:Mac ?|; )OS X)[\s/](?:(\d+)[_.](\d+)(?:[_.](\d+))?|Mach-O)' + os_replacement: 'Mac OS X' + # Leopard + - regex: ' (Dar)(win)/(9).(\d+).*\((?:i386|x86_64|Power Macintosh)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '5' + # Snow Leopard + - regex: ' (Dar)(win)/(10).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '6' + # Lion + - regex: ' (Dar)(win)/(11).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '7' + # Mountain Lion + - regex: ' (Dar)(win)/(12).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '8' + # Mavericks + - regex: ' (Dar)(win)/(13).(\d+).*\((?:i386|x86_64)\)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '9' + # Yosemite is Darwin/14.x but patch versions are inconsistent in the Darwin string; + # more accurately covered by CFNetwork regexes downstream + + # IE on Mac doesn't specify version number + - regex: 'Mac_PowerPC' + os_replacement: 'Mac OS' + + # builds before tiger don't seem to specify version? + + # ios devices spoof (mac os x), so including intel/ppc prefixes + - regex: '(?:PPC|Intel) (Mac OS X)' + + ########## + # iOS + # http://en.wikipedia.org/wiki/IOS_version_history + ########## + # keep this above generic iOS, since AppleTV UAs contain 'CPU OS' + - regex: '(Apple\s?TV)(?:/(\d+)\.(\d+))?' + os_replacement: 'ATV OS X' + + - regex: '(CPU OS|iPhone OS|CPU iPhone) +(\d+)[_\.](\d+)(?:[_\.](\d+))?' + os_replacement: 'iOS' + + # remaining cases are mostly only opera uas, so catch opera as to not catch iphone spoofs + - regex: '(iPhone|iPad|iPod); Opera' + os_replacement: 'iOS' + + # few more stragglers + - regex: '(iPhone|iPad|iPod).*Mac OS X.*Version/(\d+)\.(\d+)' + os_replacement: 'iOS' + + # CFNetwork/Darwin - The specific CFNetwork or Darwin version determines + # whether the os maps to Mac OS, or iOS, or just Darwin. + # See: http://user-agents.me/cfnetwork-version-list + - regex: '(CFNetwork)/(5)48\.0\.3.* Darwin/11\.0\.0' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(5)48\.(0)\.4.* Darwin/(1)1\.0\.0' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(5)48\.(1)\.4' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(4)85\.1(3)\.9' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(6)09\.(1)\.4' + os_replacement: 'iOS' + - regex: '(CFNetwork)/(6)(0)9' + os_replacement: 'iOS' + - regex: '(CFNetwork)/6(7)2\.(1)\.13' + os_replacement: 'iOS' + - regex: '(CFNetwork)/6(7)2\.(1)\.(1)4' + os_replacement: 'iOS' + - regex: '(CF)(Network)/6(7)(2)\.1\.15' + os_replacement: 'iOS' + os_v1_replacement: '7' + os_v2_replacement: '1' + - regex: '(CFNetwork)/6(7)2\.(0)\.(?:2|8)' + os_replacement: 'iOS' + - regex: '(CFNetwork)/709\.1' + os_replacement: 'iOS' + os_v1_replacement: '8' + os_v2_replacement: '0.b5' + - regex: '(CF)(Network)/711\.(\d)' + os_replacement: 'iOS' + os_v1_replacement: '8' + - regex: '(CF)(Network)/(720)\.(\d)' + os_replacement: 'Mac OS X' + os_v1_replacement: '10' + os_v2_replacement: '10' + - regex: '(CF)(Network)/758\.(\d)' + os_replacement: 'iOS' + os_v1_replacement: '9' + + ########## + # CFNetwork iOS Apps + # @ref: https://en.wikipedia.org/wiki/Darwin_(operating_system)#Release_history + ########## + - regex: 'CFNetwork/.* Darwin/(9)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '1' + - regex: 'CFNetwork/.* Darwin/(10)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '4' + - regex: 'CFNetwork/.* Darwin/(11)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '5' + - regex: 'CFNetwork/.* Darwin/(13)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '6' + - regex: 'CFNetwork/6.* Darwin/(14)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '7' + - regex: 'CFNetwork/7.* Darwin/(14)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '8' + os_v2_replacement: '0' + - regex: 'CFNetwork/7.* Darwin/(15)\.\d+' + os_replacement: 'iOS' + os_v1_replacement: '9' + os_v2_replacement: '0' + # iOS Apps + - regex: '\b(iOS[ /]|iPhone(?:/| v|[ _]OS[/,]|; | OS : |\d,\d/|\d,\d; )|iPad/)(\d{1,2})[_\.](\d{1,2})(?:[_\.](\d+))?' + os_replacement: 'iOS' + + ########## + # Apple TV + ########## + - regex: '(tvOS)/(\d+).(\d+)' + os_replacement: 'tvOS' + + ########## + # Chrome OS + # if version 0.0.0, probably this stuff: + # http://code.google.com/p/chromium-os/issues/detail?id=11573 + # http://code.google.com/p/chromium-os/issues/detail?id=13790 + ########## + - regex: '(CrOS) [a-z0-9_]+ (\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'Chrome OS' + + ########## + # Linux distros + ########## + - regex: '([Dd]ebian)' + os_replacement: 'Debian' + - regex: '(Linux Mint)(?:/(\d+))?' + - regex: '(Mandriva)(?: Linux)?/(?:[\d.-]+m[a-z]{2}(\d+).(\d))?' + + ########## + # Symbian + Symbian OS + # http://en.wikipedia.org/wiki/History_of_Symbian + ########## + - regex: '(Symbian[Oo][Ss])[/ ](\d+)\.(\d+)' + os_replacement: 'Symbian OS' + - regex: '(Symbian/3).+NokiaBrowser/7\.3' + os_replacement: 'Symbian^3 Anna' + - regex: '(Symbian/3).+NokiaBrowser/7\.4' + os_replacement: 'Symbian^3 Belle' + - regex: '(Symbian/3)' + os_replacement: 'Symbian^3' + - regex: '\b(Series 60|SymbOS|S60Version|S60V\d|S60\b)' + os_replacement: 'Symbian OS' + - regex: '(MeeGo)' + - regex: 'Symbian [Oo][Ss]' + os_replacement: 'Symbian OS' + - regex: 'Series40;' + os_replacement: 'Nokia Series 40' + - regex: 'Series30Plus;' + os_replacement: 'Nokia Series 30 Plus' + + ########## + # BlackBerry devices + ########## + - regex: '(BB10);.+Version/(\d+)\.(\d+)\.(\d+)' + os_replacement: 'BlackBerry OS' + - regex: '(Black[Bb]erry)[0-9a-z]+/(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'BlackBerry OS' + - regex: '(Black[Bb]erry).+Version/(\d+)\.(\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'BlackBerry OS' + - regex: '(RIM Tablet OS) (\d+)\.(\d+)\.(\d+)' + os_replacement: 'BlackBerry Tablet OS' + - regex: '(Play[Bb]ook)' + os_replacement: 'BlackBerry Tablet OS' + - regex: '(Black[Bb]erry)' + os_replacement: 'BlackBerry OS' + + ########## + # Firefox OS + ########## + - regex: '\((?:Mobile|Tablet);.+Gecko/18.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '0' + os_v3_replacement: '1' + + - regex: '\((?:Mobile|Tablet);.+Gecko/18.1 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '1' + + - regex: '\((?:Mobile|Tablet);.+Gecko/26.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '2' + + - regex: '\((?:Mobile|Tablet);.+Gecko/28.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '3' + + - regex: '\((?:Mobile|Tablet);.+Gecko/30.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '1' + os_v2_replacement: '4' + + - regex: '\((?:Mobile|Tablet);.+Gecko/32.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '2' + os_v2_replacement: '0' + + - regex: '\((?:Mobile|Tablet);.+Gecko/34.0 Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + os_v1_replacement: '2' + os_v2_replacement: '1' + + # Firefox OS Generic + - regex: '\((?:Mobile|Tablet);.+Firefox/\d+\.\d+' + os_replacement: 'Firefox OS' + + + ########## + # BREW + # yes, Brew is lower-cased for Brew MP + ########## + - regex: '(BREW)[ /](\d+)\.(\d+)\.(\d+)' + - regex: '(BREW);' + - regex: '(Brew MP|BMP)[ /](\d+)\.(\d+)\.(\d+)' + os_replacement: 'Brew MP' + - regex: 'BMP;' + os_replacement: 'Brew MP' + + ########## + # Google TV + ########## + - regex: '(GoogleTV)(?: (\d+)\.(\d+)(?:\.(\d+))?|/[\da-z]+)' + + - regex: '(WebTV)/(\d+).(\d+)' + + ########## + # Misc mobile + ########## + - regex: '(hpw|web)OS/(\d+)\.(\d+)(?:\.(\d+))?' + os_replacement: 'webOS' + - regex: '(VRE);' + + ########## + # Generic patterns + # since the majority of os cases are very specific, these go last + ########## + - regex: '(Fedora|Red Hat|PCLinuxOS|Puppy|Ubuntu|Kindle|Bada|Lubuntu|BackTrack|Slackware|(?:Free|Open|Net|\b)BSD)[/ ](\d+)\.(\d+)(?:\.(\d+)(?:\.(\d+))?)?' + + # Gentoo Linux + Kernel Version + - regex: '(Linux)[ /](\d+)\.(\d+)(?:\.(\d+))?.*gentoo' + os_replacement: 'Gentoo' + + # Opera Mini Bada + - regex: '\((Bada);' + + # just os + - regex: '(Windows|Android|WeTab|Maemo)' + - regex: '(Ubuntu|Kubuntu|Arch Linux|CentOS|Slackware|Gentoo|openSUSE|SUSE|Red Hat|Fedora|PCLinuxOS|Mageia|(?:Free|Open|Net|\b)BSD)' + # Linux + Kernel Version + - regex: '(Linux)(?:[ /](\d+)\.(\d+)(?:\.(\d+))?)?' + - regex: 'SunOS' + os_replacement: 'Solaris' + + # Roku Digital-Video-Players https://www.roku.com/ + - regex: '^(Roku)/DVP-(\d+)\.(\d+)' + +device_parsers: + + ######### + # Mobile Spiders + # Catch the mobile crawler before checking for iPhones / Androids. + ######### + - regex: '(?:(?:iPhone|Windows CE|Android).*(?:(?:Bot|Yeti)-Mobile|YRSpider|bots?/\d|(?:bot|spider)\.html)|AdsBot-Google-Mobile.*iPhone)' + regex_flag: 'i' + device_replacement: 'Spider' + brand_replacement: 'Spider' + model_replacement: 'Smartphone' + - regex: '(?:DoCoMo|\bMOT\b|\bLG\b|Nokia|Samsung|SonyEricsson).*(?:(?:Bot|Yeti)-Mobile|bots?/\d|(?:bot|crawler)\.html|(?:jump|google|Wukong)bot|ichiro/mobile|/spider|YahooSeeker)' + regex_flag: 'i' + device_replacement: 'Spider' + brand_replacement: 'Spider' + model_replacement: 'Feature Phone' + + ######### + # WebBrowser for SmartWatch + # @ref: https://play.google.com/store/apps/details?id=se.vaggan.webbrowser&hl=en + ######### + - regex: '\bSmartWatch *\( *([^;]+) *; *([^;]+) *;' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ###################################################################### + # Android parsers + # + # @ref: https://support.google.com/googleplay/answer/1727131?hl=en + ###################################################################### + + # Android Application + - regex: 'Android Application[^\-]+ - (Sony) ?(Ericsson)? (.+) \w+ - ' + device_replacement: '$1 $2' + brand_replacement: '$1$2' + model_replacement: '$3' + - regex: 'Android Application[^\-]+ - (?:HTC|HUAWEI|LGE|LENOVO|MEDION|TCT) (HTC|HUAWEI|LG|LENOVO|MEDION|ALCATEL)[ _\-](.+) \w+ - ' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + - regex: 'Android Application[^\-]+ - ([^ ]+) (.+) \w+ - ' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # 3Q + # @ref: http://www.3q-int.com/ + ######### + - regex: '; *([BLRQ]C\d{4}[A-Z]+) +Build/' + device_replacement: '3Q $1' + brand_replacement: '3Q' + model_replacement: '$1' + - regex: '; *(?:3Q_)([^;/]+) +Build' + device_replacement: '3Q $1' + brand_replacement: '3Q' + model_replacement: '$1' + + ######### + # Acer + # @ref: http://us.acer.com/ac/en/US/content/group/tablets + ######### + - regex: 'Android [34].*; *(A100|A101|A110|A200|A210|A211|A500|A501|A510|A511|A700(?: Lite| 3G)?|A701|B1-A71|A1-\d{3}|B1-\d{3}|V360|V370|W500|W500P|W501|W501P|W510|W511|W700|Slider SL101|DA22[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Acer' + model_replacement: '$1' + - regex: '; *Acer Iconia Tab ([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Acer' + model_replacement: '$1' + - regex: '; *(Z1[1235]0|E320[^/]*|S500|S510|Liquid[^;/]*|Iconia A\d+) Build' + device_replacement: '$1' + brand_replacement: 'Acer' + model_replacement: '$1' + - regex: '; *(Acer |ACER )([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Acer' + model_replacement: '$2' + + ######### + # Advent + # @ref: https://en.wikipedia.org/wiki/Advent_Vega + # @note: VegaBean and VegaComb (names derived from jellybean, honeycomb) are + # custom ROM builds for Vega + ######### + - regex: '; *(Advent )?(Vega(?:Bean|Comb)?).* Build' + device_replacement: '$1$2' + brand_replacement: 'Advent' + model_replacement: '$2' + + ######### + # Ainol + # @ref: http://www.ainol.com/plugin.php?identifier=ainol&module=product + ######### + - regex: '; *(Ainol )?((?:NOVO|[Nn]ovo)[^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Ainol' + model_replacement: '$2' + + ######### + # Airis + # @ref: http://airis.es/Tienda/Default.aspx?idG=001 + ######### + - regex: '; *AIRIS[ _\-]?([^/;\)]+) *(?:;|\)|Build)' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Airis' + model_replacement: '$1' + - regex: '; *(OnePAD[^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Airis' + model_replacement: '$1' + + ######### + # Airpad + # @ref: ?? + ######### + - regex: '; *Airpad[ \-]([^;/]+) Build' + device_replacement: 'Airpad $1' + brand_replacement: 'Airpad' + model_replacement: '$1' + + ######### + # Alcatel - TCT + # @ref: http://www.alcatelonetouch.com/global-en/products/smartphones.html + ######### + - regex: '; *(one ?touch) (EVO7|T10|T20) Build' + device_replacement: 'Alcatel One Touch $2' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch $2' + - regex: '; *(?:alcatel[ _])?(?:(?:one[ _]?touch[ _])|ot[ \-])([^;/]+);? Build' + regex_flag: 'i' + device_replacement: 'Alcatel One Touch $1' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch $1' + - regex: '; *(TCL)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # operator specific models + - regex: '; *(Vodafone Smart II|Optimus_Madrid) Build' + device_replacement: 'Alcatel $1' + brand_replacement: 'Alcatel' + model_replacement: '$1' + - regex: '; *BASE_Lutea_3 Build' + device_replacement: 'Alcatel One Touch 998' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch 998' + - regex: '; *BASE_Varia Build' + device_replacement: 'Alcatel One Touch 918D' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch 918D' + + ######### + # Allfine + # @ref: http://www.myallfine.com/Products.asp + ######### + - regex: '; *((?:FINE|Fine)\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Allfine' + model_replacement: '$1' + + ######### + # Allview + # @ref: http://www.allview.ro/produse/droseries/lista-tablete-pc/ + ######### + - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)((?:Speed|SPEED).*) Build/' + device_replacement: '$1$2' + brand_replacement: 'Allview' + model_replacement: '$2' + - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)?(AX1_Shine|AX2_Frenzy) Build' + device_replacement: '$1$2' + brand_replacement: 'Allview' + model_replacement: '$2' + - regex: '; *(ALLVIEW[ _]?|Allview[ _]?)([^;/]*) Build' + device_replacement: '$1$2' + brand_replacement: 'Allview' + model_replacement: '$2' + + ######### + # Allwinner + # @ref: http://www.allwinner.com/ + # @models: A31 (13.3"),A20,A10, + ######### + - regex: '; *(A13-MID) Build' + device_replacement: '$1' + brand_replacement: 'Allwinner' + model_replacement: '$1' + - regex: '; *(Allwinner)[ _\-]?([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Allwinner' + model_replacement: '$1' + + ######### + # Amaway + # @ref: http://www.amaway.cn/ + ######### + - regex: '; *(A651|A701B?|A702|A703|A705|A706|A707|A711|A712|A713|A717|A722|A785|A801|A802|A803|A901|A902|A1002|A1003|A1006|A1007|A9701|A9703|Q710|Q80) Build' + device_replacement: '$1' + brand_replacement: 'Amaway' + model_replacement: '$1' + + ######### + # Amoi + # @ref: http://www.amoi.com/en/prd/prd_index.jspx + ######### + - regex: '; *(?:AMOI|Amoi)[ _]([^;/]+) Build' + device_replacement: 'Amoi $1' + brand_replacement: 'Amoi' + model_replacement: '$1' + - regex: '^(?:AMOI|Amoi)[ _]([^;/]+) Linux' + device_replacement: 'Amoi $1' + brand_replacement: 'Amoi' + model_replacement: '$1' + + ######### + # Aoc + # @ref: http://latin.aoc.com/media_tablet + ######### + - regex: '; *(MW(?:0[789]|10)[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Aoc' + model_replacement: '$1' + + ######### + # Aoson + # @ref: http://www.luckystar.com.cn/en/mid.aspx?page=1 + # @ref: http://www.luckystar.com.cn/en/mobiletel.aspx?page=1 + # @note: brand owned by luckystar + ######### + - regex: '; *(G7|M1013|M1015G|M11[CG]?|M-?12[B]?|M15|M19[G]?|M30[ACQ]?|M31[GQ]|M32|M33[GQ]|M36|M37|M38|M701T|M710|M712B|M713|M715G|M716G|M71(?:G|GS|T)?|M72[T]?|M73[T]?|M75[GT]?|M77G|M79T|M7L|M7LN|M81|M810|M81T|M82|M92|M92KS|M92S|M717G|M721|M722G|M723|M725G|M739|M785|M791|M92SK|M93D) Build' + device_replacement: 'Aoson $1' + brand_replacement: 'Aoson' + model_replacement: '$1' + - regex: '; *Aoson ([^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Aoson $1' + brand_replacement: 'Aoson' + model_replacement: '$1' + + ######### + # Apanda + # @ref: http://www.apanda.com.cn/ + ######### + - regex: '; *[Aa]panda[ _\-]([^;/]+) Build' + device_replacement: 'Apanda $1' + brand_replacement: 'Apanda' + model_replacement: '$1' + + ######### + # Archos + # @ref: http://www.archos.com/de/products/tablets.html + # @ref: http://www.archos.com/de/products/smartphones/index.html + ######### + - regex: '; *(?:ARCHOS|Archos) ?(GAMEPAD.*?)(?: Build|[;/\(\)\-])' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: 'ARCHOS; GOGI; ([^;]+);' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: '(?:ARCHOS|Archos)[ _]?(.*?)(?: Build|[;/\(\)\-]|$)' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: '; *(AN(?:7|8|9|10|13)[A-Z0-9]{1,4}) Build' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + - regex: '; *(A28|A32|A43|A70(?:BHT|CHT|HB|S|X)|A101(?:B|C|IT)|A7EB|A7EB-WK|101G9|80G9) Build' + device_replacement: 'Archos $1' + brand_replacement: 'Archos' + model_replacement: '$1' + + ######### + # A-rival + # @ref: http://www.a-rival.de/de/ + ######### + - regex: '; *(PAD-FMD[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Arival' + model_replacement: '$1' + - regex: '; *(BioniQ) ?([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Arival' + model_replacement: '$1 $2' + + ######### + # Arnova + # @ref: http://arnovatech.com/ + ######### + - regex: '; *(AN\d[^;/]+|ARCHM\d+) Build' + device_replacement: 'Arnova $1' + brand_replacement: 'Arnova' + model_replacement: '$1' + - regex: '; *(?:ARNOVA|Arnova) ?([^;/]+) Build' + device_replacement: 'Arnova $1' + brand_replacement: 'Arnova' + model_replacement: '$1' + + ######### + # Assistant + # @ref: http://www.assistant.ua + ######### + - regex: '; *(?:ASSISTANT )?(AP)-?([1789]\d{2}[A-Z]{0,2}|80104) Build' + device_replacement: 'Assistant $1-$2' + brand_replacement: 'Assistant' + model_replacement: '$1-$2' + + ######### + # Asus + # @ref: http://www.asus.com/uk/Tablets_Mobile/ + ######### + - regex: '; *(ME17\d[^;/]*|ME3\d{2}[^;/]+|K00[A-Z]|Nexus 10|Nexus 7(?: 2013)?|PadFone[^;/]*|Transformer[^;/]*|TF\d{3}[^;/]*|eeepc) Build' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + - regex: '; *ASUS[ _]*([^;/]+) Build' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + + ######### + # Garmin-Asus + ######### + - regex: '; *Garmin-Asus ([^;/]+) Build' + device_replacement: 'Garmin-Asus $1' + brand_replacement: 'Garmin-Asus' + model_replacement: '$1' + - regex: '; *(Garminfone) Build' + device_replacement: 'Garmin $1' + brand_replacement: 'Garmin-Asus' + model_replacement: '$1' + + ######### + # Attab + # @ref: http://www.theattab.com/ + ######### + - regex: '; (@TAB-[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Attab' + model_replacement: '$1' + + ######### + # Audiosonic + # @ref: ?? + # @note: Take care with Docomo T-01 Toshiba + ######### + - regex: '; *(T-(?:07|[^0]\d)[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Audiosonic' + model_replacement: '$1' + + ######### + # Axioo + # @ref: http://www.axiooworld.com/ww/index.php + ######### + - regex: '; *(?:Axioo[ _\-]([^;/]+)|(picopad)[ _\-]([^;/]+)) Build' + regex_flag: 'i' + device_replacement: 'Axioo $1$2 $3' + brand_replacement: 'Axioo' + model_replacement: '$1$2 $3' + + ######### + # Azend + # @ref: http://azendcorp.com/index.php/products/portable-electronics + ######### + - regex: '; *(V(?:100|700|800)[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Azend' + model_replacement: '$1' + + ######### + # Bak + # @ref: http://www.bakinternational.com/produtos.php?cat=80 + ######### + - regex: '; *(IBAK\-[^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Bak' + model_replacement: '$1' + + ######### + # Bedove + # @ref: http://www.bedove.com/product.html + # @models: HY6501|HY5001|X12|X21|I5 + ######### + - regex: '; *(HY5001|HY6501|X12|X21|I5) Build' + device_replacement: 'Bedove $1' + brand_replacement: 'Bedove' + model_replacement: '$1' + + ######### + # Benss + # @ref: http://www.benss.net/ + ######### + - regex: '; *(JC-[^;/]*) Build' + device_replacement: 'Benss $1' + brand_replacement: 'Benss' + model_replacement: '$1' + + ######### + # Blackberry + # @ref: http://uk.blackberry.com/ + # @note: Android Apps seams to be used here + ######### + - regex: '; *(BB) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Blackberry' + model_replacement: '$2' + + ######### + # Blackbird + # @ref: http://iblackbird.co.kr + ######### + - regex: '; *(BlackBird)[ _](I8.*) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + - regex: '; *(BlackBird)[ _](.*) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Blaupunkt + # @ref: http://www.blaupunkt.com + ######### + # Endeavour + - regex: '; *([0-9]+BP[EM][^;/]*|Endeavour[^;/]+) Build' + device_replacement: 'Blaupunkt $1' + brand_replacement: 'Blaupunkt' + model_replacement: '$1' + + ######### + # Blu + # @ref: http://bluproducts.com + ######### + - regex: '; *((?:BLU|Blu)[ _\-])([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Blu' + model_replacement: '$2' + # BMOBILE = operator branded device + - regex: '; *(?:BMOBILE )?(Blu|BLU|DASH [^;/]+|VIVO 4\.3|TANK 4\.5) Build' + device_replacement: '$1' + brand_replacement: 'Blu' + model_replacement: '$1' + + ######### + # Blusens + # @ref: http://www.blusens.com/es/?sg=1&sv=al&roc=1 + ######### + # tablet + - regex: '; *(TOUCH\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Blusens' + model_replacement: '$1' + + ######### + # Bmobile + # @ref: http://bmobile.eu.com/?categoria=smartphones-2 + # @note: Might collide with Maxx as AX is used also there. + ######### + # smartphone + - regex: '; *(AX5\d+) Build' + device_replacement: '$1' + brand_replacement: 'Bmobile' + model_replacement: '$1' + + ######### + # bq + # @ref: http://bqreaders.com + ######### + - regex: '; *([Bb]q) ([^;/]+);? Build' + device_replacement: '$1 $2' + brand_replacement: 'bq' + model_replacement: '$2' + - regex: '; *(Maxwell [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'bq' + model_replacement: '$1' + + ######### + # Braun Phototechnik + # @ref: http://www.braun-phototechnik.de/en/products/list/~pcat.250/Tablet-PC.html + ######### + - regex: '; *((?:B-Tab|B-TAB) ?\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Braun' + model_replacement: '$1' + + ######### + # Broncho + # @ref: http://www.broncho.cn/ + ######### + - regex: '; *(Broncho) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Captiva + # @ref: http://www.captiva-power.de + ######### + - regex: '; *CAPTIVA ([^;/]+) Build' + device_replacement: 'Captiva $1' + brand_replacement: 'Captiva' + model_replacement: '$1' + + ######### + # Casio + # @ref: http://www.casiogzone.com/ + ######### + - regex: '; *(C771|CAL21|IS11CA) Build' + device_replacement: '$1' + brand_replacement: 'Casio' + model_replacement: '$1' + + ######### + # Cat + # @ref: http://www.cat-sound.com + ######### + - regex: '; *(?:Cat|CAT) ([^;/]+) Build' + device_replacement: 'Cat $1' + brand_replacement: 'Cat' + model_replacement: '$1' + - regex: '; *(?:Cat)(Nova.*) Build' + device_replacement: 'Cat $1' + brand_replacement: 'Cat' + model_replacement: '$1' + - regex: '; *(INM8002KP|ADM8000KP_[AB]) Build' + device_replacement: '$1' + brand_replacement: 'Cat' + model_replacement: 'Tablet PHOENIX 8.1J0' + + ######### + # Celkon + # @ref: http://www.celkonmobiles.com/?_a=products + # @models: A10, A19Q, A101, A105, A107, A107\+, A112, A118, A119, A119Q, A15, A19, A20, A200, A220, A225, A22 Race, A27, A58, A59, A60, A62, A63, A64, A66, A67, A69, A75, A77, A79, A8\+, A83, A85, A86, A87, A89 Ultima, A9\+, A90, A900, A95, A97i, A98, AR 40, AR 45, AR 50, ML5 + ######### + - regex: '; *(?:[Cc]elkon[ _\*]|CELKON[ _\*])([^;/\)]+) ?(?:Build|;|\))' + device_replacement: '$1' + brand_replacement: 'Celkon' + model_replacement: '$1' + - regex: 'Build/(?:[Cc]elkon)+_?([^;/_\)]+)' + device_replacement: '$1' + brand_replacement: 'Celkon' + model_replacement: '$1' + - regex: '; *(CT)-?(\d+) Build' + device_replacement: '$1$2' + brand_replacement: 'Celkon' + model_replacement: '$1$2' + # smartphones + - regex: '; *(A19|A19Q|A105|A107[^;/\)]*) ?(?:Build|;|\))' + device_replacement: '$1' + brand_replacement: 'Celkon' + model_replacement: '$1' + + ######### + # ChangJia + # @ref: http://www.cjshowroom.com/eproducts.aspx?classcode=004001001 + # @brief: China manufacturer makes tablets for different small brands + # (eg. http://www.zeepad.net/index.html) + ######### + - regex: '; *(TPC[0-9]{4,5}) Build' + device_replacement: '$1' + brand_replacement: 'ChangJia' + model_replacement: '$1' + + ######### + # Cloudfone + # @ref: http://www.cloudfonemobile.com/ + ######### + - regex: '; *(Cloudfone)[ _](Excite)([^ ][^;/]+) Build' + device_replacement: '$1 $2 $3' + brand_replacement: 'Cloudfone' + model_replacement: '$1 $2 $3' + - regex: '; *(Excite|ICE)[ _](\d+[^;/]+) Build' + device_replacement: 'Cloudfone $1 $2' + brand_replacement: 'Cloudfone' + model_replacement: 'Cloudfone $1 $2' + - regex: '; *(Cloudfone|CloudPad)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Cloudfone' + model_replacement: '$1 $2' + + ######### + # Cmx + # @ref: http://cmx.at/de/ + ######### + - regex: '; *((?:Aquila|Clanga|Rapax)[^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Cmx' + model_replacement: '$1' + + ######### + # CobyKyros + # @ref: http://cobykyros.com + # @note: Be careful with MID\d{3} from MpMan or Manta + ######### + - regex: '; *(?:CFW-|Kyros )?(MID[0-9]{4}(?:[ABC]|SR|TV)?)(\(3G\)-4G| GB 8K| 3G| 8K| GB)? *(?:Build|[;\)])' + device_replacement: 'CobyKyros $1$2' + brand_replacement: 'CobyKyros' + model_replacement: '$1$2' + + ######### + # Coolpad + # @ref: ?? + ######### + - regex: '; *([^;/]*)Coolpad[ _]([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Coolpad' + model_replacement: '$1$2' + + ######### + # Cube + # @ref: http://www.cube-tablet.com/buy-products.html + ######### + - regex: '; *(CUBE[ _])?([KU][0-9]+ ?GT.*|A5300) Build' + regex_flag: 'i' + device_replacement: '$1$2' + brand_replacement: 'Cube' + model_replacement: '$2' + + ######### + # Cubot + # @ref: http://www.cubotmall.com/ + ######### + - regex: '; *CUBOT ([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Cubot' + model_replacement: '$1' + - regex: '; *(BOBBY) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Cubot' + model_replacement: '$1' + + ######### + # Danew + # @ref: http://www.danew.com/produits-tablette.php + ######### + - regex: '; *(Dslide [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Danew' + model_replacement: '$1' + + ######### + # Dell + # @ref: http://www.dell.com + # @ref: http://www.softbank.jp/mobile/support/product/101dl/ + # @ref: http://www.softbank.jp/mobile/support/product/001dl/ + # @ref: http://developer.emnet.ne.jp/android.html + # @ref: http://www.dell.com/in/p/mobile-xcd28/pd + # @ref: http://www.dell.com/in/p/mobile-xcd35/pd + ######### + - regex: '; *(XCD)[ _]?(28|35) Build' + device_replacement: 'Dell $1$2' + brand_replacement: 'Dell' + model_replacement: '$1$2' + - regex: '; *(001DL) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak' + - regex: '; *(?:Dell|DELL) (Streak) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak' + - regex: '; *(101DL|GS01|Streak Pro[^;/]*) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak Pro' + - regex: '; *([Ss]treak ?7) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: 'Streak 7' + - regex: '; *(Mini-3iX) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + - regex: '; *(?:Dell|DELL)[ _](Aero|Venue|Thunder|Mini.*|Streak[ _]Pro) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + - regex: '; *Dell[ _]([^;/]+) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + - regex: '; *Dell ([^;/]+) Build' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + + ######### + # Denver + # @ref: http://www.denver-electronics.com/tablets1/ + ######### + - regex: '; *(TA[CD]-\d+[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Denver' + model_replacement: '$1' + + ######### + # Dex + # @ref: http://dex.ua/ + ######### + - regex: '; *(iP[789]\d{2}(?:-3G)?|IP10\d{2}(?:-8GB)?) Build' + device_replacement: '$1' + brand_replacement: 'Dex' + model_replacement: '$1' + + ######### + # DNS AirTab + # @ref: http://www.dns-shop.ru/ + ######### + - regex: '; *(AirTab)[ _\-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'DNS' + model_replacement: '$1 $2' + + ######### + # Docomo (Operator Branded Device) + # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent + ######### + - regex: '; *(F\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: '$1' + - regex: '; *(HT-03A) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Magic' + - regex: '; *(HT\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(L\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '; *(N\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Nec' + model_replacement: '$1' + - regex: '; *(P\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + - regex: '; *(SC\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; *(SH\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(SO\-\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + - regex: '; *(T\-0[12][^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Toshiba' + model_replacement: '$1' + + ######### + # DOOV + # @ref: http://www.doov.com.cn/ + ######### + - regex: '; *(DOOV)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'DOOV' + model_replacement: '$2' + + ######### + # Enot + # @ref: http://www.enot.ua/ + ######### + - regex: '; *(Enot|ENOT)[ -]?([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Enot' + model_replacement: '$2' + + ######### + # Evercoss + # @ref: http://evercoss.com/android/ + ######### + - regex: '; *[^;/]+ Build/(?:CROSS|Cross)+[ _\-]([^\)]+)' + device_replacement: 'CROSS $1' + brand_replacement: 'Evercoss' + model_replacement: 'Cross $1' + - regex: '; *(CROSS|Cross)[ _\-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Evercoss' + model_replacement: 'Cross $2' + + ######### + # Explay + # @ref: http://explay.ru/ + ######### + - regex: '; *Explay[_ ](.+?)(?:[\)]| Build)' + device_replacement: '$1' + brand_replacement: 'Explay' + model_replacement: '$1' + + ######### + # Fly + # @ref: http://www.fly-phone.com/ + ######### + - regex: '; *(IQ.*) Build' + device_replacement: '$1' + brand_replacement: 'Fly' + model_replacement: '$1' + - regex: '; *(Fly|FLY)[ _](IQ[^;]+|F[34]\d+[^;]*);? Build' + device_replacement: '$1 $2' + brand_replacement: 'Fly' + model_replacement: '$2' + + ######### + # Fujitsu + # @ref: http://www.fujitsu.com/global/ + ######### + - regex: '; *(M532|Q572|FJL21) Build/' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: '$1' + + ######### + # Galapad + # @ref: http://www.galapad.net/product.html + ######### + - regex: '; *(G1) Build' + device_replacement: '$1' + brand_replacement: 'Galapad' + model_replacement: '$1' + + ######### + # Geeksphone + # @ref: http://www.geeksphone.com/ + ######### + - regex: '; *(Geeksphone) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Gfive + # @ref: http://www.gfivemobile.com/en + ######### + #- regex: '; *(G\'?FIVE) ([^;/]+) Build' # there is a problem with python yaml parser here + - regex: '; *(G[^F]?FIVE) ([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Gfive' + model_replacement: '$2' + + ######### + # Gionee + # @ref: http://www.gionee.com/ + ######### + - regex: '; *(Gionee)[ _\-]([^;/]+)(?:/[^;/]+)? Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Gionee' + model_replacement: '$2' + - regex: '; *(GN\d+[A-Z]?|INFINITY_PASSION|Ctrl_V1) Build' + device_replacement: 'Gionee $1' + brand_replacement: 'Gionee' + model_replacement: '$1' + - regex: '; *(E3) Build/JOP40D' + device_replacement: 'Gionee $1' + brand_replacement: 'Gionee' + model_replacement: '$1' + + ######### + # GoClever + # @ref: http://www.goclever.com + ######### + - regex: '; *((?:FONE|QUANTUM|INSIGNIA) \d+[^;/]*|PLAYTAB) Build' + device_replacement: 'GoClever $1' + brand_replacement: 'GoClever' + model_replacement: '$1' + - regex: '; *GOCLEVER ([^;/]+) Build' + device_replacement: 'GoClever $1' + brand_replacement: 'GoClever' + model_replacement: '$1' + + ######### + # Google + # @ref: http://www.google.de/glass/start/ + ######### + - regex: '; *(Glass \d+) Build' + device_replacement: '$1' + brand_replacement: 'Google' + model_replacement: '$1' + + ######### + # Gigabyte + # @ref: http://gsmart.gigabytecm.com/en/ + ######### + - regex: '; *(GSmart)[ -]([^/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Gigabyte' + model_replacement: '$1 $2' + + ######### + # Freescale development boards + # @ref: http://www.freescale.com/webapp/sps/site/prod_summary.jsp?code=IMX53QSB + ######### + - regex: '; *(imx5[13]_[^/]+) Build' + device_replacement: 'Freescale $1' + brand_replacement: 'Freescale' + model_replacement: '$1' + + ######### + # Haier + # @ref: http://www.haier.com/ + # @ref: http://www.haier.com/de/produkte/tablet/ + ######### + - regex: '; *Haier[ _\-]([^/]+) Build' + device_replacement: 'Haier $1' + brand_replacement: 'Haier' + model_replacement: '$1' + - regex: '; *(PAD1016) Build' + device_replacement: 'Haipad $1' + brand_replacement: 'Haipad' + model_replacement: '$1' + + ######### + # Haipad + # @ref: http://www.haipad.net/ + # @models: V7P|M7SM7S|M9XM9X|M7XM7X|M9|M8|M7-M|M1002|M7|M701 + ######### + - regex: '; *(M701|M7|M8|M9) Build' + device_replacement: 'Haipad $1' + brand_replacement: 'Haipad' + model_replacement: '$1' + + ######### + # Hannspree + # @ref: http://www.hannspree.eu/ + # @models: SN10T1|SN10T2|SN70T31B|SN70T32W + ######### + - regex: '; *(SN\d+T[^;\)/]*)(?: Build|[;\)])' + device_replacement: 'Hannspree $1' + brand_replacement: 'Hannspree' + model_replacement: '$1' + + ######### + # HCLme + # @ref: http://www.hclmetablet.com/india/ + ######### + - regex: 'Build/HCL ME Tablet ([^;\)]+)[\);]' + device_replacement: 'HCLme $1' + brand_replacement: 'HCLme' + model_replacement: '$1' + - regex: '; *([^;\/]+) Build/HCL' + device_replacement: 'HCLme $1' + brand_replacement: 'HCLme' + model_replacement: '$1' + + ######### + # Hena + # @ref: http://www.henadigital.com/en/product/index.asp?id=6 + ######### + - regex: '; *(MID-?\d{4}C[EM]) Build' + device_replacement: 'Hena $1' + brand_replacement: 'Hena' + model_replacement: '$1' + + ######### + # Hisense + # @ref: http://www.hisense.com/ + ######### + - regex: '; *(EG\d{2,}|HS-[^;/]+|MIRA[^;/]+) Build' + device_replacement: 'Hisense $1' + brand_replacement: 'Hisense' + model_replacement: '$1' + - regex: '; *(andromax[^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Hisense $1' + brand_replacement: 'Hisense' + model_replacement: '$1' + + ######### + # hitech + # @ref: http://www.hitech-mobiles.com/ + ######### + - regex: '; *(?:AMAZE[ _](S\d+)|(S\d+)[ _]AMAZE) Build' + device_replacement: 'AMAZE $1$2' + brand_replacement: 'hitech' + model_replacement: 'AMAZE $1$2' + + ######### + # HP + # @ref: http://www.hp.com/ + ######### + - regex: '; *(PlayBook) Build' + device_replacement: 'HP $1' + brand_replacement: 'HP' + model_replacement: '$1' + - regex: '; *HP ([^/]+) Build' + device_replacement: 'HP $1' + brand_replacement: 'HP' + model_replacement: '$1' + - regex: '; *([^/]+_tenderloin) Build' + device_replacement: 'HP TouchPad' + brand_replacement: 'HP' + model_replacement: 'TouchPad' + + ######### + # Huawei + # @ref: http://www.huaweidevice.com + # @note: Needs to be before HTC due to Desire HD Build on U8815 + ######### + - regex: '; *(HUAWEI |Huawei-)?([UY][^;/]+) Build/(?:Huawei|HUAWEI)([UY][^\);]+)\)' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *([^;/]+) Build[/ ]Huawei(MT1-U06|[A-Z]+\d+[^\);]+)[^\);]*\)' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *(S7|M860) Build' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: '; *((?:HUAWEI|Huawei)[ \-]?)(MediaPad) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *((?:HUAWEI[ _]?|Huawei[ _])?Ascend[ _])([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *((?:HUAWEI|Huawei)[ _\-]?)((?:G700-|MT-)[^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *((?:HUAWEI|Huawei)[ _\-]?)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: '$2' + - regex: '; *(MediaPad[^;]+|SpringBoard) Build/Huawei' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: '; *([^;]+) Build/Huawei' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: '; *([Uu])([89]\d{3}) Build' + device_replacement: '$1$2' + brand_replacement: 'Huawei' + model_replacement: 'U$2' + - regex: '; *(?:Ideos |IDEOS )(S7) Build' + device_replacement: 'Huawei Ideos$1' + brand_replacement: 'Huawei' + model_replacement: 'Ideos$1' + - regex: '; *(?:Ideos |IDEOS )([^;/]+\s*|\s*)Build' + device_replacement: 'Huawei Ideos$1' + brand_replacement: 'Huawei' + model_replacement: 'Ideos$1' + - regex: '; *(Orange Daytona|Pulse|Pulse Mini|Vodafone 858|C8500|C8600|C8650|C8660|Nexus 6P) Build' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + + ######### + # HTC + # @ref: http://www.htc.com/www/products/ + # @ref: http://en.wikipedia.org/wiki/List_of_HTC_phones + ######### + + - regex: '; *HTC[ _]([^;]+); Windows Phone' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + + # Android HTC with Version Number matcher + # ; HTC_0P3Z11/1.12.161.3 Build + # ;HTC_A3335 V2.38.841.1 Build + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+))?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1 $2' + brand_replacement: 'HTC' + model_replacement: '$1 $2' + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+))?)?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1 $2 $3' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3' + - regex: '; *(?:HTC[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+))?)?)?(?:[/\\]1\.0 | V|/| +)\d+\.\d[\d\.]*(?: *Build|\))' + device_replacement: 'HTC $1 $2 $3 $4' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3 $4' + + # Android HTC without Version Number matcher + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/;]+)(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/;\)]+))?(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1 $2' + brand_replacement: 'HTC' + model_replacement: '$1 $2' + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/;\)]+))?)?(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1 $2 $3' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3' + - regex: '; *(?:(?:HTC|htc)(?:_blocked)*[ _/])+([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ _/]+)(?:[ _/]([^ /;]+))?)?)?(?: *Build|[;\)]| - )' + device_replacement: 'HTC $1 $2 $3 $4' + brand_replacement: 'HTC' + model_replacement: '$1 $2 $3 $4' + + # HTC Streaming Player + - regex: 'HTC Streaming Player [^\/]*/[^\/]*/ htc_([^/]+) /' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + # general matcher for anything else + - regex: '(?:[;,] *|^)(?:htccn_chs-)?HTC[ _-]?([^;]+?)(?: *Build|clay|Android|-?Mozilla| Opera| Profile| UNTRUSTED|[;/\(\)]|$)' + regex_flag: 'i' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + # Android matchers without HTC + - regex: '; *(A6277|ADR6200|ADR6300|ADR6350|ADR6400[A-Z]*|ADR6425[A-Z]*|APX515CKT|ARIA|Desire[^_ ]*|Dream|EndeavorU|Eris|Evo|Flyer|HD2|Hero|HERO200|Hero CDMA|HTL21|Incredible|Inspire[A-Z0-9]*|Legend|Liberty|Nexus ?(?:One|HD2)|One|One S C2|One[ _]?(?:S|V|X\+?)\w*|PC36100|PG06100|PG86100|S31HT|Sensation|Wildfire)(?: Build|[/;\(\)])' + regex_flag: 'i' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(ADR6200|ADR6400L|ADR6425LVW|Amaze|DesireS?|EndeavorU|Eris|EVO|Evo\d[A-Z]+|HD2|IncredibleS?|Inspire[A-Z0-9]*|Inspire[A-Z0-9]*|Sensation[A-Z0-9]*|Wildfire)[ _-](.+?)(?:[/;\)]|Build|MIUI|1\.0)' + regex_flag: 'i' + device_replacement: 'HTC $1 $2' + brand_replacement: 'HTC' + model_replacement: '$1 $2' + + ######### + # Hyundai + # @ref: http://www.hyundaitechnologies.com + ######### + - regex: '; *HYUNDAI (T\d[^/]*) Build' + device_replacement: 'Hyundai $1' + brand_replacement: 'Hyundai' + model_replacement: '$1' + - regex: '; *HYUNDAI ([^;/]+) Build' + device_replacement: 'Hyundai $1' + brand_replacement: 'Hyundai' + model_replacement: '$1' + # X900? http://www.amazon.com/Hyundai-X900-Retina-Android-Bluetooth/dp/B00AO07H3O + - regex: '; *(X700|Hold X|MB-6900) Build' + device_replacement: 'Hyundai $1' + brand_replacement: 'Hyundai' + model_replacement: '$1' + + ######### + # iBall + # @ref: http://www.iball.co.in/Category/Mobiles/22 + ######### + - regex: '; *(?:iBall[ _\-])?(Andi)[ _]?(\d[^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'iBall' + model_replacement: '$1 $2' + - regex: '; *(IBall)(?:[ _]([^;/]+)|) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'iBall' + model_replacement: '$2' + + ######### + # IconBIT + # @ref: http://www.iconbit.com/catalog/tablets/ + ######### + - regex: '; *(NT-\d+[^ ;/]*|Net[Tt]AB [^;/]+|Mercury [A-Z]+|iconBIT)(?: S/N:[^;/]+)? Build' + device_replacement: '$1' + brand_replacement: 'IconBIT' + model_replacement: '$1' + + ######### + # IMO + # @ref: http://www.ponselimo.com/ + ######### + - regex: '; *(IMO)[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'IMO' + model_replacement: '$2' + + ######### + # i-mobile + # @ref: http://www.i-mobilephone.com/ + ######### + - regex: '; *i-?mobile[ _]([^/]+) Build/' + regex_flag: 'i' + device_replacement: 'i-mobile $1' + brand_replacement: 'imobile' + model_replacement: '$1' + - regex: '; *(i-(?:style|note)[^/]*) Build/' + regex_flag: 'i' + device_replacement: 'i-mobile $1' + brand_replacement: 'imobile' + model_replacement: '$1' + + ######### + # Impression + # @ref: http://impression.ua/planshetnye-kompyutery + ######### + - regex: '; *(ImPAD) ?(\d+(?:.)*) Build' + device_replacement: '$1 $2' + brand_replacement: 'Impression' + model_replacement: '$1 $2' + + ######### + # Infinix + # @ref: http://www.infinixmobility.com/index.html + ######### + - regex: '; *(Infinix)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Infinix' + model_replacement: '$2' + + ######### + # Informer + # @ref: ?? + ######### + - regex: '; *(Informer)[ \-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'Informer' + model_replacement: '$2' + + ######### + # Intenso + # @ref: http://www.intenso.de + # @models: 7":TAB 714,TAB 724;8":TAB 814,TAB 824;10":TAB 1004 + ######### + - regex: '; *(TAB) ?([78][12]4) Build' + device_replacement: 'Intenso $1' + brand_replacement: 'Intenso' + model_replacement: '$1 $2' + + ######### + # Intex + # @ref: http://intexmobile.in/index.aspx + # @note: Zync also offers a "Cloud Z5" device + ######### + # smartphones + - regex: '; *(?:Intex[ _])?(AQUA|Aqua)([ _\.\-])([^;/]+) *(?:Build|;)' + device_replacement: '$1$2$3' + brand_replacement: 'Intex' + model_replacement: '$1 $3' + # matches "INTEX CLOUD X1" + - regex: '; *(?:INTEX|Intex)(?:[_ ]([^\ _;/]+))(?:[_ ]([^\ _;/]+))? *(?:Build|;)' + device_replacement: '$1 $2' + brand_replacement: 'Intex' + model_replacement: '$1 $2' + # tablets + - regex: '; *([iI]Buddy)[ _]?(Connect)(?:_|\?_| )?([^;/]*) *(?:Build|;)' + device_replacement: '$1 $2 $3' + brand_replacement: 'Intex' + model_replacement: 'iBuddy $2 $3' + - regex: '; *(I-Buddy)[ _]([^;/]+) *(?:Build|;)' + device_replacement: '$1 $2' + brand_replacement: 'Intex' + model_replacement: 'iBuddy $2' + + ######### + # iOCEAN + # @ref: http://www.iocean.cc/ + ######### + - regex: '; *(iOCEAN) ([^/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'iOCEAN' + model_replacement: '$2' + + ######### + # i.onik + # @ref: http://www.i-onik.de/ + ######### + - regex: '; *(TP\d+(?:\.\d+)?\-\d[^;/]+) Build' + device_replacement: 'ionik $1' + brand_replacement: 'ionik' + model_replacement: '$1' + + ######### + # IRU.ru + # @ref: http://www.iru.ru/catalog/soho/planetable/ + ######### + - regex: '; *(M702pro) Build' + device_replacement: '$1' + brand_replacement: 'Iru' + model_replacement: '$1' + + ######### + # Ivio + # @ref: http://www.ivio.com/mobile.php + # @models: DG80,DG20,DE38,DE88,MD70 + ######### + - regex: '; *(DE88Plus|MD70) Build' + device_replacement: '$1' + brand_replacement: 'Ivio' + model_replacement: '$1' + - regex: '; *IVIO[_\-]([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Ivio' + model_replacement: '$1' + + ######### + # Jaytech + # @ref: http://www.jay-tech.de/jaytech/servlet/frontend/ + ######### + - regex: '; *(TPC-\d+|JAY-TECH) Build' + device_replacement: '$1' + brand_replacement: 'Jaytech' + model_replacement: '$1' + + ######### + # Jiayu + # @ref: http://www.ejiayu.com/en/Product.html + ######### + - regex: '; *(JY-[^;/]+|G[234]S?) Build' + device_replacement: '$1' + brand_replacement: 'Jiayu' + model_replacement: '$1' + + ######### + # JXD + # @ref: http://www.jxd.hk/ + ######### + - regex: '; *(JXD)[ _\-]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'JXD' + model_replacement: '$2' + + ######### + # Karbonn + # @ref: http://www.karbonnmobiles.com/products_tablet.php + ######### + - regex: '; *Karbonn[ _]?([^;/]+) *(?:Build|;)' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Karbonn' + model_replacement: '$1' + - regex: '; *([^;]+) Build/Karbonn' + device_replacement: '$1' + brand_replacement: 'Karbonn' + model_replacement: '$1' + - regex: '; *(A11|A39|A37|A34|ST8|ST10|ST7|Smart Tab3|Smart Tab2|Titanium S\d) +Build' + device_replacement: '$1' + brand_replacement: 'Karbonn' + model_replacement: '$1' + + ######### + # KDDI (Operator Branded Device) + # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent + ######### + - regex: '; *(IS01|IS03|IS05|IS\d{2}SH) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(IS04) Build' + device_replacement: '$1' + brand_replacement: 'Regza' + model_replacement: '$1' + - regex: '; *(IS06|IS\d{2}PT) Build' + device_replacement: '$1' + brand_replacement: 'Pantech' + model_replacement: '$1' + - regex: '; *(IS11S) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: 'Xperia Acro' + - regex: '; *(IS11CA) Build' + device_replacement: '$1' + brand_replacement: 'Casio' + model_replacement: 'GzOne $1' + - regex: '; *(IS11LG) Build' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: 'Optimus X' + - regex: '; *(IS11N) Build' + device_replacement: '$1' + brand_replacement: 'Medias' + model_replacement: '$1' + - regex: '; *(IS11PT) Build' + device_replacement: '$1' + brand_replacement: 'Pantech' + model_replacement: 'MIRACH' + - regex: '; *(IS12F) Build' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: 'Arrows ES' + # @ref: https://ja.wikipedia.org/wiki/IS12M + - regex: '; *(IS12M) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: 'XT909' + - regex: '; *(IS12S) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: 'Xperia Acro HD' + - regex: '; *(ISW11F) Build' + device_replacement: '$1' + brand_replacement: 'Fujitsu' + model_replacement: 'Arrowz Z' + - regex: '; *(ISW11HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'EVO' + - regex: '; *(ISW11K) Build' + device_replacement: '$1' + brand_replacement: 'Kyocera' + model_replacement: 'DIGNO' + - regex: '; *(ISW11M) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: 'Photon' + - regex: '; *(ISW11SC) Build' + device_replacement: '$1' + brand_replacement: 'Samsung' + model_replacement: 'GALAXY S II WiMAX' + - regex: '; *(ISW12HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'EVO 3D' + - regex: '; *(ISW13HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'J' + - regex: '; *(ISW?[0-9]{2}[A-Z]{0,2}) Build' + device_replacement: '$1' + brand_replacement: 'KDDI' + model_replacement: '$1' + - regex: '; *(INFOBAR [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'KDDI' + model_replacement: '$1' + + ######### + # Kingcom + # @ref: http://www.e-kingcom.com + ######### + - regex: '; *(JOYPAD|Joypad)[ _]([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Kingcom' + model_replacement: '$1 $2' + + ######### + # Kobo + # @ref: https://en.wikipedia.org/wiki/Kobo_Inc. + # @ref: http://www.kobo.com/devices#tablets + ######### + - regex: '; *(Vox|VOX|Arc|K080) Build/' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Kobo' + model_replacement: '$1' + - regex: '\b(Kobo Touch)\b' + device_replacement: '$1' + brand_replacement: 'Kobo' + model_replacement: '$1' + + ######### + # K-Touch + # @ref: ?? + ######### + - regex: '; *(K-Touch)[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Ktouch' + model_replacement: '$2' + + ######### + # KT Tech + # @ref: http://www.kttech.co.kr + ######### + - regex: '; *((?:EV|KM)-S\d+[A-Z]?) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'KTtech' + model_replacement: '$1' + + ######### + # Kyocera + # @ref: http://www.android.com/devices/?country=all&m=kyocera + ######### + - regex: '; *(Zio|Hydro|Torque|Event|EVENT|Echo|Milano|Rise|URBANO PROGRESSO|WX04K|WX06K|WX10K|KYL21|101K|C5[12]\d{2}) Build/' + device_replacement: '$1' + brand_replacement: 'Kyocera' + model_replacement: '$1' + + ######### + # Lava + # @ref: http://www.lavamobiles.com/ + ######### + - regex: '; *(?:LAVA[ _])?IRIS[ _\-]?([^/;\)]+) *(?:;|\)|Build)' + regex_flag: 'i' + device_replacement: 'Iris $1' + brand_replacement: 'Lava' + model_replacement: 'Iris $1' + - regex: '; *LAVA[ _]([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Lava' + model_replacement: '$1' + + ######### + # Lemon + # @ref: http://www.lemonmobiles.com/products.php?type=1 + ######### + - regex: '; *(?:(Aspire A1)|(?:LEMON|Lemon)[ _]([^;/]+))_? Build' + device_replacement: 'Lemon $1$2' + brand_replacement: 'Lemon' + model_replacement: '$1$2' + + ######### + # Lenco + # @ref: http://www.lenco.com/c/tablets/ + ######### + - regex: '; *(TAB-1012) Build/' + device_replacement: 'Lenco $1' + brand_replacement: 'Lenco' + model_replacement: '$1' + - regex: '; Lenco ([^;/]+) Build/' + device_replacement: 'Lenco $1' + brand_replacement: 'Lenco' + model_replacement: '$1' + + ######### + # Lenovo + # @ref: http://support.lenovo.com/en_GB/downloads/default.page?# + ######### + - regex: '; *(A1_07|A2107A-H|S2005A-H|S1-37AH0) Build' + device_replacement: '$1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + - regex: '; *(Idea[Tp]ab)[ _]([^;/]+);? Build' + device_replacement: 'Lenovo $1 $2' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2' + - regex: '; *(Idea(?:Tab|pad)) ?([^;/]+) Build' + device_replacement: 'Lenovo $1 $2' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2' + - regex: '; *(ThinkPad) ?(Tablet) Build/' + device_replacement: 'Lenovo $1 $2' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2' + - regex: '; *(?:LNV-)?(?:=?[Ll]enovo[ _\-]?|LENOVO[ _])+(.+?)(?:Build|[;/\)])' + device_replacement: 'Lenovo $1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + - regex: '[;,] (?:Vodafone )?(SmartTab) ?(II) ?(\d+) Build/' + device_replacement: 'Lenovo $1 $2 $3' + brand_replacement: 'Lenovo' + model_replacement: '$1 $2 $3' + - regex: '; *(?:Ideapad )?K1 Build/' + device_replacement: 'Lenovo Ideapad K1' + brand_replacement: 'Lenovo' + model_replacement: 'Ideapad K1' + - regex: '; *(3GC101|3GW10[01]|A390) Build/' + device_replacement: '$1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + - regex: '\b(?:Lenovo|LENOVO)+[ _\-]?([^,;:/ ]+)' + device_replacement: 'Lenovo $1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + + ######### + # Lexibook + # @ref: http://www.lexibook.com/fr + ######### + - regex: '; *(MFC\d+)[A-Z]{2}([^;,/]*),? Build' + device_replacement: '$1$2' + brand_replacement: 'Lexibook' + model_replacement: '$1$2' + + ######### + # LG + # @ref: http://www.lg.com/uk/mobile + ######### + - regex: '; *(E[34][0-9]{2}|LS[6-8][0-9]{2}|VS[6-9][0-9]+[^;/]+|Nexus 4|Nexus 5X?|GT540f?|Optimus (?:2X|G|4X HD)|OptimusX4HD) *(?:Build|;)' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '[;:] *(L-\d+[A-Z]|LGL\d+[A-Z]?)(?:/V\d+)? *(?:Build|[;\)])' + device_replacement: '$1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '; *(LG-)([A-Z]{1,2}\d{2,}[^,;/\)\(]*?)(?:Build| V\d+|[,;/\)\(]|$)' + device_replacement: '$1$2' + brand_replacement: 'LG' + model_replacement: '$2' + - regex: '; *(LG[ \-]|LG)([^;/]+)[;/]? Build' + device_replacement: '$1$2' + brand_replacement: 'LG' + model_replacement: '$2' + - regex: '^(LG)-([^;/]+)/ Mozilla/.*; Android' + device_replacement: '$1 $2' + brand_replacement: 'LG' + model_replacement: '$2' + + ######### + # Malata + # @ref: http://www.malata.com/en/products.aspx?classid=680 + ######### + - regex: '; *((?:SMB|smb)[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Malata' + model_replacement: '$1' + - regex: '; *(?:Malata|MALATA) ([^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Malata' + model_replacement: '$1' + + ######### + # Manta + # @ref: http://www.manta.com.pl/en + ######### + - regex: '; *(MS[45][0-9]{3}|MID0[568][NS]?|MID[1-9]|MID[78]0[1-9]|MID970[1-9]|MID100[1-9]) Build/' + device_replacement: '$1' + brand_replacement: 'Manta' + model_replacement: '$1' + + ######### + # Match + # @ref: http://www.match.net.cn/products.asp + ######### + - regex: '; *(M1052|M806|M9000|M9100|M9701|MID100|MID120|MID125|MID130|MID135|MID140|MID701|MID710|MID713|MID727|MID728|MID731|MID732|MID733|MID735|MID736|MID737|MID760|MID800|MID810|MID820|MID830|MID833|MID835|MID860|MID900|MID930|MID933|MID960|MID980) Build/' + device_replacement: '$1' + brand_replacement: 'Match' + model_replacement: '$1' + + ######### + # Maxx + # @ref: http://www.maxxmobile.in/ + # @models: Maxx MSD7-Play, Maxx MX245+ Trance, Maxx AX8 Race, Maxx MSD7 3G- AX50, Maxx Genx Droid 7 - AX40, Maxx AX5 Duo, + # Maxx AX3 Duo, Maxx AX3, Maxx AX8 Note II (Note 2), Maxx AX8 Note I, Maxx AX8, Maxx AX5 Plus, Maxx MSD7 Smarty, + # Maxx AX9Z Race, + # Maxx MT150, Maxx MQ601, Maxx M2020, Maxx Sleek MX463neo, Maxx MX525, Maxx MX192-Tune, Maxx Genx Droid 7 AX353, + # @note: Need more User-Agents!!! + ######### + - regex: '; *(GenxDroid7|MSD7.*|AX\d.*|Tab 701|Tab 722) Build/' + device_replacement: 'Maxx $1' + brand_replacement: 'Maxx' + model_replacement: '$1' + + ######### + # Mediacom + # @ref: http://www.mediacomeurope.it/ + ######### + - regex: '; *(M-PP[^;/]+|PhonePad ?\d{2,}[^;/]+) Build' + device_replacement: 'Mediacom $1' + brand_replacement: 'Mediacom' + model_replacement: '$1' + - regex: '; *(M-MP[^;/]+|SmartPad ?\d{2,}[^;/]+) Build' + device_replacement: 'Mediacom $1' + brand_replacement: 'Mediacom' + model_replacement: '$1' + + ######### + # Medion + # @ref: http://www.medion.com/en/ + ######### + - regex: '; *(?:MD_)?LIFETAB[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Medion Lifetab $1' + brand_replacement: 'Medion' + model_replacement: 'Lifetab $1' + - regex: '; *MEDION ([^;/]+) Build' + device_replacement: 'Medion $1' + brand_replacement: 'Medion' + model_replacement: '$1' + + ######### + # Meizu + # @ref: http://www.meizu.com + ######### + - regex: '; *(M030|M031|M035|M040|M065|m9) Build' + device_replacement: 'Meizu $1' + brand_replacement: 'Meizu' + model_replacement: '$1' + - regex: '; *(?:meizu_|MEIZU )(.+?) *(?:Build|[;\)])' + device_replacement: 'Meizu $1' + brand_replacement: 'Meizu' + model_replacement: '$1' + + ######### + # Micromax + # @ref: http://www.micromaxinfo.com + ######### + - regex: '; *(?:Micromax[ _](A111|A240)|(A111|A240)) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1$2' + brand_replacement: 'Micromax' + model_replacement: '$1$2' + - regex: '; *Micromax[ _](A\d{2,3}[^;/]*) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + # be carefull here with Acer e.g. A500 + - regex: '; *(A\d{2}|A[12]\d{2}|A90S|A110Q) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + - regex: '; *Micromax[ _](P\d{3}[^;/]*) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + - regex: '; *(P\d{3}|P\d{3}\(Funbook\)) Build' + regex_flag: 'i' + device_replacement: 'Micromax $1' + brand_replacement: 'Micromax' + model_replacement: '$1' + + ######### + # Mito + # @ref: http://new.mitomobile.com/ + ######### + - regex: '; *(MITO)[ _\-]?([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Mito' + model_replacement: '$2' + + ######### + # Mobistel + # @ref: http://www.mobistel.com/ + ######### + - regex: '; *(Cynus)[ _](F5|T\d|.+?) *(?:Build|[;/\)])' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Mobistel' + model_replacement: '$1 $2' + + ######### + # Modecom + # @ref: http://www.modecom.eu/tablets/portal/ + ######### + - regex: '; *(MODECOM )?(FreeTab) ?([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1$2 $3' + brand_replacement: 'Modecom' + model_replacement: '$2 $3' + - regex: '; *(MODECOM )([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Modecom' + model_replacement: '$2' + + ######### + # Motorola + # @ref: http://www.motorola.com/us/shop-all-mobile-phones/ + ######### + - regex: '; *(MZ\d{3}\+?|MZ\d{3} 4G|Xoom|XOOM[^;/]*) Build' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: '; *(Milestone )(XT[^;/]*) Build' + device_replacement: 'Motorola $1$2' + brand_replacement: 'Motorola' + model_replacement: '$2' + - regex: '; *(Motoroi ?x|Droid X|DROIDX) Build' + regex_flag: 'i' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: 'DROID X' + - regex: '; *(Droid[^;/]*|DROID[^;/]*|Milestone[^;/]*|Photon|Triumph|Devour|Titanium) Build' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: '; *(A555|A85[34][^;/]*|A95[356]|ME[58]\d{2}\+?|ME600|ME632|ME722|MB\d{3}\+?|MT680|MT710|MT870|MT887|MT917|WX435|WX453|WX44[25]|XT\d{3,4}[A-Z\+]*|CL[iI]Q|CL[iI]Q XT) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: '; *(Motorola MOT-|Motorola[ _\-]|MOT\-?)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Motorola' + model_replacement: '$2' + - regex: '; *(Moto[_ ]?|MOT\-)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Motorola' + model_replacement: '$2' + + ######### + # MpMan + # @ref: http://www.mpmaneurope.com + ######### + - regex: '; *((?:MP[DQ]C|MPG\d{1,4}|MP\d{3,4}|MID(?:(?:10[234]|114|43|7[247]|8[24]|7)C|8[01]1))[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Mpman' + model_replacement: '$1' + + ######### + # MSI + # @ref: http://www.msi.com/product/windpad/ + ######### + - regex: '; *(?:MSI[ _])?(Primo\d+|Enjoy[ _\-][^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'Msi' + model_replacement: '$1' + + ######### + # Multilaser + # http://www.multilaser.com.br/listagem_produtos.php?cat=5 + ######### + - regex: '; *Multilaser[ _]([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Multilaser' + model_replacement: '$1' + + ######### + # MyPhone + # @ref: http://myphone.com.ph/ + ######### + - regex: '; *(My)[_]?(Pad)[ _]([^;/]+) Build' + device_replacement: '$1$2 $3' + brand_replacement: 'MyPhone' + model_replacement: '$1$2 $3' + - regex: '; *(My)\|?(Phone)[ _]([^;/]+) Build' + device_replacement: '$1$2 $3' + brand_replacement: 'MyPhone' + model_replacement: '$3' + - regex: '; *(A\d+)[ _](Duo)? Build' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'MyPhone' + model_replacement: '$1 $2' + + ######### + # Mytab + # @ref: http://www.mytab.eu/en/category/mytab-products/ + ######### + - regex: '; *(myTab[^;/]*) Build' + device_replacement: '$1' + brand_replacement: 'Mytab' + model_replacement: '$1' + + ######### + # Nabi + # @ref: https://www.nabitablet.com + ######### + - regex: '; *(NABI2?-)([^;/]+) Build/' + device_replacement: '$1$2' + brand_replacement: 'Nabi' + model_replacement: '$2' + + ######### + # Nec Medias + # @ref: http://www.n-keitai.com/ + ######### + - regex: '; *(N-\d+[CDE]) Build/' + device_replacement: '$1' + brand_replacement: 'Nec' + model_replacement: '$1' + - regex: '; ?(NEC-)(.*) Build/' + device_replacement: '$1$2' + brand_replacement: 'Nec' + model_replacement: '$2' + - regex: '; *(LT-NA7) Build/' + device_replacement: '$1' + brand_replacement: 'Nec' + model_replacement: 'Lifetouch Note' + + ######### + # Nextbook + # @ref: http://nextbookusa.com + ######### + - regex: '; *(NXM\d+[A-z0-9_]*|Next\d[A-z0-9_ \-]*|NEXT\d[A-z0-9_ \-]*|Nextbook [A-z0-9_ ]*|DATAM803HC|M805)(?: Build|[\);])' + device_replacement: '$1' + brand_replacement: 'Nextbook' + model_replacement: '$1' + + ######### + # Nokia + # @ref: http://www.nokia.com + ######### + - regex: '; *(Nokia)([ _\-]*)([^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1$2$3' + brand_replacement: 'Nokia' + model_replacement: '$3' + + ######### + # Nook + # @ref: + # TODO nook browser/1.0 + ######### + - regex: '; *(Nook ?|Barnes & Noble Nook |BN )([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Nook' + model_replacement: '$2' + - regex: '; *(NOOK )?(BNRV200|BNRV200A|BNTV250|BNTV250A|BNTV400|BNTV600|LogicPD Zoom2) Build' + device_replacement: '$1$2' + brand_replacement: 'Nook' + model_replacement: '$2' + - regex: '; Build/(Nook)' + device_replacement: '$1' + brand_replacement: 'Nook' + model_replacement: 'Tablet' + + ######### + # Olivetti + # @ref: http://www.olivetti.de/EN/Page/t02/view_html?idp=348 + ######### + - regex: '; *(OP110|OliPad[^;/]+) Build' + device_replacement: 'Olivetti $1' + brand_replacement: 'Olivetti' + model_replacement: '$1' + + ######### + # Omega + # @ref: http://omega-technology.eu/en/produkty/346/tablets + # @note: MID tablets might get matched by CobyKyros first + # @models: (T107|MID(?:700[2-5]|7031|7108|7132|750[02]|8001|8500|9001|971[12]) + ######### + - regex: '; *OMEGA[ _\-](MID[^;/]+) Build' + device_replacement: 'Omega $1' + brand_replacement: 'Omega' + model_replacement: '$1' + - regex: '^(MID7500|MID\d+) Mozilla/5\.0 \(iPad;' + device_replacement: 'Omega $1' + brand_replacement: 'Omega' + model_replacement: '$1' + + ######### + # OpenPeak + # @ref: https://support.google.com/googleplay/answer/1727131?hl=en + ######### + - regex: '; *((?:CIUS|cius)[^;/]*) Build' + device_replacement: 'Openpeak $1' + brand_replacement: 'Openpeak' + model_replacement: '$1' + + ######### + # Oppo + # @ref: http://en.oppo.com/products/ + ######### + - regex: '; *(Find ?(?:5|7a)|R8[012]\d{1,2}|T703\d{0,1}|U70\d{1,2}T?|X90\d{1,2}) Build' + device_replacement: 'Oppo $1' + brand_replacement: 'Oppo' + model_replacement: '$1' + - regex: '; *OPPO ?([^;/]+) Build/' + device_replacement: 'Oppo $1' + brand_replacement: 'Oppo' + model_replacement: '$1' + + ######### + # Odys + # @ref: http://odys.de + ######### + - regex: '; *(?:Odys\-|ODYS\-|ODYS )([^;/]+) Build' + device_replacement: 'Odys $1' + brand_replacement: 'Odys' + model_replacement: '$1' + - regex: '; *(SELECT) ?(7) Build' + device_replacement: 'Odys $1 $2' + brand_replacement: 'Odys' + model_replacement: '$1 $2' + - regex: '; *(PEDI)_(PLUS)_(W) Build' + device_replacement: 'Odys $1 $2 $3' + brand_replacement: 'Odys' + model_replacement: '$1 $2 $3' + # Weltbild - Tablet PC 4 = Cat Phoenix = Odys Tablet PC 4? + - regex: '; *(AEON|BRAVIO|FUSION|FUSION2IN1|Genio|EOS10|IEOS[^;/]*|IRON|Loox|LOOX|LOOX Plus|Motion|NOON|NOON_PRO|NEXT|OPOS|PEDI[^;/]*|PRIME[^;/]*|STUDYTAB|TABLO|Tablet-PC-4|UNO_X8|XELIO[^;/]*|Xelio ?\d+ ?[Pp]ro|XENO10|XPRESS PRO) Build' + device_replacement: 'Odys $1' + brand_replacement: 'Odys' + model_replacement: '$1' + + ######### + # Orion + # @ref: http://www.orion.ua/en/products/computer-products/tablet-pcs.html + ######### + - regex: '; *(TP-\d+) Build/' + device_replacement: 'Orion $1' + brand_replacement: 'Orion' + model_replacement: '$1' + + ######### + # PackardBell + # @ref: http://www.packardbell.com/pb/en/AE/content/productgroup/tablets + ######### + - regex: '; *(G100W?) Build/' + device_replacement: 'PackardBell $1' + brand_replacement: 'PackardBell' + model_replacement: '$1' + + ######### + # Panasonic + # @ref: http://panasonic.jp/mobile/ + # @models: T11, T21, T31, P11, P51, Eluga Power, Eluga DL1 + # @models: (tab) Toughpad FZ-A1, Toughpad JT-B1 + ######### + - regex: '; *(Panasonic)[_ ]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # Toughpad + - regex: '; *(FZ-A1B|JT-B1) Build' + device_replacement: 'Panasonic $1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + # Eluga Power + - regex: '; *(dL1|DL1) Build' + device_replacement: 'Panasonic $1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + + ######### + # Pantech + # @href: http://www.pantech.co.kr/en/prod/prodList.do?gbrand=PANTECH + # @href: http://www.pantech.co.kr/en/prod/prodList.do?gbrand=VEGA + # @models: ADR8995, ADR910L, ADR930VW, C790, CDM8992, CDM8999, IS06, IS11PT, P2000, P2020, P2030, P4100, P5000, P6010, P6020, P6030, P7000, P7040, P8000, P8010, P9020, P9050, P9060, P9070, P9090, PT001, PT002, PT003, TXT8040, TXT8045, VEGA PTL21 + ######### + - regex: '; *(SKY[ _])?(IM\-[AT]\d{3}[^;/]+).* Build/' + device_replacement: 'Pantech $1$2' + brand_replacement: 'Pantech' + model_replacement: '$1$2' + - regex: '; *((?:ADR8995|ADR910L|ADR930L|ADR930VW|PTL21|P8000)(?: 4G)?) Build/' + device_replacement: '$1' + brand_replacement: 'Pantech' + model_replacement: '$1' + - regex: '; *Pantech([^;/]+).* Build/' + device_replacement: 'Pantech $1' + brand_replacement: 'Pantech' + model_replacement: '$1' + + ######### + # Papayre + # @ref: http://grammata.es/ + ######### + - regex: '; *(papyre)[ _\-]([^;/]+) Build/' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Papyre' + model_replacement: '$2' + + ######### + # Pearl + # @ref: http://www.pearl.de/c-1540.shtml + ######### + - regex: '; *(?:Touchlet )?(X10\.[^;/]+) Build/' + device_replacement: 'Pearl $1' + brand_replacement: 'Pearl' + model_replacement: '$1' + + ######### + # Phicomm + # @ref: http://www.phicomm.com.cn/ + ######### + - regex: '; PHICOMM (i800) Build/' + device_replacement: 'Phicomm $1' + brand_replacement: 'Phicomm' + model_replacement: '$1' + - regex: '; PHICOMM ([^;/]+) Build/' + device_replacement: 'Phicomm $1' + brand_replacement: 'Phicomm' + model_replacement: '$1' + - regex: '; *(FWS\d{3}[^;/]+) Build/' + device_replacement: 'Phicomm $1' + brand_replacement: 'Phicomm' + model_replacement: '$1' + + ######### + # Philips + # @ref: http://www.support.philips.com/support/catalog/products.jsp?_dyncharset=UTF-8&country=&categoryid=MOBILE_PHONES_SMART_SU_CN_CARE&userLanguage=en&navCount=2&groupId=PC_PRODUCTS_AND_PHONES_GR_CN_CARE&catalogType=&navAction=push&userCountry=cn&title=Smartphones&cateId=MOBILE_PHONES_CA_CN_CARE + # @TODO: Philips Tablets User-Agents missing! + # @ref: http://www.support.philips.com/support/catalog/products.jsp?_dyncharset=UTF-8&country=&categoryid=ENTERTAINMENT_TABLETS_SU_CN_CARE&userLanguage=en&navCount=0&groupId=&catalogType=&navAction=push&userCountry=cn&title=Entertainment+Tablets&cateId=TABLETS_CA_CN_CARE + ######### + # @note: this a best guess according to available philips models. Need more User-Agents + - regex: '; *(D633|D822|D833|T539|T939|V726|W335|W336|W337|W3568|W536|W5510|W626|W632|W6350|W6360|W6500|W732|W736|W737|W7376|W820|W832|W8355|W8500|W8510|W930) Build' + device_replacement: '$1' + brand_replacement: 'Philips' + model_replacement: '$1' + - regex: '; *(?:Philips|PHILIPS)[ _]([^;/]+) Build' + device_replacement: 'Philips $1' + brand_replacement: 'Philips' + model_replacement: '$1' + + ######### + # Pipo + # @ref: http://www.pipo.cn/En/ + ######### + - regex: 'Android 4\..*; *(M[12356789]|U[12368]|S[123])\ ?(pro)? Build' + device_replacement: 'Pipo $1$2' + brand_replacement: 'Pipo' + model_replacement: '$1$2' + + ######### + # Ployer + # @ref: http://en.ployer.cn/ + ######### + - regex: '; *(MOMO[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Ployer' + model_replacement: '$1' + + ######### + # Polaroid/ Acho + # @ref: http://polaroidstore.com/store/start.asp?category_id=382&category_id2=0&order=title&filter1=&filter2=&filter3=&view=all + ######### + - regex: '; *(?:Polaroid[ _])?((?:MIDC\d{3,}|PMID\d{2,}|PTAB\d{3,})[^;/]*)(\/[^;/]*)? Build/' + device_replacement: '$1' + brand_replacement: 'Polaroid' + model_replacement: '$1' + - regex: '; *(?:Polaroid )(Tablet) Build/' + device_replacement: '$1' + brand_replacement: 'Polaroid' + model_replacement: '$1' + + ######### + # Pomp + # @ref: http://pompmobileshop.com/ + ######### + #~ TODO + - regex: '; *(POMP)[ _\-](.+?) *(?:Build|[;/\)])' + device_replacement: '$1 $2' + brand_replacement: 'Pomp' + model_replacement: '$2' + + ######### + # Positivo + # @ref: http://www.positivoinformatica.com.br/www/pessoal/tablet-ypy/ + ######### + - regex: '; *(TB07STA|TB10STA|TB07FTA|TB10FTA) Build/' + device_replacement: '$1' + brand_replacement: 'Positivo' + model_replacement: '$1' + - regex: '; *(?:Positivo )?((?:YPY|Ypy)[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Positivo' + model_replacement: '$1' + + ######### + # POV + # @ref: http://www.pointofview-online.com/default2.php + # @TODO: Smartphone Models MOB-3515, MOB-5045-B missing + ######### + - regex: '; *(MOB-[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'POV' + model_replacement: '$1' + - regex: '; *POV[ _\-]([^;/]+) Build/' + device_replacement: 'POV $1' + brand_replacement: 'POV' + model_replacement: '$1' + - regex: '; *((?:TAB-PLAYTAB|TAB-PROTAB|PROTAB|PlayTabPro|Mobii[ _\-]|TAB-P)[^;/]*) Build/' + device_replacement: 'POV $1' + brand_replacement: 'POV' + model_replacement: '$1' + + ######### + # Prestigio + # @ref: http://www.prestigio.com/catalogue/MultiPhones + # @ref: http://www.prestigio.com/catalogue/MultiPads + ######### + - regex: '; *(?:Prestigio )?((?:PAP|PMP)\d[^;/]+) Build/' + device_replacement: 'Prestigio $1' + brand_replacement: 'Prestigio' + model_replacement: '$1' + + ######### + # Proscan + # @ref: http://www.proscanvideo.com/products-search.asp?itemClass=TABLET&itemnmbr= + ######### + - regex: '; *(PLT[0-9]{4}.*) Build/' + device_replacement: '$1' + brand_replacement: 'Proscan' + model_replacement: '$1' + + ######### + # QMobile + # @ref: http://www.qmobile.com.pk/ + ######### + - regex: '; *(A2|A5|A8|A900)_?(Classic)? Build' + device_replacement: '$1 $2' + brand_replacement: 'Qmobile' + model_replacement: '$1 $2' + - regex: '; *(Q[Mm]obile)_([^_]+)_([^_]+) Build' + device_replacement: 'Qmobile $2 $3' + brand_replacement: 'Qmobile' + model_replacement: '$2 $3' + - regex: '; *(Q\-?[Mm]obile)[_ ](A[^;/]+) Build' + device_replacement: 'Qmobile $2' + brand_replacement: 'Qmobile' + model_replacement: '$2' + + ######### + # Qmobilevn + # @ref: http://qmobile.vn/san-pham.html + ######### + - regex: '; *(Q\-Smart)[ _]([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Qmobilevn' + model_replacement: '$2' + - regex: '; *(Q\-?[Mm]obile)[ _\-](S[^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Qmobilevn' + model_replacement: '$2' + + ######### + # Quanta + # @ref: ? + ######### + - regex: '; *(TA1013) Build' + device_replacement: '$1' + brand_replacement: 'Quanta' + model_replacement: '$1' + + ######### + # Rockchip + # @ref: http://www.rock-chips.com/a/cn/product/index.html + # @note: manufacturer sells chipsets - I assume that these UAs are dev-boards + ######### + - regex: '; *(RK\d+),? Build/' + device_replacement: '$1' + brand_replacement: 'Rockchip' + model_replacement: '$1' + - regex: ' Build/(RK\d+)' + device_replacement: '$1' + brand_replacement: 'Rockchip' + model_replacement: '$1' + + ######### + # Samsung Android Devices + # @ref: http://www.samsung.com/us/mobile/cell-phones/all-products + ######### + - regex: '; *(SAMSUNG |Samsung )?((?:Galaxy (?:Note II|S\d)|GT-I9082|GT-I9205|GT-N7\d{3}|SM-N9005)[^;/]*)\/?[^;/]* Build/' + device_replacement: 'Samsung $1$2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(Google )?(Nexus [Ss](?: 4G)?) Build/' + device_replacement: 'Samsung $1$2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(SAMSUNG |Samsung )([^\/]*)\/[^ ]* Build/' + device_replacement: 'Samsung $2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(Galaxy(?: Ace| Nexus| S ?II+|Nexus S| with MCR 1.2| Mini Plus 4G)?) Build/' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; *(SAMSUNG[ _\-] *)+([^;/]+) Build' + device_replacement: 'Samsung $2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *(SAMSUNG-)?(GT\-[BINPS]\d{4}[^\/]*)(\/[^ ]*) Build' + device_replacement: 'Samsung $1$2$3' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '(?:; *|^)((?:GT\-[BIiNPS]\d{4}|I9\d{2}0[A-Za-z\+]?\b)[^;/\)]*?)(?:Build|Linux|MIUI|[;/\)])' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; (SAMSUNG-)([A-Za-z0-9\-]+).* Build/' + device_replacement: 'Samsung $1$2' + brand_replacement: 'Samsung' + model_replacement: '$2' + - regex: '; *((?:SCH|SGH|SHV|SHW|SPH|SC|SM)\-[A-Za-z0-9 ]+)(/?[^ ]*)? Build' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: ' ((?:SCH)\-[A-Za-z0-9 ]+)(/?[^ ]*)? Build' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '; *(Behold ?(?:2|II)|YP\-G[^;/]+|EK-GC100|SCL21|I9300) Build' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ######### + # Sharp + # @ref: http://www.sharp-phone.com/en/index.html + # @ref: http://www.android.com/devices/?country=all&m=sharp + ######### + - regex: '; *(SH\-?\d\d[^;/]+|SBM\d[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(SHARP[ -])([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'Sharp' + model_replacement: '$2' + + ######### + # Simvalley + # @ref: http://www.simvalley-mobile.de/ + ######### + - regex: '; *(SPX[_\-]\d[^;/]*) Build/' + device_replacement: '$1' + brand_replacement: 'Simvalley' + model_replacement: '$1' + - regex: '; *(SX7\-PEARL\.GmbH) Build/' + device_replacement: '$1' + brand_replacement: 'Simvalley' + model_replacement: '$1' + - regex: '; *(SP[T]?\-\d{2}[^;/]*) Build/' + device_replacement: '$1' + brand_replacement: 'Simvalley' + model_replacement: '$1' + + ######### + # SK Telesys + # @ref: http://www.sk-w.com/phone/phone_list.jsp + # @ref: http://www.android.com/devices/?country=all&m=sk-telesys + ######### + - regex: '; *(SK\-.*) Build/' + device_replacement: '$1' + brand_replacement: 'SKtelesys' + model_replacement: '$1' + + ######### + # Skytex + # @ref: http://skytex.com/android + ######### + - regex: '; *(?:SKYTEX|SX)-([^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Skytex' + model_replacement: '$1' + - regex: '; *(IMAGINE [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Skytex' + model_replacement: '$1' + + ######### + # SmartQ + # @ref: http://en.smartdevices.com.cn/Products/ + # @models: Z8, X7, U7H, U7, T30, T20, Ten3, V5-II, T7-3G, SmartQ5, K7, S7, Q8, T19, Ten2, Ten, R10, T7, R7, V5, V7, SmartQ7 + ######### + - regex: '; *(SmartQ) ?([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Smartbitt + # @ref: http://www.smartbitt.com/ + # @missing: SBT Useragents + ######### + - regex: '; *(WF7C|WF10C|SBT[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Smartbitt' + model_replacement: '$1' + + ######### + # Softbank (Operator Branded Devices) + # @ref: http://www.ipentec.com/document/document.aspx?page=android-useragent + ######### + - regex: '; *(SBM(?:003SH|005SH|006SH|007SH|102SH)) Build' + device_replacement: '$1' + brand_replacement: 'Sharp' + model_replacement: '$1' + - regex: '; *(003P|101P|101P11C|102P) Build' + device_replacement: '$1' + brand_replacement: 'Panasonic' + model_replacement: '$1' + - regex: '; *(00\dZ) Build/' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; HTC(X06HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(001HT|X06HT) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: '$1' + - regex: '; *(201M) Build' + device_replacement: '$1' + brand_replacement: 'Motorola' + model_replacement: 'XT902' + + ######### + # Trekstor + # @ref: http://www.trekstor.co.uk/surftabs-en.html + # @note: Must come before SonyEricsson + ######### + - regex: '; *(ST\d{4}.*)Build/ST' + device_replacement: 'Trekstor $1' + brand_replacement: 'Trekstor' + model_replacement: '$1' + - regex: '; *(ST\d{4}.*) Build/' + device_replacement: 'Trekstor $1' + brand_replacement: 'Trekstor' + model_replacement: '$1' + + ######### + # SonyEricsson + # @note: Must come before nokia since they also use symbian + # @ref: http://www.android.com/devices/?country=all&m=sony-ericssons + # @TODO: type! + ######### + # android matchers + - regex: '; *(Sony ?Ericsson ?)([^;/]+) Build' + device_replacement: '$1$2' + brand_replacement: 'SonyEricsson' + model_replacement: '$2' + - regex: '; *((?:SK|ST|E|X|LT|MK|MT|WT)\d{2}[a-z0-9]*(?:-o)?|R800i|U20i) Build' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + # TODO X\d+ is wrong + - regex: '; *(Xperia (?:A8|Arc|Acro|Active|Live with Walkman|Mini|Neo|Play|Pro|Ray|X\d+)[^;/]*) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + + ######### + # Sony + # @ref: http://www.sonymobile.co.jp/index.html + # @ref: http://www.sonymobile.com/global-en/products/phones/ + # @ref: http://www.sony.jp/tablet/ + ######### + - regex: '; Sony (Tablet[^;/]+) Build' + device_replacement: 'Sony $1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; Sony ([^;/]+) Build' + device_replacement: 'Sony $1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(Sony)([A-Za-z0-9\-]+) Build' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + - regex: '; *(Xperia [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(C(?:1[0-9]|2[0-9]|53|55|6[0-9])[0-9]{2}|D[25]\d{3}|D6[56]\d{2}) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(SGP\d{3}|SGPT\d{2}) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + - regex: '; *(NW-Z1000Series) Build' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + + ########## + # Sony PlayStation + # @ref: http://playstation.com + # The Vita spoofs the Kindle + ########## + - regex: 'PLAYSTATION 3' + device_replacement: 'PlayStation 3' + brand_replacement: 'Sony' + model_replacement: 'PlayStation 3' + - regex: '(PlayStation (?:Portable|Vita|\d+))' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1' + + ######### + # Spice + # @ref: http://www.spicemobilephones.co.in/ + ######### + - regex: '; *((?:CSL_Spice|Spice|SPICE|CSL)[ _\-]?)?([Mm][Ii])([ _\-])?(\d{3}[^;/]*) Build/' + device_replacement: '$1$2$3$4' + brand_replacement: 'Spice' + model_replacement: 'Mi$4' + + ######### + # Sprint (Operator Branded Devices) + # @ref: + ######### + - regex: '; *(Sprint )(.+?) *(?:Build|[;/])' + device_replacement: '$1$2' + brand_replacement: 'Sprint' + model_replacement: '$2' + - regex: '\b(Sprint)[: ]([^;,/ ]+)' + device_replacement: '$1$2' + brand_replacement: 'Sprint' + model_replacement: '$2' + + ######### + # Tagi + # @ref: ?? + ######### + - regex: '; *(TAGI[ ]?)(MID) ?([^;/]+) Build/' + device_replacement: '$1$2$3' + brand_replacement: 'Tagi' + model_replacement: '$2$3' + + ######### + # Tecmobile + # @ref: http://www.tecmobile.com/ + ######### + - regex: '; *(Oyster500|Opal 800) Build' + device_replacement: 'Tecmobile $1' + brand_replacement: 'Tecmobile' + model_replacement: '$1' + + ######### + # Tecno + # @ref: www.tecno-mobile.com/‎ + ######### + - regex: '; *(TECNO[ _])([^;/]+) Build/' + device_replacement: '$1$2' + brand_replacement: 'Tecno' + model_replacement: '$2' + + ######### + # Telechips, Techvision evaluation boards + # @ref: + ######### + - regex: '; *Android for (Telechips|Techvision) ([^ ]+) ' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Telstra + # @ref: http://www.telstra.com.au/home-phone/thub-2/ + # @ref: https://support.google.com/googleplay/answer/1727131?hl=en + ######### + - regex: '; *(T-Hub2) Build/' + device_replacement: '$1' + brand_replacement: 'Telstra' + model_replacement: '$1' + + ######### + # Terra + # @ref: http://www.wortmann.de/ + ######### + - regex: '; *(PAD) ?(100[12]) Build/' + device_replacement: 'Terra $1$2' + brand_replacement: 'Terra' + model_replacement: '$1$2' + + ######### + # Texet + # @ref: http://www.texet.ru/tablet/ + ######### + - regex: '; *(T[BM]-\d{3}[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Texet' + model_replacement: '$1' + + ######### + # Thalia + # @ref: http://www.thalia.de/shop/tolino-shine-ereader/show/ + ######### + - regex: '; *(tolino [^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Thalia' + model_replacement: '$1' + - regex: '; *Build/.* (TOLINO_BROWSER)' + device_replacement: '$1' + brand_replacement: 'Thalia' + model_replacement: 'Tolino Shine' + + ######### + # Thl + # @ref: http://en.thl.com.cn/Mobile + # @ref: http://thlmobilestore.com + ######### + - regex: '; *(?:CJ[ -])?(ThL|THL)[ -]([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Thl' + model_replacement: '$2' + - regex: '; *(T100|T200|T5|W100|W200|W8s) Build/' + device_replacement: '$1' + brand_replacement: 'Thl' + model_replacement: '$1' + + ######### + # T-Mobile (Operator Branded Devices) + ######### + # @ref: https://en.wikipedia.org/wiki/HTC_Hero + - regex: '; *(T-Mobile[ _]G2[ _]Touch) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Hero' + # @ref: https://en.wikipedia.org/wiki/HTC_Desire_Z + - regex: '; *(T-Mobile[ _]G2) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Desire Z' + - regex: '; *(T-Mobile myTouch Q) Build' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: 'U8730' + - regex: '; *(T-Mobile myTouch) Build' + device_replacement: '$1' + brand_replacement: 'Huawei' + model_replacement: 'U8680' + - regex: '; *(T-Mobile_Espresso) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Espresso' + - regex: '; *(T-Mobile G1) Build' + device_replacement: '$1' + brand_replacement: 'HTC' + model_replacement: 'Dream' + - regex: '\b(T-Mobile ?)?(myTouch)[ _]?([34]G)[ _]?([^\/]*) (?:Mozilla|Build)' + device_replacement: '$1$2 $3 $4' + brand_replacement: 'HTC' + model_replacement: '$2 $3 $4' + - regex: '\b(T-Mobile)_([^_]+)_(.*) Build' + device_replacement: '$1 $2 $3' + brand_replacement: 'Tmobile' + model_replacement: '$2 $3' + - regex: '\b(T-Mobile)[_ ]?(.*?)Build' + device_replacement: '$1 $2' + brand_replacement: 'Tmobile' + model_replacement: '$2' + + ######### + # Tomtec + # @ref: http://www.tom-tec.eu/pages/tablets.php + ######### + - regex: ' (ATP[0-9]{4}) Build' + device_replacement: '$1' + brand_replacement: 'Tomtec' + model_replacement: '$1' + + ######### + # Tooky + # @ref: http://www.tookymobile.com/ + ######### + - regex: ' *(TOOKY)[ _\-]([^;/]+) ?(?:Build|;)' + regex_flag: 'i' + device_replacement: '$1 $2' + brand_replacement: 'Tooky' + model_replacement: '$2' + + ######### + # Toshiba + # @ref: http://www.toshiba.co.jp/ + # @missing: LT170, Thrive 7, TOSHIBA STB10 + ######### + - regex: '\b(TOSHIBA_AC_AND_AZ|TOSHIBA_FOLIO_AND_A|FOLIO_AND_A)' + device_replacement: '$1' + brand_replacement: 'Toshiba' + model_replacement: 'Folio 100' + - regex: '; *([Ff]olio ?100) Build/' + device_replacement: '$1' + brand_replacement: 'Toshiba' + model_replacement: 'Folio 100' + - regex: '; *(AT[0-9]{2,3}(?:\-A|LE\-A|PE\-A|SE|a)?|AT7-A|AT1S0|Hikari-iFrame/WDPF-[^;/]+|THRiVE|Thrive) Build/' + device_replacement: 'Toshiba $1' + brand_replacement: 'Toshiba' + model_replacement: '$1' + + ######### + # Touchmate + # @ref: http://touchmatepc.com/new/ + ######### + - regex: '; *(TM-MID\d+[^;/]+|TOUCHMATE|MID-750) Build' + device_replacement: '$1' + brand_replacement: 'Touchmate' + model_replacement: '$1' + # @todo: needs verification user-agents missing + - regex: '; *(TM-SM\d+[^;/]+) Build' + device_replacement: '$1' + brand_replacement: 'Touchmate' + model_replacement: '$1' + + ######### + # Treq + # @ref: http://www.treq.co.id/product + ######### + - regex: '; *(A10 [Bb]asic2?) Build/' + device_replacement: '$1' + brand_replacement: 'Treq' + model_replacement: '$1' + - regex: '; *(TREQ[ _\-])([^;/]+) Build' + regex_flag: 'i' + device_replacement: '$1$2' + brand_replacement: 'Treq' + model_replacement: '$2' + + ######### + # Umeox + # @ref: http://umeox.com/ + # @models: A936|A603|X-5|X-3 + ######### + # @todo: guessed markers + - regex: '; *(X-?5|X-?3) Build/' + device_replacement: '$1' + brand_replacement: 'Umeox' + model_replacement: '$1' + # @todo: guessed markers + - regex: '; *(A502\+?|A936|A603|X1|X2) Build/' + device_replacement: '$1' + brand_replacement: 'Umeox' + model_replacement: '$1' + + ######### + # Versus + # @ref: http://versusuk.com/support.html + ######### + - regex: '(TOUCH(?:TAB|PAD).+?) Build/' + regex_flag: 'i' + device_replacement: 'Versus $1' + brand_replacement: 'Versus' + model_replacement: '$1' + + ######### + # Vertu + # @ref: http://www.vertu.com/ + ######### + - regex: '(VERTU) ([^;/]+) Build/' + device_replacement: '$1 $2' + brand_replacement: 'Vertu' + model_replacement: '$2' + + ######### + # Videocon + # @ref: http://www.videoconmobiles.com + ######### + - regex: '; *(Videocon)[ _\-]([^;/]+) *(?:Build|;)' + device_replacement: '$1 $2' + brand_replacement: 'Videocon' + model_replacement: '$2' + - regex: ' (VT\d{2}[A-Za-z]*) Build' + device_replacement: '$1' + brand_replacement: 'Videocon' + model_replacement: '$1' + + ######### + # Viewsonic + # @ref: http://viewsonic.com + ######### + - regex: '; *((?:ViewPad|ViewPhone|VSD)[^;/]+) Build/' + device_replacement: '$1' + brand_replacement: 'Viewsonic' + model_replacement: '$1' + - regex: '; *(ViewSonic-)([^;/]+) Build/' + device_replacement: '$1$2' + brand_replacement: 'Viewsonic' + model_replacement: '$2' + - regex: '; *(GTablet.*) Build/' + device_replacement: '$1' + brand_replacement: 'Viewsonic' + model_replacement: '$1' + + ######### + # vivo + # @ref: http://vivo.cn/ + ######### + - regex: '; *([Vv]ivo)[ _]([^;/]+) Build' + device_replacement: '$1 $2' + brand_replacement: 'vivo' + model_replacement: '$2' + + ######### + # Vodafone (Operator Branded Devices) + # @ref: ?? + ######### + - regex: '(Vodafone) (.*) Build/' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Walton + # @ref: http://www.waltonbd.com/ + ######### + - regex: '; *(?:Walton[ _\-])?(Primo[ _\-][^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Walton $1' + brand_replacement: 'Walton' + model_replacement: '$1' + + ######### + # Wiko + # @ref: http://fr.wikomobile.com/collection.php?s=Smartphones + ######### + - regex: '; *(?:WIKO[ \-])?(CINK\+?|BARRY|BLOOM|DARKFULL|DARKMOON|DARKNIGHT|DARKSIDE|FIZZ|HIGHWAY|IGGY|OZZY|RAINBOW|STAIRWAY|SUBLIM|WAX|CINK [^;/]+) Build/' + regex_flag: 'i' + device_replacement: 'Wiko $1' + brand_replacement: 'Wiko' + model_replacement: '$1' + + ######### + # WellcoM + # @ref: ?? + ######### + - regex: '; *WellcoM-([^;/]+) Build' + device_replacement: 'Wellcom $1' + brand_replacement: 'Wellcom' + model_replacement: '$1' + + ########## + # WeTab + # @ref: http://wetab.mobi/ + ########## + - regex: '(?:(WeTab)-Browser|; (wetab) Build)' + device_replacement: '$1' + brand_replacement: 'WeTab' + model_replacement: 'WeTab' + + ######### + # Wolfgang + # @ref: http://wolfgangmobile.com/ + ######### + - regex: '; *(AT-AS[^;/]+) Build' + device_replacement: 'Wolfgang $1' + brand_replacement: 'Wolfgang' + model_replacement: '$1' + + ######### + # Woxter + # @ref: http://www.woxter.es/es-es/categories/index + ######### + - regex: '; *(?:Woxter|Wxt) ([^;/]+) Build' + device_replacement: 'Woxter $1' + brand_replacement: 'Woxter' + model_replacement: '$1' + + ######### + # Yarvik Zania + # @ref: http://yarvik.com + ######### + - regex: '; *(?:Xenta |Luna )?(TAB[234][0-9]{2}|TAB0[78]-\d{3}|TAB0?9-\d{3}|TAB1[03]-\d{3}|SMP\d{2}-\d{3}) Build/' + device_replacement: 'Yarvik $1' + brand_replacement: 'Yarvik' + model_replacement: '$1' + + ######### + # Yifang + # @note: Needs to be at the very last as manufacturer builds for other brands. + # @ref: http://www.yifangdigital.com/ + # @models: M1010, M1011, M1007, M1008, M1005, M899, M899LP, M909, M8000, + # M8001, M8002, M8003, M849, M815, M816, M819, M805, M878, M780LPW, + # M778, M7000, M7000AD, M7000NBD, M7001, M7002, M7002KBD, M777, M767, + # M789, M799, M769, M757, M755, M753, M752, M739, M729, M723, M712, M727 + ######### + - regex: '; *([A-Z]{2,4})(M\d{3,}[A-Z]{2})([^;\)\/]*)(?: Build|[;\)])' + device_replacement: 'Yifang $1$2$3' + brand_replacement: 'Yifang' + model_replacement: '$2' + + ######### + # XiaoMi + # @ref: http://www.xiaomi.com/event/buyphone + ######### + - regex: '; *((MI|HM|MI-ONE|Redmi)[ -](NOTE |Note )?[^;/]*) (Build|MIUI)/' + device_replacement: 'XiaoMi $1' + brand_replacement: 'XiaoMi' + model_replacement: '$1' + + ######### + # Xolo + # @ref: http://www.xolo.in/ + ######### + - regex: '; *XOLO[ _]([^;/]*tab.*) Build' + regex_flag: 'i' + device_replacement: 'Xolo $1' + brand_replacement: 'Xolo' + model_replacement: '$1' + - regex: '; *XOLO[ _]([^;/]+) Build' + regex_flag: 'i' + device_replacement: 'Xolo $1' + brand_replacement: 'Xolo' + model_replacement: '$1' + - regex: '; *(q\d0{2,3}[a-z]?) Build' + regex_flag: 'i' + device_replacement: 'Xolo $1' + brand_replacement: 'Xolo' + model_replacement: '$1' + + ######### + # Xoro + # @ref: http://www.xoro.de/produkte/ + ######### + - regex: '; *(PAD ?[79]\d+[^;/]*|TelePAD\d+[^;/]) Build' + device_replacement: 'Xoro $1' + brand_replacement: 'Xoro' + model_replacement: '$1' + + ######### + # Zopo + # @ref: http://www.zopomobiles.com/products.html + ######### + - regex: '; *(?:(?:ZOPO|Zopo)[ _]([^;/]+)|(ZP ?(?:\d{2}[^;/]+|C2))|(C[2379])) Build' + device_replacement: '$1$2$3' + brand_replacement: 'Zopo' + model_replacement: '$1$2$3' + + ######### + # ZiiLabs + # @ref: http://www.ziilabs.com/products/platforms/androidreferencetablets.php + ######### + - regex: '; *(ZiiLABS) (Zii[^;/]*) Build' + device_replacement: '$1 $2' + brand_replacement: 'ZiiLabs' + model_replacement: '$2' + - regex: '; *(Zii)_([^;/]*) Build' + device_replacement: '$1 $2' + brand_replacement: 'ZiiLabs' + model_replacement: '$2' + + ######### + # ZTE + # @ref: http://www.ztedevices.com/ + ######### + - regex: '; *(ARIZONA|(?:ATLAS|Atlas) W|D930|Grand (?:[SX][^;]*|Era|Memo[^;]*)|JOE|(?:Kis|KIS)\b[^;]*|Libra|Light [^;]*|N8[056][01]|N850L|N8000|N9[15]\d{2}|N9810|NX501|Optik|(?:Vip )Racer[^;]*|RacerII|RACERII|San Francisco[^;]*|V9[AC]|V55|V881|Z[679][0-9]{2}[A-z]?) Build' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *([A-Z]\d+)_USA_[^;]* Build' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *(SmartTab\d+)[^;]* Build' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *(?:Blade|BLADE|ZTE-BLADE)([^;/]*) Build' + device_replacement: 'ZTE Blade$1' + brand_replacement: 'ZTE' + model_replacement: 'Blade$1' + - regex: '; *(?:Skate|SKATE|ZTE-SKATE)([^;/]*) Build' + device_replacement: 'ZTE Skate$1' + brand_replacement: 'ZTE' + model_replacement: 'Skate$1' + - regex: '; *(Orange |Optimus )(Monte Carlo|San Francisco) Build' + device_replacement: '$1$2' + brand_replacement: 'ZTE' + model_replacement: '$1$2' + - regex: '; *(?:ZXY-ZTE_|ZTE\-U |ZTE[\- _]|ZTE-C[_ ])([^;/]+) Build' + device_replacement: 'ZTE $1' + brand_replacement: 'ZTE' + model_replacement: '$1' + # operator specific + - regex: '; (BASE) (lutea|Lutea 2|Tab[^;]*) Build' + device_replacement: '$1 $2' + brand_replacement: 'ZTE' + model_replacement: '$1 $2' + - regex: '; (Avea inTouch 2|soft stone|tmn smart a7|Movistar[ _]Link) Build' + regex_flag: 'i' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + - regex: '; *(vp9plus)\)' + device_replacement: '$1' + brand_replacement: 'ZTE' + model_replacement: '$1' + + ########## + # Zync + # @ref: http://www.zync.in/index.php/our-products/tablet-phablets + ########## + - regex: '; ?(Cloud[ _]Z5|z1000|Z99 2G|z99|z930|z999|z990|z909|Z919|z900) Build/' + device_replacement: '$1' + brand_replacement: 'Zync' + model_replacement: '$1' + + ########## + # Kindle + # @note: Needs to be after Sony Playstation Vita as this UA contains Silk/3.2 + # @ref: https://developer.amazon.com/sdk/fire/specifications.html + # @ref: http://amazonsilk.wordpress.com/useful-bits/silk-user-agent/ + ########## + - regex: '; ?(KFOT|Kindle Fire) Build\b' + device_replacement: 'Kindle Fire' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire' + - regex: '; ?(KFOTE|Amazon Kindle Fire2) Build\b' + device_replacement: 'Kindle Fire 2' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire 2' + - regex: '; ?(KFTT) Build\b' + device_replacement: 'Kindle Fire HD' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 7"' + - regex: '; ?(KFJWI) Build\b' + device_replacement: 'Kindle Fire HD 8.9" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 8.9" WiFi' + - regex: '; ?(KFJWA) Build\b' + device_replacement: 'Kindle Fire HD 8.9" 4G' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 8.9" 4G' + - regex: '; ?(KFSOWI) Build\b' + device_replacement: 'Kindle Fire HD 7" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HD 7" WiFi' + - regex: '; ?(KFTHWI) Build\b' + device_replacement: 'Kindle Fire HDX 7" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 7" WiFi' + - regex: '; ?(KFTHWA) Build\b' + device_replacement: 'Kindle Fire HDX 7" 4G' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 7" 4G' + - regex: '; ?(KFAPWI) Build\b' + device_replacement: 'Kindle Fire HDX 8.9" WiFi' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 8.9" WiFi' + - regex: '; ?(KFAPWA) Build\b' + device_replacement: 'Kindle Fire HDX 8.9" 4G' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire HDX 8.9" 4G' + - regex: '; ?Amazon ([^;/]+) Build\b' + device_replacement: '$1' + brand_replacement: 'Amazon' + model_replacement: '$1' + - regex: '; ?(Kindle) Build\b' + device_replacement: 'Kindle' + brand_replacement: 'Amazon' + model_replacement: 'Kindle' + - regex: '; ?(Silk)/(\d+)\.(\d+)(?:\.([0-9\-]+))? Build\b' + device_replacement: 'Kindle Fire' + brand_replacement: 'Amazon' + model_replacement: 'Kindle Fire$2' + - regex: ' (Kindle)/(\d+\.\d+)' + device_replacement: 'Kindle' + brand_replacement: 'Amazon' + model_replacement: '$1 $2' + - regex: ' (Silk|Kindle)/(\d+)\.' + device_replacement: 'Kindle' + brand_replacement: 'Amazon' + model_replacement: 'Kindle' + + ######### + # Devices from chinese manufacturer(s) + # @note: identified by x-wap-profile http://218.249.47.94/Xianghe/.* + ######### + - regex: '(sprd)\-([^/]+)/' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # @ref: http://eshinechina.en.alibaba.com/ + - regex: '; *(H\d{2}00\+?) Build' + device_replacement: '$1' + brand_replacement: 'Hero' + model_replacement: '$1' + - regex: '; *(iphone|iPhone5) Build/' + device_replacement: 'Xianghe $1' + brand_replacement: 'Xianghe' + model_replacement: '$1' + - regex: '; *(e\d{4}[a-z]?_?v\d+|v89_[^;/]+)[^;/]+ Build/' + device_replacement: 'Xianghe $1' + brand_replacement: 'Xianghe' + model_replacement: '$1' + + ######### + # Cellular + # @ref: + # @note: Operator branded devices + ######### + - regex: '\bUSCC[_\-]?([^ ;/\)]+)' + device_replacement: '$1' + brand_replacement: 'Cellular' + model_replacement: '$1' + + ###################################################################### + # Windows Phone Parsers + ###################################################################### + + ######### + # Alcatel Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:ALCATEL)[^;]*; *([^;,\)]+)' + device_replacement: 'Alcatel $1' + brand_replacement: 'Alcatel' + model_replacement: '$1' + + ######### + # Asus Windows Phones + ######### + #~ - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:ASUS|Asus)[^;]*; *([^;,\)]+)' + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:ASUS|Asus)[^;]*; *([^;,\)]+)' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + + ######### + # Dell Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:DELL|Dell)[^;]*; *([^;,\)]+)' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + + ######### + # HTC Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:HTC|Htc|HTC_blocked[^;]*)[^;]*; *(?:HTC)?([^;,\)]+)' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + + ######### + # Huawei Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:HUAWEI)[^;]*; *(?:HUAWEI )?([^;,\)]+)' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + + ######### + # LG Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:LG|Lg)[^;]*; *(?:LG[ \-])?([^;,\)]+)' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + + ######### + # Noka Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(?:NOKIA ?|Nokia ?|LUMIA ?|[Ll]umia ?)*(\d{3,}[^;\)]*)' + device_replacement: 'Lumia $1' + brand_replacement: 'Nokia' + model_replacement: 'Lumia $1' + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(RM-\d{3,})' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1' + - regex: '(?:Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)]|WPDesktop;) ?(?:ARM; ?Touch; ?|Touch; ?)?(?:NOKIA|Nokia)[^;]*; *(?:NOKIA ?|Nokia ?|LUMIA ?|[Ll]umia ?)*([^;\)]+)' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1' + + ######### + # Microsoft Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?)?(?:Microsoft(?: Corporation)?)[^;]*; *([^;,\)]+)' + device_replacement: 'Microsoft $1' + brand_replacement: 'Microsoft' + model_replacement: '$1' + + ######### + # Samsung Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:SAMSUNG)[^;]*; *(?:SAMSUNG )?([^;,\.\)]+)' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ######### + # Toshiba Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?(?:TOSHIBA|FujitsuToshibaMobileCommun)[^;]*; *([^;,\)]+)' + device_replacement: 'Toshiba $1' + brand_replacement: 'Toshiba' + model_replacement: '$1' + + ######### + # Generic Windows Phones + ######### + - regex: 'Windows Phone [^;]+; .*?IEMobile/[^;\)]+[;\)] ?(?:ARM; ?Touch; ?|Touch; ?|WpsLondonTest; ?)?([^;]+); *([^;,\)]+)' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ###################################################################### + # Other Devices Parser + ###################################################################### + + ######### + # Samsung Bada Phones + ######### + - regex: '(?:^|; )SAMSUNG\-([A-Za-z0-9\-]+).* Bada/' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ######### + # Firefox OS + ######### + - regex: '\(Mobile; ALCATEL ?(One|ONE) ?(Touch|TOUCH) ?([^;/]+)(?:/[^;]+)?; rv:[^\)]+\) Gecko/[^\/]+ Firefox/' + device_replacement: 'Alcatel $1 $2 $3' + brand_replacement: 'Alcatel' + model_replacement: 'One Touch $3' + - regex: '\(Mobile; (?:ZTE([^;]+)|(OpenC)); rv:[^\)]+\) Gecko/[^\/]+ Firefox/' + device_replacement: 'ZTE $1$2' + brand_replacement: 'ZTE' + model_replacement: '$1$2' + + ########## + # NOKIA + # @note: NokiaN8-00 comes before iphone. Sometimes spoofs iphone + ########## + - regex: 'Nokia(N[0-9]+)([A-z_\-][A-z0-9_\-]*)' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1$2' + - regex: '(?:NOKIA|Nokia)(?:\-| *)(?:([A-Za-z0-9]+)\-[0-9a-f]{32}|([A-Za-z0-9\-]+)(?:UCBrowser)|([A-Za-z0-9\-]+))' + device_replacement: 'Nokia $1$2$3' + brand_replacement: 'Nokia' + model_replacement: '$1$2$3' + - regex: 'Lumia ([A-Za-z0-9\-]+)' + device_replacement: 'Lumia $1' + brand_replacement: 'Nokia' + model_replacement: 'Lumia $1' + # UCWEB Browser on Symbian + - regex: '\(Symbian; U; S60 V5; [A-z]{2}\-[A-z]{2}; (SonyEricsson|Samsung|Nokia|LG)([^;/]+)\)' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # Nokia Symbian + - regex: '\(Symbian(?:/3)?; U; ([^;]+);' + device_replacement: 'Nokia $1' + brand_replacement: 'Nokia' + model_replacement: '$1' + + ########## + # BlackBerry + # @ref: http://www.useragentstring.com/pages/BlackBerry/ + ########## + - regex: 'BB10; ([A-Za-z0-9\- ]+)\)' + device_replacement: 'BlackBerry $1' + brand_replacement: 'BlackBerry' + model_replacement: '$1' + - regex: 'Play[Bb]ook.+RIM Tablet OS' + device_replacement: 'BlackBerry Playbook' + brand_replacement: 'BlackBerry' + model_replacement: 'Playbook' + - regex: 'Black[Bb]erry ([0-9]+);' + device_replacement: 'BlackBerry $1' + brand_replacement: 'BlackBerry' + model_replacement: '$1' + - regex: 'Black[Bb]erry([0-9]+)' + device_replacement: 'BlackBerry $1' + brand_replacement: 'BlackBerry' + model_replacement: '$1' + - regex: 'Black[Bb]erry;' + device_replacement: 'BlackBerry' + brand_replacement: 'BlackBerry' + + ########## + # PALM / HP + # @note: some palm devices must come before iphone. sometimes spoofs iphone in ua + ########## + - regex: '(Pre|Pixi)/\d+\.\d+' + device_replacement: 'Palm $1' + brand_replacement: 'Palm' + model_replacement: '$1' + - regex: 'Palm([0-9]+)' + device_replacement: 'Palm $1' + brand_replacement: 'Palm' + model_replacement: '$1' + - regex: 'Treo([A-Za-z0-9]+)' + device_replacement: 'Palm Treo $1' + brand_replacement: 'Palm' + model_replacement: 'Treo $1' + - regex: 'webOS.*(P160U(?:NA)?)/(\d+).(\d+)' + device_replacement: 'HP Veer' + brand_replacement: 'HP' + model_replacement: 'Veer' + - regex: '(Touch[Pp]ad)/\d+\.\d+' + device_replacement: 'HP TouchPad' + brand_replacement: 'HP' + model_replacement: 'TouchPad' + - regex: 'HPiPAQ([A-Za-z0-9]+)/\d+.\d+' + device_replacement: 'HP iPAQ $1' + brand_replacement: 'HP' + model_replacement: 'iPAQ $1' + - regex: 'PDA; (PalmOS)/sony/model ([a-z]+)/Revision' + device_replacement: '$1' + brand_replacement: 'Sony' + model_replacement: '$1 $2' + + ########## + # AppleTV + # No built in browser that I can tell + # Stack Overflow indicated iTunes-AppleTV/4.1 as a known UA for app available and I'm seeing it in live traffic + ########## + - regex: '(Apple\s?TV)' + device_replacement: 'AppleTV' + brand_replacement: 'Apple' + model_replacement: 'AppleTV' + + ######### + # Tesla Model S + ######### + - regex: '(QtCarBrowser)' + device_replacement: 'Tesla Model S' + brand_replacement: 'Tesla' + model_replacement: 'Model S' + + ########## + # iSTUFF + # @note: complete but probably catches spoofs + # ipad and ipod must be parsed before iphone + # cannot determine specific device type from ua string. (3g, 3gs, 4, etc) + ########## + # @note: on some ua the device can be identified e.g. iPhone5,1 + - regex: '((?:iPhone|iPad|iPod)\d+,\d+)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + # @note: iPad needs to be before iPhone + - regex: '(iPad)(?:;| Simulator;)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + - regex: '(iPod)(?:;| touch;| Simulator;)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + - regex: '(iPhone)(?:;| Simulator;)' + device_replacement: '$1' + brand_replacement: 'Apple' + model_replacement: '$1' + # @note: desktop applications show device info + - regex: 'CFNetwork/.* Darwin/\d.*\(((?:Mac|iMac|PowerMac|PowerBook)[^\d]*)(\d+)(?:,|%2C)(\d+)' + device_replacement: '$1$2,$3' + brand_replacement: 'Apple' + model_replacement: '$1$2,$3' + # @note: iOS applications do not show device info + - regex: 'CFNetwork/.* Darwin/\d' + device_replacement: 'iOS-Device' + brand_replacement: 'Apple' + model_replacement: 'iOS-Device' + + ########## + # Acer + ########## + - regex: 'acer_([A-Za-z0-9]+)_' + device_replacement: 'Acer $1' + brand_replacement: 'Acer' + model_replacement: '$1' + + ########## + # Alcatel + ########## + - regex: '(?:ALCATEL|Alcatel)-([A-Za-z0-9\-]+)' + device_replacement: 'Alcatel $1' + brand_replacement: 'Alcatel' + model_replacement: '$1' + + ########## + # Amoi + ########## + - regex: '(?:Amoi|AMOI)\-([A-Za-z0-9]+)' + device_replacement: 'Amoi $1' + brand_replacement: 'Amoi' + model_replacement: '$1' + + ########## + # Asus + ########## + - regex: '(?:; |\/|^)((?:Transformer (?:Pad|Prime) |Transformer |PadFone[ _]?)[A-Za-z0-9]*)' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + - regex: '(?:asus.*?ASUS|Asus|ASUS|asus)[\- ;]*((?:Transformer (?:Pad|Prime) |Transformer |Padfone |Nexus[ _])?[A-Za-z0-9]+)' + device_replacement: 'Asus $1' + brand_replacement: 'Asus' + model_replacement: '$1' + + + ########## + # Bird + ########## + - regex: '\bBIRD[ \-\.]([A-Za-z0-9]+)' + device_replacement: 'Bird $1' + brand_replacement: 'Bird' + model_replacement: '$1' + + ########## + # Dell + ########## + - regex: '\bDell ([A-Za-z0-9]+)' + device_replacement: 'Dell $1' + brand_replacement: 'Dell' + model_replacement: '$1' + + ########## + # DoCoMo + ########## + - regex: 'DoCoMo/2\.0 ([A-Za-z0-9]+)' + device_replacement: 'DoCoMo $1' + brand_replacement: 'DoCoMo' + model_replacement: '$1' + - regex: '([A-Za-z0-9]+)_W;FOMA' + device_replacement: 'DoCoMo $1' + brand_replacement: 'DoCoMo' + model_replacement: '$1' + - regex: '([A-Za-z0-9]+);FOMA' + device_replacement: 'DoCoMo $1' + brand_replacement: 'DoCoMo' + model_replacement: '$1' + + ########## + # htc + ########## + - regex: '\b(?:HTC/|HTC/[a-z0-9]+/)?HTC[ _\-;]? *(.*?)(?:-?Mozilla|fingerPrint|[;/\(\)]|$)' + device_replacement: 'HTC $1' + brand_replacement: 'HTC' + model_replacement: '$1' + + ########## + # Huawei + ########## + - regex: 'Huawei([A-Za-z0-9]+)' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: 'HUAWEI-([A-Za-z0-9]+)' + device_replacement: 'Huawei $1' + brand_replacement: 'Huawei' + model_replacement: '$1' + - regex: 'vodafone([A-Za-z0-9]+)' + device_replacement: 'Huawei Vodafone $1' + brand_replacement: 'Huawei' + model_replacement: 'Vodafone $1' + + ########## + # i-mate + ########## + - regex: 'i\-mate ([A-Za-z0-9]+)' + device_replacement: 'i-mate $1' + brand_replacement: 'i-mate' + model_replacement: '$1' + + ########## + # kyocera + ########## + - regex: 'Kyocera\-([A-Za-z0-9]+)' + device_replacement: 'Kyocera $1' + brand_replacement: 'Kyocera' + model_replacement: '$1' + - regex: 'KWC\-([A-Za-z0-9]+)' + device_replacement: 'Kyocera $1' + brand_replacement: 'Kyocera' + model_replacement: '$1' + + ########## + # lenovo + ########## + - regex: 'Lenovo[_\-]([A-Za-z0-9]+)' + device_replacement: 'Lenovo $1' + brand_replacement: 'Lenovo' + model_replacement: '$1' + + ########## + # HbbTV (European and Australian standard) + # written before the LG regexes, as LG is making HbbTV too + ########## + - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+ \([^;]*; *(LG)E *; *([^;]*) *;[^;]*;[^;]*;\)' + device_replacement: '$1' + brand_replacement: '$2' + model_replacement: '$3' + - regex: '(HbbTV)/1\.1\.1.*CE-HTML/1\.\d;(Vendor/)*(THOM[^;]*?)[;\s](?:.*SW-Version/.*)*(LF[^;]+);?' + device_replacement: '$1' + brand_replacement: 'Thomson' + model_replacement: '$4' + - regex: '(HbbTV)(?:/1\.1\.1)?(?: ?\(;;;;;\))?; *CE-HTML(?:/1\.\d)?; *([^ ]+) ([^;]+);' + device_replacement: '$1' + brand_replacement: '$2' + model_replacement: '$3' + - regex: '(HbbTV)/1\.1\.1 \(;;;;;\) Maple_2011' + device_replacement: '$1' + brand_replacement: 'Samsung' + - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+ \([^;]*; *(?:CUS:([^;]*)|([^;]+)) *; *([^;]*) *;.*;' + device_replacement: '$1' + brand_replacement: '$2$3' + model_replacement: '$4' + - regex: '(HbbTV)/[0-9]+\.[0-9]+\.[0-9]+' + device_replacement: '$1' + + ########## + # LGE NetCast TV + ########## + - regex: 'LGE; (?:Media\/)?([^;]*);[^;]*;[^;]*;?\); "?LG NetCast(\.TV|\.Media|)-\d+' + device_replacement: 'NetCast$2' + brand_replacement: 'LG' + model_replacement: '$1' + + ########## + # InettvBrowser + ########## + - regex: 'InettvBrowser/[0-9]+\.[0-9A-Z]+ \([^;]*;(Sony)([^;]*);[^;]*;[^\)]*\)' + device_replacement: 'Inettv' + brand_replacement: '$1' + model_replacement: '$2' + - regex: 'InettvBrowser/[0-9]+\.[0-9A-Z]+ \([^;]*;([^;]*);[^;]*;[^\)]*\)' + device_replacement: 'Inettv' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + - regex: '(?:InettvBrowser|TSBNetTV|NETTV|HBBTV)' + device_replacement: 'Inettv' + brand_replacement: 'Generic_Inettv' + + ########## + # lg + ########## + # LG Symbian Phones + - regex: 'Series60/\d\.\d (LG)[\-]?([A-Za-z0-9 \-]+)' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + # other LG phones + - regex: '\b(?:LGE[ \-]LG\-(?:AX)?|LGE |LGE?-LG|LGE?[ \-]|LG[ /\-]|lg[\-])([A-Za-z0-9]+)\b' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '(?:^LG[\-]?|^LGE[\-/]?)([A-Za-z]+[0-9]+[A-Za-z]*)' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + - regex: '^LG([0-9]+[A-Za-z]*)' + device_replacement: 'LG $1' + brand_replacement: 'LG' + model_replacement: '$1' + + ########## + # microsoft + ########## + - regex: '(KIN\.[^ ]+) (\d+)\.(\d+)' + device_replacement: 'Microsoft $1' + brand_replacement: 'Microsoft' + model_replacement: '$1' + - regex: '(?:MSIE|XBMC).*\b(Xbox)\b' + device_replacement: '$1' + brand_replacement: 'Microsoft' + model_replacement: '$1' + - regex: '; ARM; Trident/6\.0; Touch[\);]' + device_replacement: 'Microsoft Surface RT' + brand_replacement: 'Microsoft' + model_replacement: 'Surface RT' + + ########## + # motorola + ########## + - regex: 'Motorola\-([A-Za-z0-9]+)' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: 'MOTO\-([A-Za-z0-9]+)' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + - regex: 'MOT\-([A-z0-9][A-z0-9\-]*)' + device_replacement: 'Motorola $1' + brand_replacement: 'Motorola' + model_replacement: '$1' + + ########## + # nintendo + ########## + - regex: 'Nintendo WiiU' + device_replacement: 'Nintendo Wii U' + brand_replacement: 'Nintendo' + model_replacement: 'Wii U' + - regex: 'Nintendo (DS|3DS|DSi|Wii);' + device_replacement: 'Nintendo $1' + brand_replacement: 'Nintendo' + model_replacement: '$1' + + ########## + # pantech + ########## + - regex: '(?:Pantech|PANTECH)[ _-]?([A-Za-z0-9\-]+)' + device_replacement: 'Pantech $1' + brand_replacement: 'Pantech' + model_replacement: '$1' + + ########## + # philips + ########## + - regex: 'Philips([A-Za-z0-9]+)' + device_replacement: 'Philips $1' + brand_replacement: 'Philips' + model_replacement: '$1' + - regex: 'Philips ([A-Za-z0-9]+)' + device_replacement: 'Philips $1' + brand_replacement: 'Philips' + model_replacement: '$1' + + ########## + # Samsung + ########## + # Samsung Symbian Devices + - regex: 'SymbianOS/9\.\d.* Samsung[/\-]([A-Za-z0-9 \-]+)' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + - regex: '(Samsung)(SGH)(i[0-9]+)' + device_replacement: '$1 $2$3' + brand_replacement: '$1' + model_replacement: '$2-$3' + - regex: 'SAMSUNG-ANDROID-MMS/([^;/]+)' + device_replacement: '$1' + brand_replacement: 'Samsung' + model_replacement: '$1' + # Other Samsung + #- regex: 'SAMSUNG(?:; |-)([A-Za-z0-9\-]+)' + - regex: 'SAMSUNG(?:; |[ -/])([A-Za-z0-9\-]+)' + regex_flag: 'i' + device_replacement: 'Samsung $1' + brand_replacement: 'Samsung' + model_replacement: '$1' + + ########## + # Sega + ########## + - regex: '(Dreamcast)' + device_replacement: 'Sega $1' + brand_replacement: 'Sega' + model_replacement: '$1' + + ########## + # Siemens mobile + ########## + - regex: '^SIE-([A-Za-z0-9]+)' + device_replacement: 'Siemens $1' + brand_replacement: 'Siemens' + model_replacement: '$1' + + ########## + # Softbank + ########## + - regex: 'Softbank/[12]\.0/([A-Za-z0-9]+)' + device_replacement: 'Softbank $1' + brand_replacement: 'Softbank' + model_replacement: '$1' + + ########## + # SonyEricsson + ########## + - regex: 'SonyEricsson ?([A-Za-z0-9\-]+)' + device_replacement: 'Ericsson $1' + brand_replacement: 'SonyEricsson' + model_replacement: '$1' + + ########## + # Sony + ########## + - regex: 'Android [^;]+; ([^ ]+) (Sony)/' + device_replacement: '$2 $1' + brand_replacement: '$2' + model_replacement: '$1' + - regex: '(Sony)(?:BDP\/|\/)?([^ /;\)]+)[ /;\)]' + device_replacement: '$1 $2' + brand_replacement: '$1' + model_replacement: '$2' + + ######### + # Puffin Browser Device detect + # A=Android, I=iOS, P=Phone, T=Tablet + # AT=Android+Tablet + ######### + - regex: 'Puffin/[\d\.]+IT' + device_replacement: 'iPad' + brand_replacement: 'Apple' + model_replacement: 'iPad' + - regex: 'Puffin/[\d\.]+IP' + device_replacement: 'iPhone' + brand_replacement: 'Apple' + model_replacement: 'iPhone' + - regex: 'Puffin/[\d\.]+AT' + device_replacement: 'Generic Tablet' + brand_replacement: 'Generic' + model_replacement: 'Tablet' + - regex: 'Puffin/[\d\.]+AP' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + + ######### + # Android General Device Matching (far from perfect) + ######### + - regex: 'Android[\- ][\d]+\.[\d]+; [A-Za-z]{2}\-[A-Za-z]{0,2}; WOWMobile (.+) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+\.[\d]+\-update1; [A-Za-z]{2}\-[A-Za-z]{0,2} *; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[A-Za-z]{2}[_\-][A-Za-z]{0,2}\-? *; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[A-Za-z]{0,2}\- *; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + # No build info at all - "Build" follows locale immediately + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *[a-z]{0,2}[_\-]?[A-Za-z]{0,2};? Build' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}; *\-?[A-Za-z]{2}; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + - regex: 'Android[\- ][\d]+(?:\.[\d]+){1,2}(?:;.*)?; *(.+?) Build' + brand_replacement: 'Generic_Android' + model_replacement: '$1' + + ########## + # Google TV + ########## + - regex: '(GoogleTV)' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + + ########## + # WebTV + ########## + - regex: '(WebTV)/\d+.\d+' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + # Roku Digital-Video-Players https://www.roku.com/ + - regex: '^(Roku)/DVP-\d+\.\d+' + brand_replacement: 'Generic_Inettv' + model_replacement: '$1' + + ########## + # Generic Tablet + ########## + - regex: '(Android 3\.\d|Opera Tablet|Tablet; .+Firefox/|Android.*(?:Tab|Pad))' + regex_flag: 'i' + device_replacement: 'Generic Tablet' + brand_replacement: 'Generic' + model_replacement: 'Tablet' + + ########## + # Generic Smart Phone + ########## + - regex: '(Symbian|\bS60(Version|V\d)|\bS60\b|\((Series 60|Windows Mobile|Palm OS|Bada); Opera Mini|Windows CE|Opera Mobi|BREW|Brew|Mobile; .+Firefox/|iPhone OS|Android|MobileSafari|Windows *Phone|\(webOS/|PalmOS)' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + - regex: '(hiptop|avantgo|plucker|xiino|blazer|elaine)' + regex_flag: 'i' + device_replacement: 'Generic Smartphone' + brand_replacement: 'Generic' + model_replacement: 'Smartphone' + + ########## + # Spiders (this is hack...) + ########## + - regex: '(bot|zao|borg|DBot|oegp|silk|Xenu|zeal|^NING|CCBot|crawl|htdig|lycos|slurp|teoma|voila|yahoo|Sogou|CiBra|Nutch|^Java/|^JNLP/|Daumoa|Genieo|ichiro|larbin|pompos|Scrapy|snappy|speedy|spider|msnbot|msrbot|vortex|^vortex|crawler|favicon|indexer|Riddler|scooter|scraper|scrubby|WhatWeb|WinHTTP|bingbot|openbot|gigabot|furlbot|polybot|seekbot|^voyager|archiver|Icarus6j|mogimogi|Netvibes|blitzbot|altavista|charlotte|findlinks|Retreiver|TLSProber|WordPress|SeznamBot|ProoXiBot|wsr\-agent|Squrl Java|EtaoSpider|PaperLiBot|SputnikBot|A6\-Indexer|netresearch|searchsight|baiduspider|YisouSpider|ICC\-Crawler|http%20client|Python-urllib|dataparksearch|converacrawler|Screaming Frog|AppEngine-Google|YahooCacheSystem|fast\-webcrawler|Sogou Pic Spider|semanticdiscovery|Innovazion Crawler|facebookexternalhit|Google.*/\+/web/snippet|Google-HTTP-Java-Client|BlogBridge|IlTrovatore-Setaccio|InternetArchive|GomezAgent|WebThumbnail|heritrix|NewsGator|PagePeeker|Reaper|ZooShot|holmes)' + regex_flag: 'i' + device_replacement: 'Spider' + brand_replacement: 'Spider' + model_replacement: 'Desktop' + + ########## + # Generic Feature Phone + # take care to do case insensitive matching + ########## + - regex: '^(1207|3gso|4thp|501i|502i|503i|504i|505i|506i|6310|6590|770s|802s|a wa|acer|acs\-|airn|alav|asus|attw|au\-m|aur |aus |abac|acoo|aiko|alco|alca|amoi|anex|anny|anyw|aptu|arch|argo|bmobile|bell|bird|bw\-n|bw\-u|beck|benq|bilb|blac|c55/|cdm\-|chtm|capi|comp|cond|dall|dbte|dc\-s|dica|ds\-d|ds12|dait|devi|dmob|doco|dopo|dorado|el(?:38|39|48|49|50|55|58|68)|el[3456]\d{2}dual|erk0|esl8|ex300|ez40|ez60|ez70|ezos|ezze|elai|emul|eric|ezwa|fake|fly\-|fly_|g\-mo|g1 u|g560|gf\-5|grun|gene|go.w|good|grad|hcit|hd\-m|hd\-p|hd\-t|hei\-|hp i|hpip|hs\-c|htc |htc\-|htca|htcg)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(htcp|htcs|htct|htc_|haie|hita|huaw|hutc|i\-20|i\-go|i\-ma|i\-mobile|i230|iac|iac\-|iac/|ig01|im1k|inno|iris|jata|kddi|kgt|kgt/|kpt |kwc\-|klon|lexi|lg g|lg\-a|lg\-b|lg\-c|lg\-d|lg\-f|lg\-g|lg\-k|lg\-l|lg\-m|lg\-o|lg\-p|lg\-s|lg\-t|lg\-u|lg\-w|lg/k|lg/l|lg/u|lg50|lg54|lge\-|lge/|leno|m1\-w|m3ga|m50/|maui|mc01|mc21|mcca|medi|meri|mio8|mioa|mo01|mo02|mode|modo|mot |mot\-|mt50|mtp1|mtv |mate|maxo|merc|mits|mobi|motv|mozz|n100|n101|n102|n202|n203|n300|n302|n500|n502|n505|n700|n701|n710|nec\-|nem\-|newg|neon)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(netf|noki|nzph|o2 x|o2\-x|opwv|owg1|opti|oran|ot\-s|p800|pand|pg\-1|pg\-2|pg\-3|pg\-6|pg\-8|pg\-c|pg13|phil|pn\-2|pt\-g|palm|pana|pire|pock|pose|psio|qa\-a|qc\-2|qc\-3|qc\-5|qc\-7|qc07|qc12|qc21|qc32|qc60|qci\-|qwap|qtek|r380|r600|raks|rim9|rove|s55/|sage|sams|sc01|sch\-|scp\-|sdk/|se47|sec\-|sec0|sec1|semc|sgh\-|shar|sie\-|sk\-0|sl45|slid|smb3|smt5|sp01|sph\-|spv |spv\-|sy01|samm|sany|sava|scoo|send|siem|smar|smit|soft|sony|t\-mo|t218|t250|t600|t610|t618|tcl\-|tdg\-|telm|tim\-|ts70|tsm\-|tsm3|tsm5|tx\-9|tagt)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(talk|teli|topl|tosh|up.b|upg1|utst|v400|v750|veri|vk\-v|vk40|vk50|vk52|vk53|vm40|vx98|virg|vertu|vite|voda|vulc|w3c |w3c\-|wapj|wapp|wapu|wapm|wig |wapi|wapr|wapv|wapy|wapa|waps|wapt|winc|winw|wonu|x700|xda2|xdag|yas\-|your|zte\-|zeto|aste|audi|avan|blaz|brew|brvw|bumb|ccwa|cell|cldc|cmd\-|dang|eml2|fetc|hipt|http|ibro|idea|ikom|ipaq|jbro|jemu|jigs|keji|kyoc|kyok|libw|m\-cr|midp|mmef|moto|mwbp|mywa|newt|nok6|o2im|pant|pdxg|play|pluc|port|prox|rozo|sama|seri|smal|symb|treo|upsi|vx52|vx53|vx60|vx61|vx70|vx80|vx81|vx83|vx85|wap\-|webc|whit|wmlb|xda\-|xda_)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '^(Ice)$' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' + - regex: '(wap[\-\ ]browser|maui|netfront|obigo|teleca|up\.browser|midp|Opera Mini)' + regex_flag: 'i' + device_replacement: 'Generic Feature Phone' + brand_replacement: 'Generic' + model_replacement: 'Feature Phone' diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java new file mode 100644 index 00000000000..52a10840387 --- /dev/null +++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; + +public class UserAgentProcessorFactoryTests extends ESTestCase { + + private static Map userAgentParsers; + + private static String regexWithoutDevicesFilename = "regexes_without_devices.yaml"; + private static Path userAgentConfigDir; + + @BeforeClass + public static void createUserAgentParsers() throws IOException { + Path configDir = createTempDir(); + userAgentConfigDir = configDir.resolve("ingest-useragent"); + Files.createDirectories(userAgentConfigDir); + + // Copy file, leaving out the device parsers at the end + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(UserAgentProcessor.class.getResourceAsStream("/regexes.yaml"), StandardCharsets.UTF_8)); + BufferedWriter writer = Files.newBufferedWriter(userAgentConfigDir.resolve(regexWithoutDevicesFilename));) { + String line; + while ((line = reader.readLine()) != null) { + if (line.startsWith("device_parsers:")) { + break; + } + + writer.write(line); + writer.newLine(); + } + } + + userAgentParsers = IngestUserAgentPlugin.createUserAgentParsers(userAgentConfigDir, new UserAgentCache(1000)); + } + + public void testBuildDefaults() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + + String processorTag = randomAsciiOfLength(10); + + UserAgentProcessor processor = factory.create(processorTag, config); + assertThat(processor.getTag(), equalTo(processorTag)); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getTargetField(), equalTo("useragent")); + assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0)); + assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class))); + } + + public void testBuildTargetField() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("target_field", "_target_field"); + + UserAgentProcessor processor = factory.create(null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getTargetField(), equalTo("_target_field")); + } + + public void testBuildRegexFile() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("regex_file", regexWithoutDevicesFilename); + + UserAgentProcessor processor = factory.create(null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); + assertThat(processor.getUaParser().getDevicePatterns().size(), equalTo(0)); + } + + public void testBuildNonExistingRegexFile() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("regex_file", "does-not-exist.yaml"); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); + assertThat(e.getMessage(), equalTo("[regex_file] regex file [does-not-exist.yaml] doesn't exist (has to exist at node startup)")); + } + + public void testBuildFields() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Set properties = EnumSet.noneOf(UserAgentProcessor.Property.class); + List fieldNames = new ArrayList<>(); + int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length); + for (int i = 0; i < numFields; i++) { + UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i]; + properties.add(property); + fieldNames.add(property.name().toLowerCase(Locale.ROOT)); + } + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("properties", fieldNames); + + UserAgentProcessor processor = factory.create(null, config); + assertThat(processor.getField(), equalTo("_field")); + assertThat(processor.getProperties(), equalTo(properties)); + } + + public void testInvalidProperty() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("properties", Collections.singletonList("invalid")); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " + + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); + } + + public void testInvalidPropertiesType() throws Exception { + UserAgentProcessor.Factory factory = new UserAgentProcessor.Factory(userAgentParsers); + + Map config = new HashMap<>(); + config.put("field", "_field"); + config.put("properties", "invalid"); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, config)); + assertThat(e.getMessage(), equalTo("[properties] property isn't a list, but of type [java.lang.String]")); + } +} diff --git a/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java new file mode 100644 index 00000000000..d9b5eed059a --- /dev/null +++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -0,0 +1,161 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.useragent; + +import org.elasticsearch.ingest.RandomDocumentPicks; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.useragent.UserAgentProcessor; +import org.elasticsearch.test.ESTestCase; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.io.InputStream; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Map; + +import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; + +public class UserAgentProcessorTests extends ESTestCase { + + private static UserAgentProcessor processor; + + @BeforeClass + public static void setupProcessor() throws IOException { + InputStream regexStream = UserAgentProcessor.class.getResourceAsStream("/regexes.yaml"); + assertNotNull(regexStream); + + UserAgentParser parser = new UserAgentParser(randomAsciiOfLength(10), regexStream, new UserAgentCache(1000)); + + processor = new UserAgentProcessor(randomAsciiOfLength(10), "source_field", "target_field", parser, + EnumSet.allOf(UserAgentProcessor.Property.class)); + } + + @SuppressWarnings("unchecked") + public void testCommonBrowser() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Chrome")); + assertThat(target.get("major"), is("33")); + assertThat(target.get("minor"), is("0")); + assertThat(target.get("patch"), is("1750")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Mac OS X 10.9.2")); + assertThat(target.get("os_name"), is("Mac OS X")); + assertThat(target.get("os_major"), is("10")); + assertThat(target.get("os_minor"), is("9")); + + assertThat(target.get("device"), is("Other")); + } + + @SuppressWarnings("unchecked") + public void testUncommonDevice() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/525.10+ " + + "(KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Android")); + assertThat(target.get("major"), is("3")); + assertThat(target.get("minor"), is("0")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Android 3.0")); + assertThat(target.get("os_name"), is("Android")); + assertThat(target.get("os_major"), is("3")); + assertThat(target.get("os_minor"), is("0")); + + assertThat(target.get("device"), is("Motorola Xoom")); + } + + @SuppressWarnings("unchecked") + public void testSpider() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("EasouSpider")); + assertNull(target.get("major")); + assertNull(target.get("minor")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Other")); + assertThat(target.get("os_name"), is("Other")); + assertNull(target.get("os_major")); + assertNull(target.get("os_minor")); + + assertThat(target.get("device"), is("Spider")); + } + + @SuppressWarnings("unchecked") + public void testUnknown() throws Exception { + Map document = new HashMap<>(); + document.put("source_field", + "Something I made up v42.0.1"); + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); + + processor.execute(ingestDocument); + Map data = ingestDocument.getSourceAndMetadata(); + + assertThat(data, hasKey("target_field")); + Map target = (Map) data.get("target_field"); + + assertThat(target.get("name"), is("Other")); + assertNull(target.get("major")); + assertNull(target.get("minor")); + assertNull(target.get("patch")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Other")); + assertThat(target.get("os_name"), is("Other")); + assertNull(target.get("os_major")); + assertNull(target.get("os_minor")); + + assertThat(target.get("device"), is("Other")); + } +} + diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java similarity index 51% rename from plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java rename to plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java index bca1c1d8a18..010f85f671a 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/S3BlobStoreContainerTests.java +++ b/plugins/ingest-useragent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentRestIT.java @@ -17,23 +17,24 @@ * under the License. */ -package org.elasticsearch.cloud.aws.blobstore; +package org.elasticsearch.ingest.useragent; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.test.rest.RestTestCandidate; +import org.elasticsearch.test.rest.parser.RestTestParseException; import java.io.IOException; -import java.util.Locale; -public class S3BlobStoreContainerTests extends ESBlobStoreContainerTestCase { - protected BlobStore newBlobStore() throws IOException { - MockAmazonS3 client = new MockAmazonS3(); - String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); +public class UserAgentRestIT extends ESRestTestCase { - return new S3BlobStore(Settings.EMPTY, client, bucket, null, false, - new ByteSizeValue(10, ByteSizeUnit.MB), 5, "public-read-write", "standard"); + public UserAgentRestIT(@Name("yaml") RestTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, RestTestParseException { + return ESRestTestCase.createParameters(0, 1); } } diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml new file mode 100644 index 00000000000..d37ad674cb9 --- /dev/null +++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/10_basic.yaml @@ -0,0 +1,11 @@ +"ingest-useragent plugin installed": + - do: + cluster.state: {} + + - set: {master_node: master} + + - do: + nodes.info: {} + + - match: { nodes.$master.plugins.0.name: ingest-useragent } + - match: { nodes.$master.ingest.processors.0.type: useragent } diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml new file mode 100644 index 00000000000..13451a075d7 --- /dev/null +++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yaml @@ -0,0 +1,86 @@ +--- +"Test user agent processor with defaults": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "useragent" : { + "field" : "field1" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.useragent.name: "Chrome" } + - match: { _source.useragent.os: "Mac OS X 10.9.2" } + - match: { _source.useragent.os_name: "Mac OS X" } + - match: { _source.useragent.os_major: "10" } + - match: { _source.useragent.os_minor: "9" } + - match: { _source.useragent.major: "33" } + - match: { _source.useragent.minor: "0" } + - match: { _source.useragent.patch: "1750" } + - match: { _source.useragent.device: "Other" } + +--- +"Test user agent processor with parameters": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "useragent" : { + "field" : "field1", + "target_field": "field2", + "properties": ["os"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.field2.os: "Mac OS X 10.9.2" } + - is_false: _source.useragent + - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_major + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch + - is_false: _source.field2.device diff --git a/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml new file mode 100644 index 00000000000..3472cd03c3f --- /dev/null +++ b/plugins/ingest-useragent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yaml @@ -0,0 +1,42 @@ +--- +"Test user agent processor with custom regex file": + - do: + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "useragent" : { + "field": "field1", + "regex_file": "test-regexes.yaml" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.useragent.name: "Test" } + - match: { _source.useragent.os: "Other" } + - match: { _source.useragent.os_name: "Other" } + - match: { _source.useragent.device: "Other" } + - is_false: _source.useragent.os_major + - is_false: _source.useragent.os_minor + - is_false: _source.useragent.major + - is_false: _source.useragent.minor + - is_false: _source.useragent.patch diff --git a/plugins/ingest-useragent/test/test-regexes.yaml b/plugins/ingest-useragent/test/test-regexes.yaml new file mode 100644 index 00000000000..e41dec700c0 --- /dev/null +++ b/plugins/ingest-useragent/test/test-regexes.yaml @@ -0,0 +1,3 @@ +user_agent_parsers: + - regex: '.*' + family_replacement: 'Test' \ No newline at end of file diff --git a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java index 75ae79d5f7d..8bcc0ca1fc8 100644 --- a/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java +++ b/plugins/jvm-example/src/main/java/org/elasticsearch/plugin/example/ExampleCatAction.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.plugin.example; -import org.elasticsearch.client.Client; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Table; import org.elasticsearch.common.inject.Inject; @@ -40,7 +39,7 @@ public class ExampleCatAction extends AbstractCatAction { @Inject public ExampleCatAction(Settings settings, RestController controller, ExamplePluginConfiguration config) { - super(settings, controller); + super(settings); this.config = config; controller.registerHandler(GET, "/_cat/configured_example", this); } diff --git a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java index 8c86800e52b..6b80baa7c28 100644 --- a/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java +++ b/plugins/mapper-attachments/src/test/java/org/elasticsearch/mapper/attachments/SimpleAttachmentMapperTests.java @@ -116,7 +116,7 @@ public class SimpleAttachmentMapperTests extends AttachmentUnitTestCase { .endObject() .endObject(); - byte[] mapping = mappingBuilder.bytes().toBytes(); + byte[] mapping = BytesReference.toBytes(mappingBuilder.bytes()); MapperService mapperService = MapperTestUtils.newMapperService(createTempDir(), Settings.EMPTY, getIndicesModuleWithRegisteredAttachmentMapper()); DocumentMapper docMapper = mapperService.parse("mail", new CompressedXContent(mapping), true); // this should not throw an exception diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java index 6117062fc29..e6c3a469076 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java @@ -70,11 +70,6 @@ public class AzureBlobContainer extends AbstractBlobContainer { @Override public InputStream readBlob(String blobName) throws IOException { logger.trace("readBlob({})", blobName); - - if (!blobExists(blobName)) { - throw new IOException("Blob [" + blobName + "] does not exist"); - } - try { return blobStore.getInputStream(blobStore.container(), buildKey(blobName)); } catch (StorageException e) { @@ -121,11 +116,6 @@ public class AzureBlobContainer extends AbstractBlobContainer { @Override public void deleteBlob(String blobName) throws IOException { logger.trace("deleteBlob({})", blobName); - - if (!blobExists(blobName)) { - throw new IOException("Blob [" + blobName + "] does not exist"); - } - try { blobStore.deleteBlob(blobStore.container(), buildKey(blobName)); } catch (URISyntaxException | StorageException e) { diff --git a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java index 4d3459cdcd4..1333c755e7c 100644 --- a/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java +++ b/plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java @@ -65,7 +65,7 @@ public class AzureRepository extends BlobStoreRepository { private static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(64, ByteSizeUnit.MB); - public final static String TYPE = "azure"; + public static final String TYPE = "azure"; public static final class Repository { public static final Setting ACCOUNT_SETTING = Setting.simpleString("account", Property.NodeScope); diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java index 506d574ea62..8160c560325 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceMock.java @@ -95,13 +95,13 @@ public class AzureStorageServiceMock extends AbstractLifecycleComponent blobsBuilder = MapBuilder.newMapBuilder(); for (String blobName : blobs.keySet()) { final String checkBlob; - if (keyPath != null && !keyPath.isEmpty()) { + if (keyPath != null) { // strip off key path from the beginning of the blob name checkBlob = blobName.replace(keyPath, ""); } else { checkBlob = blobName; } - if (prefix == null || startsWithIgnoreCase(checkBlob, prefix)) { + if (startsWithIgnoreCase(checkBlob, prefix)) { blobsBuilder.put(blobName, new PlainBlobMetaData(checkBlob, blobs.get(blobName).size())); } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java index 5959f964644..ad2b14e368b 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageServiceTests.java @@ -32,7 +32,7 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; public class AzureStorageServiceTests extends ESTestCase { - final static Settings settings = Settings.builder() + static final Settings settings = Settings.builder() .put("cloud.azure.storage.azure1.account", "myaccount1") .put("cloud.azure.storage.azure1.key", "mykey1") .put("cloud.azure.storage.azure1.default", true) diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java index db3bd617da4..4b74d2aa0d1 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/cloud/azure/storage/AzureStorageSettingsFilterTests.java @@ -35,7 +35,7 @@ import java.io.IOException; import static org.hamcrest.Matchers.contains; public class AzureStorageSettingsFilterTests extends ESTestCase { - final static Settings settings = Settings.builder() + static final Settings settings = Settings.builder() .put("cloud.azure.storage.azure1.account", "myaccount1") .put("cloud.azure.storage.azure1.key", "mykey1") .put("cloud.azure.storage.azure1.default", true) diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java deleted file mode 100644 index 5b161613c9b..00000000000 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreContainerTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.azure; - -import com.microsoft.azure.storage.StorageException; -import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore; -import org.elasticsearch.cloud.azure.storage.AzureStorageServiceMock; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; -import org.elasticsearch.repositories.RepositoryName; -import org.elasticsearch.repositories.RepositorySettings; - -import java.io.IOException; -import java.net.URISyntaxException; - -public class AzureBlobStoreContainerTests extends ESBlobStoreContainerTestCase { - @Override - protected BlobStore newBlobStore() throws IOException { - try { - RepositoryName repositoryName = new RepositoryName("azure", "ittest"); - RepositorySettings repositorySettings = new RepositorySettings( - Settings.EMPTY, Settings.EMPTY); - AzureStorageServiceMock client = new AzureStorageServiceMock(Settings.EMPTY); - return new AzureBlobStore(repositoryName, Settings.EMPTY, repositorySettings, client); - } catch (URISyntaxException | StorageException e) { - throw new IOException(e); - } - } -} diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index c8b3d9f7e1d..a22178315f7 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -68,16 +68,16 @@ final class HdfsBlobContainer extends AbstractBlobContainer { @Override public void deleteBlob(String blobName) throws IOException { - if (!blobExists(blobName)) { - throw new IOException("Blob [" + blobName + "] does not exist"); + try { + store.execute(new Operation() { + @Override + public Boolean run(FileContext fileContext) throws IOException { + return fileContext.delete(new Path(path, blobName), true); + } + }); + } catch (FileNotFoundException ok) { + // behaves like Files.deleteIfExists } - - store.execute(new Operation() { - @Override - public Boolean run(FileContext fileContext) throws IOException { - return fileContext.delete(new Path(path, blobName), true); - } - }); } @Override @@ -131,7 +131,7 @@ final class HdfsBlobContainer extends AbstractBlobContainer { } @Override - public Map listBlobsByPrefix(final @Nullable String prefix) throws IOException { + public Map listBlobsByPrefix(@Nullable final String prefix) throws IOException { FileStatus[] files = store.execute(new Operation() { @Override public FileStatus[] run(FileContext fileContext) throws IOException { diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java deleted file mode 100644 index a96a8183e58..00000000000 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.hdfs; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.AbstractFileSystem; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; - -import javax.security.auth.Subject; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.AccessController; -import java.security.Principal; -import java.security.PrivilegedAction; -import java.util.Collections; - -public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase { - - @Override - protected BlobStore newBlobStore() throws IOException { - return AccessController.doPrivileged( - new PrivilegedAction() { - @Override - public HdfsBlobStore run() { - try { - FileContext fileContext = createContext(new URI("hdfs:///")); - return new HdfsBlobStore(fileContext, "temp", 1024); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - } - }); - } - - public FileContext createContext(URI uri) { - // mirrors HdfsRepository.java behaviour - Configuration cfg = new Configuration(true); - cfg.setClassLoader(HdfsRepository.class.getClassLoader()); - cfg.reloadConfiguration(); - - Constructor ctor; - Subject subject; - - try { - Class clazz = Class.forName("org.apache.hadoop.security.User"); - ctor = clazz.getConstructor(String.class); - ctor.setAccessible(true); - } catch (ClassNotFoundException | NoSuchMethodException e) { - throw new RuntimeException(e); - } - - try { - Principal principal = (Principal) ctor.newInstance(System.getProperty("user.name")); - subject = new Subject(false, Collections.singleton(principal), - Collections.emptySet(), Collections.emptySet()); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - - // disable file system cache - cfg.setBoolean("fs.hdfs.impl.disable.cache", true); - - // set file system to TestingFs to avoid a bunch of security - // checks, similar to what is done in HdfsTests.java - cfg.set(String.format("fs.AbstractFileSystem.%s.impl", uri.getScheme()), - TestingFs.class.getName()); - - // create the FileContext with our user - return Subject.doAs(subject, new PrivilegedAction() { - @Override - public FileContext run() { - try { - TestingFs fs = (TestingFs) AbstractFileSystem.get(uri, cfg); - return FileContext.getFileContext(fs, cfg); - } catch (UnsupportedFileSystemException e) { - throw new RuntimeException(e); - } - } - }); - } -} diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java index 5e014ab3ecd..7dc6f3b6a83 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/blobstore/S3BlobContainer.java @@ -26,6 +26,7 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.elasticsearch.SpecialPermission; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobMetaData; import org.elasticsearch.common.blobstore.BlobPath; @@ -40,6 +41,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; import java.util.Map; /** @@ -60,8 +64,14 @@ public class S3BlobContainer extends AbstractBlobContainer { @Override public boolean blobExists(String blobName) { try { - blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName)); - return true; + return doPrivileged(() -> { + try { + blobStore.client().getObjectMetadata(blobStore.bucket(), buildKey(blobName)); + return true; + } catch (AmazonS3Exception e) { + return false; + } + }); } catch (AmazonS3Exception e) { return false; } catch (Throwable e) { @@ -108,10 +118,6 @@ public class S3BlobContainer extends AbstractBlobContainer { @Override public void deleteBlob(String blobName) throws IOException { - if (!blobExists(blobName)) { - throw new IOException("Blob [" + blobName + "] does not exist"); - } - try { blobStore.client().deleteObject(blobStore.bucket(), buildKey(blobName)); } catch (AmazonClientException e) { @@ -180,4 +186,19 @@ public class S3BlobContainer extends AbstractBlobContainer { return keyPath + blobName; } + /** + * + * Executes a {@link PrivilegedExceptionAction} with privileges enabled. + * + + */ + T doPrivileged(PrivilegedExceptionAction operation) throws IOException { + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + try { + return AccessController.doPrivileged(operation); + } catch (PrivilegedActionException e) { + throw (IOException) e.getException(); + } + } } diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java index fe184a06aef..e3b7e8296e0 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java @@ -57,7 +57,7 @@ import java.util.function.Function; */ public class S3Repository extends BlobStoreRepository { - public final static String TYPE = "s3"; + public static final String TYPE = "s3"; /** * Global S3 repositories settings. Starting with: repositories.s3 diff --git a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy index e5f26c3e9d1..1f09cada2e5 100644 --- a/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy @@ -22,4 +22,16 @@ grant { // TODO: get these fixed in aws sdk permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "getClassLoader"; + // Needed because of problems in AmazonS3Client: + // When no region is set on a AmazonS3Client instance, the + // AWS SDK loads all known partitions from a JSON file and + // uses a Jackson's ObjectMapper for that: this one, in + // version 2.5.3 with the default binding options, tries + // to suppress access checks of ctor/field/method and thus + // requires this special permission. AWS must be fixed to + // uses Jackson correctly and have the correct modifiers + // on binded classes. + // TODO: get these fixed in aws sdk + // See https://github.com/aws/aws-sdk-java/issues/766 + permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java b/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java deleted file mode 100644 index 8124f693943..00000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/cloud/aws/blobstore/MockAmazonS3.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.cloud.aws.blobstore; - -import com.amazonaws.AmazonClientException; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AbstractAmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CopyObjectRequest; -import com.amazonaws.services.s3.model.CopyObjectResult; -import com.amazonaws.services.s3.model.DeleteObjectRequest; -import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.util.Base64; - -import java.io.IOException; -import java.io.InputStream; -import java.security.DigestInputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -class MockAmazonS3 extends AbstractAmazonS3 { - - private Map blobs = new ConcurrentHashMap<>(); - - // in ESBlobStoreContainerTestCase.java, the maximum - // length of the input data is 100 bytes - private byte[] byteCounter = new byte[100]; - - @Override - public boolean doesBucketExist(String bucket) { - return true; - } - - @Override - public ObjectMetadata getObjectMetadata( - GetObjectMetadataRequest getObjectMetadataRequest) - throws AmazonClientException, AmazonServiceException { - String blobName = getObjectMetadataRequest.getKey(); - - if (!blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] does not exist."); - } - - return new ObjectMetadata(); // nothing is done with it - } - - @Override - public PutObjectResult putObject(PutObjectRequest putObjectRequest) - throws AmazonClientException, AmazonServiceException { - String blobName = putObjectRequest.getKey(); - DigestInputStream stream = (DigestInputStream) putObjectRequest.getInputStream(); - - if (blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] already exists."); - } - - blobs.put(blobName, stream); - - // input and output md5 hashes need to match to avoid an exception - String md5 = Base64.encodeAsString(stream.getMessageDigest().digest()); - PutObjectResult result = new PutObjectResult(); - result.setContentMd5(md5); - - return result; - } - - @Override - public S3Object getObject(GetObjectRequest getObjectRequest) - throws AmazonClientException, AmazonServiceException { - // in ESBlobStoreContainerTestCase.java, the prefix is empty, - // so the key and blobName are equivalent to each other - String blobName = getObjectRequest.getKey(); - - if (!blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] does not exist."); - } - - // the HTTP request attribute is irrelevant for reading - S3ObjectInputStream stream = new S3ObjectInputStream( - blobs.get(blobName), null, false); - S3Object s3Object = new S3Object(); - s3Object.setObjectContent(stream); - return s3Object; - } - - @Override - public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) - throws AmazonClientException, AmazonServiceException { - MockObjectListing list = new MockObjectListing(); - list.setTruncated(false); - - String blobName; - String prefix = listObjectsRequest.getPrefix(); - - ArrayList mockObjectSummaries = new ArrayList<>(); - - for (Map.Entry blob : blobs.entrySet()) { - blobName = blob.getKey(); - S3ObjectSummary objectSummary = new S3ObjectSummary(); - - if (prefix.isEmpty() || blobName.startsWith(prefix)) { - objectSummary.setKey(blobName); - - try { - objectSummary.setSize(getSize(blob.getValue())); - } catch (IOException e) { - throw new AmazonS3Exception("Object listing " + - "failed for blob [" + blob.getKey() + "]"); - } - - mockObjectSummaries.add(objectSummary); - } - } - - list.setObjectSummaries(mockObjectSummaries); - return list; - } - - @Override - public CopyObjectResult copyObject(CopyObjectRequest copyObjectRequest) - throws AmazonClientException, AmazonServiceException { - String sourceBlobName = copyObjectRequest.getSourceKey(); - String targetBlobName = copyObjectRequest.getDestinationKey(); - - if (!blobs.containsKey(sourceBlobName)) { - throw new AmazonS3Exception("Source blob [" + - sourceBlobName + "] does not exist."); - } - - if (blobs.containsKey(targetBlobName)) { - throw new AmazonS3Exception("Target blob [" + - targetBlobName + "] already exists."); - } - - blobs.put(targetBlobName, blobs.get(sourceBlobName)); - return new CopyObjectResult(); // nothing is done with it - } - - @Override - public void deleteObject(DeleteObjectRequest deleteObjectRequest) - throws AmazonClientException, AmazonServiceException { - String blobName = deleteObjectRequest.getKey(); - - if (!blobs.containsKey(blobName)) { - throw new AmazonS3Exception("[" + blobName + "] does not exist."); - } - - blobs.remove(blobName); - } - - private int getSize(InputStream stream) throws IOException { - int size = stream.read(byteCounter); - stream.reset(); // in case we ever need the size again - return size; - } - - private class MockObjectListing extends ObjectListing { - // the objectSummaries attribute in ObjectListing.java - // is read-only, but we need to be able to write to it, - // so we create a mock of it to work around this - private List mockObjectSummaries; - - @Override - public List getObjectSummaries() { - return mockObjectSummaries; - } - - private void setObjectSummaries(List objectSummaries) { - mockObjectSummaries = objectSummaries; - } - } -} diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java index 2e196610c1a..93c19df9c04 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AbstractS3SnapshotRestoreTest.java @@ -52,7 +52,7 @@ import static org.hamcrest.Matchers.notNullValue; /** */ @ClusterScope(scope = Scope.SUITE, numDataNodes = 2, numClientNodes = 0, transportClientRatio = 0.0) -abstract public class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase { +public abstract class AbstractS3SnapshotRestoreTest extends AbstractAwsTestCase { @Override public Settings nodeSettings(int nodeOrdinal) { diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java index 9e29d6f091a..6016b0ca335 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/AbstractAzureFsTestCase.java @@ -28,7 +28,7 @@ import java.util.Collection; import static org.hamcrest.Matchers.is; -abstract public class AbstractAzureFsTestCase extends ESIntegTestCase { +public abstract class AbstractAzureFsTestCase extends ESIntegTestCase { @Override protected Collection> nodePlugins() { return pluginList(SMBStorePlugin.class); diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash index 21882c0ad53..eac0a80e4f4 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash @@ -228,6 +228,10 @@ fi install_and_check_plugin ingest geoip geoip2-*.jar jackson-annotations-*.jar jackson-databind-*.jar maxmind-db-*.jar } +@test "[$GROUP] install ingest-useragent plugin" { + install_and_check_plugin ingest useragent +} + @test "[$GROUP] check ingest-common module" { check_module ingest-common jcodings-*.jar joni-*.jar } @@ -357,6 +361,10 @@ fi remove_plugin ingest-geoip } +@test "[$GROUP] remove ingest-useragent plugin" { + remove_plugin ingest-useragent +} + @test "[$GROUP] remove javascript plugin" { remove_plugin lang-javascript } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml index b814856144b..16260151a31 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/20_headers.yaml @@ -17,12 +17,8 @@ headers: Accept: application/yaml - - match: - $body: | - /^---\n - -\s+alias:\s+"test_alias"\s+ - index:\s+"test"\s+ - filter:\s+"-"\s+ - routing.index:\s+"-"\s+ - routing.search:\s+"-"\s+$/ - + - match: {0.alias: test_alias} + - match: {0.index: test} + - match: {0.filter: "-"} + - match: {0.routing\.index: "-"} + - match: {0.routing\.search: "-"} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml index c892891f08f..178b77ce60d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.aliases/30_yaml.yaml @@ -1,8 +1,5 @@ --- -"Simple alias with yaml body through format argument": - - - skip: - features: yaml +"Simple alias with json body through format argument": - do: indices.create: @@ -15,15 +12,10 @@ - do: cat.aliases: - format: yaml - - - match: - $body: | - /^---\n - -\s+alias:\s+"test_alias"\s+ - index:\s+"test"\s+ - filter:\s+"-"\s+ - routing.index:\s+"-"\s+ - routing.search:\s+"-"\s+$/ - + format: json + - match: {0.alias: test_alias} + - match: {0.index: test} + - match: {0.filter: "-"} + - match: {0.routing\.index: "-"} + - match: {0.routing\.search: "-"} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml index 51f8fe9ed4c..1e485fc6d30 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.indices/10_basic.yaml @@ -26,6 +26,7 @@ /^(green \s+ open \s+ index1 \s+ + ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+ 1 \s+ 0 \s+ 0 \s+ @@ -62,6 +63,7 @@ /^( \s+ close \s+ index1 \s+ + ([a-zA-Z0-9=/_+]|[\\\-]){22} \s+ \s+ \s+ \s+ diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml index 1bb031f0878..ed9a5b7c99f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/get/50_with_headers.yaml @@ -18,13 +18,9 @@ type: _all id: 1 - - match: - $body: | - /^---\n - _index:\s+\"test_1"\n - _type:\s+"test"\n - _id:\s+"1"\n - _version:\s+1\n - found:\s+true\n - _source:\n - \s+body:\s+"foo"\n$/ + - match: {_index: "test_1"} + - match: {_type: "test"} + - match: {_id: "1"} + - match: {_version: 1} + - match: {found: true} + - match: { _source: { body: foo }} diff --git a/settings.gradle b/settings.gradle index 6588b605a9d..0c1e4b01e5c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -36,6 +36,7 @@ List projects = [ 'plugins:discovery-gce', 'plugins:ingest-geoip', 'plugins:ingest-attachment', + 'plugins:ingest-useragent', 'plugins:lang-javascript', 'plugins:lang-python', 'plugins:mapper-attachments', diff --git a/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java b/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java deleted file mode 100644 index a1c9da18470..00000000000 --- a/test/framework/src/main/java/org/elasticsearch/common/bytes/ByteBufferBytesReference.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.bytes; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; -import org.jboss.netty.util.CharsetUtil; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.nio.charset.CharacterCodingException; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CoderResult; -import java.nio.charset.StandardCharsets; - -/** - * Note: this is only used by one lone test method. - */ -public class ByteBufferBytesReference implements BytesReference { - - private final ByteBuffer buffer; - - public ByteBufferBytesReference(ByteBuffer buffer) { - this.buffer = buffer; - } - - @Override - public byte get(int index) { - return buffer.get(buffer.position() + index); - } - - @Override - public int length() { - return buffer.remaining(); - } - - @Override - public BytesReference slice(int from, int length) { - ByteBuffer dup = buffer.duplicate(); - dup.position(buffer.position() + from); - dup.limit(buffer.position() + from + length); - return new ByteBufferBytesReference(dup); - } - - @Override - public StreamInput streamInput() { - return new ByteBufferStreamInput(buffer); - } - - @Override - public void writeTo(OutputStream os) throws IOException { - if (buffer.hasArray()) { - os.write(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); - } else { - byte[] tmp = new byte[8192]; - ByteBuffer buf = buffer.duplicate(); - while (buf.hasRemaining()) { - buf.get(tmp, 0, Math.min(tmp.length, buf.remaining())); - os.write(tmp); - } - } - } - - @Override - public byte[] toBytes() { - if (!buffer.hasRemaining()) { - return BytesRef.EMPTY_BYTES; - } - byte[] tmp = new byte[buffer.remaining()]; - buffer.duplicate().get(tmp); - return tmp; - } - - @Override - public BytesArray toBytesArray() { - if (buffer.hasArray()) { - return new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); - } - return new BytesArray(toBytes()); - } - - @Override - public BytesArray copyBytesArray() { - return new BytesArray(toBytes()); - } - - @Override - public boolean hasArray() { - return buffer.hasArray(); - } - - @Override - public byte[] array() { - return buffer.array(); - } - - @Override - public int arrayOffset() { - return buffer.arrayOffset() + buffer.position(); - } - - @Override - public int hashCode() { - return Helper.bytesHashCode(this); - } - - @Override - public boolean equals(Object obj) { - return Helper.bytesEqual(this, (BytesReference) obj); - } - - @Override - public String toUtf8() { - if (!buffer.hasRemaining()) { - return ""; - } - final CharsetDecoder decoder = CharsetUtil.getDecoder(StandardCharsets.UTF_8); - final CharBuffer dst = CharBuffer.allocate( - (int) ((double) buffer.remaining() * decoder.maxCharsPerByte())); - try { - CoderResult cr = decoder.decode(buffer, dst, true); - if (!cr.isUnderflow()) { - cr.throwException(); - } - cr = decoder.flush(dst); - if (!cr.isUnderflow()) { - cr.throwException(); - } - } catch (CharacterCodingException x) { - throw new IllegalStateException(x); - } - return dst.flip().toString(); - } - - @Override - public BytesRef toBytesRef() { - if (buffer.hasArray()) { - return new BytesRef(buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); - } - return new BytesRef(toBytes()); - } - - @Override - public BytesRef copyBytesRef() { - return new BytesRef(toBytes()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java index 613445c2271..22a1e2660b6 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java @@ -246,7 +246,7 @@ public class MockBigArrays extends BigArrays { return arr; } - private static abstract class AbstractArrayWrapper { + private abstract static class AbstractArrayWrapper { final BigArray in; boolean clearOnResize; diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java index 67ad0eb7358..6ff0b71cdcc 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java @@ -112,22 +112,6 @@ public abstract class ESBlobStoreContainerTestCase extends ESTestCase { } } - public void testDeleteBlob() throws IOException { - try (final BlobStore store = newBlobStore()) { - final String blobName = "foobar"; - final BlobContainer container = store.blobContainer(new BlobPath()); - expectThrows(IOException.class, () -> container.deleteBlob(blobName)); - - byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); - final BytesArray bytesArray = new BytesArray(data); - container.writeBlob(blobName, bytesArray); - container.deleteBlob(blobName); // should not raise - - // blob deleted, so should raise again - expectThrows(IOException.class, () -> container.deleteBlob(blobName)); - } - } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/15579") public void testOverwriteFails() throws IOException { try (final BlobStore store = newBlobStore()) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index e86854b34c4..0d9ed196838 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -326,11 +326,11 @@ public abstract class AbstractQueryTestCase> /** * Parses the query provided as string argument and compares it with the expected result provided as argument as a {@link QueryBuilder} */ - protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException { + protected static final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery) throws IOException { assertParsedQuery(queryAsString, expectedQuery, ParseFieldMatcher.STRICT); } - protected final static void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { + protected static final void assertParsedQuery(String queryAsString, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { QueryBuilder newQuery = parseQuery(queryAsString, matcher); assertNotSame(newQuery, expectedQuery); assertEquals(expectedQuery, newQuery); @@ -340,31 +340,31 @@ public abstract class AbstractQueryTestCase> /** * Parses the query provided as bytes argument and compares it with the expected result provided as argument as a {@link QueryBuilder} */ - protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException { + protected static final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery) throws IOException { assertParsedQuery(queryAsBytes, expectedQuery, ParseFieldMatcher.STRICT); } - protected final static void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { + protected static final void assertParsedQuery(BytesReference queryAsBytes, QueryBuilder expectedQuery, ParseFieldMatcher matcher) throws IOException { QueryBuilder newQuery = parseQuery(queryAsBytes, matcher); assertNotSame(newQuery, expectedQuery); assertEquals(expectedQuery, newQuery); assertEquals(expectedQuery.hashCode(), newQuery.hashCode()); } - protected final static QueryBuilder parseQuery(String queryAsString) throws IOException { + protected static final QueryBuilder parseQuery(String queryAsString) throws IOException { return parseQuery(queryAsString, ParseFieldMatcher.STRICT); } - protected final static QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException { + protected static final QueryBuilder parseQuery(String queryAsString, ParseFieldMatcher matcher) throws IOException { XContentParser parser = XContentFactory.xContent(queryAsString).createParser(queryAsString); return parseQuery(parser, matcher); } - protected final static QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException { + protected static final QueryBuilder parseQuery(BytesReference queryAsBytes) throws IOException { return parseQuery(queryAsBytes, ParseFieldMatcher.STRICT); } - protected final static QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException { + protected static final QueryBuilder parseQuery(BytesReference queryAsBytes, ParseFieldMatcher matcher) throws IOException { XContentParser parser = XContentFactory.xContent(queryAsBytes).createParser(queryAsBytes); return parseQuery(parser, matcher); } @@ -513,7 +513,7 @@ public abstract class AbstractQueryTestCase> protected QueryBuilder assertSerialization(QueryBuilder testQuery) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.writeNamedWriteable(testQuery); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) { QueryBuilder deserializedQuery = in.readNamedWriteable(QueryBuilder.class); assertEquals(testQuery, deserializedQuery); assertEquals(testQuery.hashCode(), deserializedQuery.hashCode()); @@ -562,7 +562,7 @@ public abstract class AbstractQueryTestCase> protected QB copyQuery(QB query) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { output.writeNamedWriteable(query); - try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), serviceHolder.namedWriteableRegistry)) { + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), serviceHolder.namedWriteableRegistry)) { return (QB) in.readNamedWriteable(QueryBuilder.class); } } @@ -690,7 +690,7 @@ public abstract class AbstractQueryTestCase> } public static class GeohashGenerator extends CodepointSetGenerator { - private final static char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray(); + private static final char[] ASCII_SET = "0123456789bcdefghjkmnpqrstuvwxyz".toCharArray(); public GeohashGenerator() { super(ASCII_SET); diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index a49c33f5be8..d13f69a3765 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -106,7 +106,6 @@ import org.elasticsearch.index.MergeSchedulerConfig; import org.elasticsearch.index.MockEngineFactoryPlugin; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesQueryCache; import org.elasticsearch.indices.IndicesRequestCache; @@ -405,7 +404,7 @@ public abstract class ESIntegTestCase extends ESTestCase { .setOrder(0) .setSettings(randomSettingsBuilder); if (mappings != null) { - logger.info("test using _default_ mappings: [{}]", mappings.bytes().toUtf8()); + logger.info("test using _default_ mappings: [{}]", mappings.bytes().utf8ToString()); putTemplate.addMapping("_default_", mappings); } assertAcked(putTemplate.execute().actionGet()); @@ -925,7 +924,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * This saves on unneeded searches. * @return the actual number of docs seen. */ - public long waitForDocs(final long numDocs, final @Nullable BackgroundIndexer indexer) throws InterruptedException { + public long waitForDocs(final long numDocs, @Nullable final BackgroundIndexer indexer) throws InterruptedException { // indexing threads can wait for up to ~1m before retrying when they first try to index into a shard which is not STARTED. return waitForDocs(numDocs, 90, TimeUnit.SECONDS, indexer); } @@ -940,7 +939,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * This saves on unneeded searches. * @return the actual number of docs seen. */ - public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, final @Nullable BackgroundIndexer indexer) + public long waitForDocs(final long numDocs, int maxWaitTime, TimeUnit maxWaitTimeUnit, @Nullable final BackgroundIndexer indexer) throws InterruptedException { final AtomicLong lastKnownCount = new AtomicLong(-1); long lastStartCount = -1; @@ -2035,7 +2034,7 @@ public abstract class ESIntegTestCase extends ESTestCase { * The returned client gets automatically closed when needed, it shouldn't be closed as part of tests otherwise * it cannot be reused by other tests anymore. */ - protected synchronized static RestClient getRestClient() { + protected static synchronized RestClient getRestClient() { if (restClient == null) { restClient = createRestClient(null); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index b6b75b1ec64..11bad415764 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -1337,7 +1337,7 @@ public final class InternalTestCluster extends TestCluster { /** * Restarts a node and calls the callback during restart. */ - synchronized public void restartNode(String nodeName, RestartCallback callback) throws Exception { + public synchronized void restartNode(String nodeName, RestartCallback callback) throws Exception { ensureOpen(); NodeAndClient nodeAndClient = nodes.get(nodeName); if (nodeAndClient != null) { @@ -1346,7 +1346,7 @@ public final class InternalTestCluster extends TestCluster { } } - synchronized private void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception { + private synchronized void restartAllNodes(boolean rollingRestart, RestartCallback callback) throws Exception { ensureOpen(); List toRemove = new ArrayList<>(); try { diff --git a/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java b/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java index 10469286e1a..1d0eaa7ce51 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/StreamsUtils.java @@ -19,6 +19,7 @@ package org.elasticsearch.test; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -53,7 +54,7 @@ public class StreamsUtils { } try (BytesStreamOutput out = new BytesStreamOutput()) { Streams.copy(is, out); - return out.bytes().toBytes(); + return BytesReference.toBytes(out.bytes()); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java index 7c832118ca0..48b726dc3cc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java +++ b/test/framework/src/main/java/org/elasticsearch/test/discovery/ClusterDiscoveryConfiguration.java @@ -126,7 +126,7 @@ public class ClusterDiscoveryConfiguration extends NodeConfigurationSource { } @SuppressForbidden(reason = "we know we pass a IP address") - protected synchronized static int[] unicastHostPorts(int numHosts) { + protected static synchronized int[] unicastHostPorts(int numHosts) { int[] unicastHostPorts = new int[numHosts]; final int basePort = calcBasePort(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java index d957220c6df..caab35e4b42 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/IntermittentLongGCDisruption.java @@ -61,7 +61,7 @@ public class IntermittentLongGCDisruption extends LongGCDisruption { this.delayDurationMax = delayDurationMax; } - final static AtomicInteger thread_ids = new AtomicInteger(); + static final AtomicInteger thread_ids = new AtomicInteger(); @Override public void startDisrupting() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java index 591540e72b9..b210a20cf70 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java +++ b/test/framework/src/main/java/org/elasticsearch/test/disruption/LongGCDisruption.java @@ -33,7 +33,7 @@ import java.util.regex.Pattern; */ public class LongGCDisruption extends SingleNodeDisruption { - private final static Pattern[] unsafeClasses = new Pattern[]{ + private static final Pattern[] unsafeClasses = new Pattern[]{ // logging has shared JVM locks - we may suspend a thread and block other nodes from doing their thing Pattern.compile("Logger") }; diff --git a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java index bf32b6b8575..304e3047496 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/engine/MockEngineSupport.java @@ -172,7 +172,7 @@ public final class MockEngineSupport { return reader; } - public static abstract class DirectoryReaderWrapper extends FilterDirectoryReader { + public abstract static class DirectoryReaderWrapper extends FilterDirectoryReader { protected final SubReaderWrapper subReaderWrapper; public DirectoryReaderWrapper(DirectoryReader in, SubReaderWrapper subReaderWrapper) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java index baad3ef6f04..82f2346c421 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java +++ b/test/framework/src/main/java/org/elasticsearch/test/hamcrest/ElasticsearchAssertions.java @@ -647,7 +647,7 @@ public class ElasticsearchAssertions { ((ActionRequest) streamable).validate(); } BytesReference orig = serialize(version, streamable); - StreamInput input = StreamInput.wrap(orig); + StreamInput input = orig.streamInput(); if (namedWriteableRegistry != null) { input = new NamedWriteableAwareStreamInput(input, namedWriteableRegistry); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java similarity index 78% rename from test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java rename to test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java index b338d76d985..8c492d279b0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/json/JsonPath.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ObjectPath.java @@ -16,11 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.test.rest.json; +package org.elasticsearch.test.rest; +import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.test.rest.Stash; import java.io.IOException; import java.util.ArrayList; @@ -28,22 +27,23 @@ import java.util.List; import java.util.Map; /** - * Holds a json object and allows to extract specific values from it + * Holds an object and allows to extract specific values from it given their path */ -public class JsonPath { +public class ObjectPath { - final String json; - final Map jsonMap; + private final Object object; - public JsonPath(String json) throws IOException { - this.json = json; - this.jsonMap = convertToMap(json); + public static ObjectPath createFromXContent(XContent xContent, String input) throws IOException { + try (XContentParser parser = xContent.createParser(input)) { + if (parser.nextToken() == XContentParser.Token.START_ARRAY) { + return new ObjectPath(parser.listOrderedMap()); + } + return new ObjectPath(parser.mapOrdered()); + } } - private static Map convertToMap(String json) throws IOException { - try (XContentParser parser = JsonXContent.jsonXContent.createParser(json)) { - return parser.mapOrdered(); - } + public ObjectPath(Object object) { + this.object = object; } /** @@ -58,7 +58,7 @@ public class JsonPath { */ public Object evaluate(String path, Stash stash) throws IOException { String[] parts = parsePath(path); - Object object = jsonMap; + Object object = this.object; for (String part : parts) { object = evaluate(part, object, stash); if (object == null) { @@ -70,8 +70,8 @@ public class JsonPath { @SuppressWarnings("unchecked") private Object evaluate(String key, Object object, Stash stash) throws IOException { - if (stash.isStashedValue(key)) { - key = stash.unstashValue(key).toString(); + if (stash.containsStashedValue(key)) { + key = stash.getValue(key).toString(); } if (object instanceof Map) { @@ -84,7 +84,8 @@ public class JsonPath { } catch (NumberFormatException e) { throw new IllegalArgumentException("element was a list, but [" + key + "] was not numeric", e); } catch (IndexOutOfBoundsException e) { - throw new IllegalArgumentException("element was a list with " + list.size() + " elements, but [" + key + "] was out of bounds", e); + throw new IllegalArgumentException("element was a list with " + list.size() + + " elements, but [" + key + "] was out of bounds", e); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java index 34397f03d94..d7295e1dca7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/RestTestExecutionContext.java @@ -66,8 +66,8 @@ public class RestTestExecutionContext implements Closeable { //makes a copy of the parameters before modifying them for this specific request HashMap requestParams = new HashMap<>(params); for (Map.Entry entry : requestParams.entrySet()) { - if (stash.isStashedValue(entry.getValue())) { - entry.setValue(stash.unstashValue(entry.getValue()).toString()); + if (stash.containsStashedValue(entry.getValue())) { + entry.setValue(stash.getValue(entry.getValue()).toString()); } } @@ -76,7 +76,7 @@ public class RestTestExecutionContext implements Closeable { try { response = callApiInternal(apiName, requestParams, body, headers); //we always stash the last response body - stash.stashResponse(response); + stash.stashValue("body", response.getBody()); return response; } catch(ResponseException e) { response = new RestTestResponse(e); @@ -90,12 +90,12 @@ public class RestTestExecutionContext implements Closeable { } if (bodies.size() == 1) { - return bodyAsString(stash.unstashMap(bodies.get(0))); + return bodyAsString(stash.replaceStashedValues(bodies.get(0))); } StringBuilder bodyBuilder = new StringBuilder(); for (Map body : bodies) { - bodyBuilder.append(bodyAsString(stash.unstashMap(body))).append("\n"); + bodyBuilder.append(bodyAsString(stash.replaceStashedValues(body))).append("\n"); } return bodyBuilder.toString(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java index 885df395c2b..a4218f83340 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/Stash.java @@ -24,25 +24,27 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.test.rest.client.RestTestResponse; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Allows to cache the last obtained test response and or part of it within variables * that can be used as input values in following requests and assertions. */ public class Stash implements ToXContent { + private static final Pattern EXTENDED_KEY = Pattern.compile("\\$\\{([^}]+)\\}"); private static final ESLogger logger = Loggers.getLogger(Stash.class); public static final Stash EMPTY = new Stash(); private final Map stash = new HashMap<>(); - private RestTestResponse response; + private final ObjectPath stashObjectPath = new ObjectPath(stash); /** * Allows to saved a specific field in the stash as key-value pair @@ -55,12 +57,6 @@ public class Stash implements ToXContent { } } - public void stashResponse(RestTestResponse response) throws IOException { - // TODO we can almost certainly save time by lazily evaluating the body - stashValue("body", response.getBody()); - this.response = response; - } - /** * Clears the previously stashed values */ @@ -69,41 +65,63 @@ public class Stash implements ToXContent { } /** - * Tells whether a particular value needs to be looked up in the stash + * Tells whether a particular key needs to be looked up in the stash based on its name. + * Returns true if the string representation of the key starts with "$", false otherwise * The stash contains fields eventually extracted from previous responses that can be reused * as arguments for following requests (e.g. scroll_id) */ - public boolean isStashedValue(Object key) { + public boolean containsStashedValue(Object key) { if (key == null) { return false; } String stashKey = key.toString(); - return Strings.hasLength(stashKey) && stashKey.startsWith("$"); + if (false == Strings.hasLength(stashKey)) { + return false; + } + if (stashKey.startsWith("$")) { + return true; + } + return EXTENDED_KEY.matcher(stashKey).find(); } /** - * Extracts a value from the current stash + * Retrieves a value from the current stash. * The stash contains fields eventually extracted from previous responses that can be reused * as arguments for following requests (e.g. scroll_id) */ - public Object unstashValue(String value) throws IOException { - if (value.startsWith("$body.")) { - if (response == null) { - return null; - } - return response.evaluate(value.substring("$body".length()), this); + public Object getValue(String key) throws IOException { + if (key.charAt(0) == '$' && key.charAt(1) != '{') { + return unstash(key.substring(1)); } - Object stashedValue = stash.get(value.substring(1)); + Matcher matcher = EXTENDED_KEY.matcher(key); + /* + * String*Buffer* because that is what the Matcher API takes. In modern versions of java the uncontended synchronization is very, + * very cheap so that should not be a problem. + */ + StringBuffer result = new StringBuffer(key.length()); + if (false == matcher.find()) { + throw new IllegalArgumentException("Doesn't contain any stash keys [" + key + "]"); + } + do { + matcher.appendReplacement(result, Matcher.quoteReplacement(unstash(matcher.group(1)).toString())); + } while (matcher.find()); + matcher.appendTail(result); + return result.toString(); + } + + private Object unstash(String key) throws IOException { + Object stashedValue = stashObjectPath.evaluate(key); if (stashedValue == null) { - throw new IllegalArgumentException("stashed value not found for key [" + value + "]"); + throw new IllegalArgumentException("stashed value not found for key [" + key + "]"); } return stashedValue; } /** - * Recursively unstashes map values if needed + * Goes recursively against each map entry and replaces any string value starting with "$" with its + * corresponding value retrieved from the stash */ - public Map unstashMap(Map map) throws IOException { + public Map replaceStashedValues(Map map) throws IOException { Map copy = new HashMap<>(map); unstashObject(copy); return copy; @@ -115,8 +133,8 @@ public class Stash implements ToXContent { List list = (List) obj; for (int i = 0; i < list.size(); i++) { Object o = list.get(i); - if (isStashedValue(o)) { - list.set(i, unstashValue(o.toString())); + if (containsStashedValue(o)) { + list.set(i, getValue(o.toString())); } else { unstashObject(o); } @@ -125,8 +143,8 @@ public class Stash implements ToXContent { if (obj instanceof Map) { Map map = (Map) obj; for (Map.Entry entry : map.entrySet()) { - if (isStashedValue(entry.getValue())) { - entry.setValue(unstashValue(entry.getValue().toString())); + if (containsStashedValue(entry.getValue())) { + entry.setValue(getValue(entry.getValue().toString())); } else { unstashObject(entry.getValue()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java index 5b5773d6fdc..4644b87b8e7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/client/RestTestResponse.java @@ -23,23 +23,24 @@ import org.apache.http.util.EntityUtils; import org.apache.lucene.util.IOUtils; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.rest.ObjectPath; import org.elasticsearch.test.rest.Stash; -import org.elasticsearch.test.rest.json.JsonPath; import java.io.IOException; import java.nio.charset.StandardCharsets; /** * Response obtained from a REST call, eagerly reads the response body into a string for later optional parsing. - * Supports parsing the response body as json when needed and returning specific values extracted from it. + * Supports parsing the response body when needed and returning specific values extracted from it. */ public class RestTestResponse { private final Response response; private final String body; - private JsonPath parsedResponse; + private ObjectPath parsedResponse; - public RestTestResponse(Response response) { + public RestTestResponse(Response response) throws IOException { this.response = response; if (response.getEntity() != null) { try { @@ -53,11 +54,24 @@ public class RestTestResponse { } else { this.body = null; } + parseResponseBody(); } - public RestTestResponse(ResponseException responseException) { + public RestTestResponse(ResponseException responseException) throws IOException { this.response = responseException.getResponse(); this.body = responseException.getResponseBody(); + parseResponseBody(); + } + + private void parseResponseBody() throws IOException { + if (body != null) { + String contentType = response.getHeader("Content-Type"); + XContentType xContentType = XContentType.fromMediaTypeOrFormat(contentType); + //skip parsing if we got text back (e.g. if we called _cat apis) + if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) { + this.parsedResponse = ObjectPath.createFromXContent(xContentType.xContent(), body); + } + } } public int getStatusCode() { @@ -73,11 +87,7 @@ public class RestTestResponse { * Might be a string or a json object parsed as a map. */ public Object getBody() throws IOException { - if (isJson()) { - JsonPath parsedResponse = parsedResponse(); - if (parsedResponse == null) { - return null; - } + if (parsedResponse != null) { return parsedResponse.evaluate(""); } return body; @@ -95,23 +105,21 @@ public class RestTestResponse { } /** - * Parses the response body as json and extracts a specific value from it (identified by the provided path) + * Parses the response body and extracts a specific value from it (identified by the provided path) */ public Object evaluate(String path) throws IOException { return evaluate(path, Stash.EMPTY); } /** - * Parses the response body as json and extracts a specific value from it (identified by the provided path) + * Parses the response body and extracts a specific value from it (identified by the provided path) */ public Object evaluate(String path, Stash stash) throws IOException { if (response == null) { return null; } - JsonPath jsonPath = parsedResponse(); - - if (jsonPath == null) { + if (parsedResponse == null) { //special case: api that don't support body (e.g. exists) return true if 200, false if 404, even if no body //is_true: '' means the response had no body but the client returned true (caused by 200) //is_false: '' means the response had no body but the client returned false (caused by 404) @@ -121,21 +129,6 @@ public class RestTestResponse { return null; } - return jsonPath.evaluate(path, stash); - } - - private boolean isJson() { - String contentType = response.getHeader("Content-Type"); - return contentType != null && contentType.contains("application/json"); - } - - private JsonPath parsedResponse() throws IOException { - if (parsedResponse != null) { - return parsedResponse; - } - if (response == null || body == null) { - return null; - } - return parsedResponse = new JsonPath(body); + return parsedResponse.evaluate(path, stash); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java index 68f833d35c7..7a4cd0f316a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanEqualToParser.java @@ -33,7 +33,8 @@ public class GreaterThanEqualToParser implements RestTestFragmentParser stringObjectTuple = parseContext.parseTuple(); if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new RestTestParseException("gte section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName()); + throw new RestTestParseException("gte section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); } return new GreaterThanEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java index a66122138c1..7e1ca1ece7f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/GreaterThanParser.java @@ -32,7 +32,8 @@ public class GreaterThanParser implements RestTestFragmentParser stringObjectTuple = parseContext.parseTuple(); if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new RestTestParseException("gt section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName()); + throw new RestTestParseException("gt section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); } return new GreaterThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java index f2d53d05a56..a30979c6a3c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanOrEqualToParser.java @@ -33,7 +33,8 @@ public class LessThanOrEqualToParser implements RestTestFragmentParser stringObjectTuple = parseContext.parseTuple(); if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new RestTestParseException("lte section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName()); + throw new RestTestParseException("lte section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); } return new LessThanOrEqualToAssertion(stringObjectTuple.v1(), stringObjectTuple.v2()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java index 065dd19d6a1..fc31f221758 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/LessThanParser.java @@ -32,7 +32,8 @@ public class LessThanParser implements RestTestFragmentParser public LessThanAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException { Tuple stringObjectTuple = parseContext.parseTuple(); if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new RestTestParseException("lt section can only be used with objects that support natural ordering, found " + stringObjectTuple.v2().getClass().getSimpleName()); + throw new RestTestParseException("lt section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); } return new LessThanAssertion(stringObjectTuple.v1(), stringObjectTuple.v2()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java index e972aea641a..f7325443deb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParseContext.java @@ -18,22 +18,21 @@ */ package org.elasticsearch.test.rest.parser; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.test.rest.section.DoSection; import org.elasticsearch.test.rest.section.ExecutableSection; -import org.elasticsearch.test.rest.section.ResponseBodyAssertion; import org.elasticsearch.test.rest.section.SetupSection; import org.elasticsearch.test.rest.section.SkipSection; import org.elasticsearch.test.rest.section.TeardownSection; import org.elasticsearch.test.rest.section.TestSection; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + /** * Context shared across the whole tests parse phase. * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version) @@ -57,7 +56,6 @@ public class RestTestSuiteParseContext { EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser()); EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser()); EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser()); - EXECUTABLE_SECTIONS_PARSERS.put("response_body", ResponseBodyAssertion.PARSER); } private final String api; @@ -160,7 +158,8 @@ public class RestTestSuiteParseContext { token = parser.nextToken(); } if (token != XContentParser.Token.FIELD_NAME) { - throw new RestTestParseException("malformed test section: field name expected but found " + token + " at " + parser.getTokenLocation()); + throw new RestTestParseException("malformed test section: field name expected but found " + token + " at " + + parser.getTokenLocation()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java index c6986d3eac8..f22f0109594 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/parser/RestTestSuiteParser.java @@ -71,7 +71,8 @@ public class RestTestSuiteParser implements RestTestFragmentParser map = (Map) expectedValue; - return executionContext.stash().unstashMap(map); + return executionContext.stash().replaceStashedValues(map); } - if (executionContext.stash().isStashedValue(expectedValue)) { - return executionContext.stash().unstashValue(expectedValue.toString()); + if (executionContext.stash().containsStashedValue(expectedValue)) { + return executionContext.stash().getValue(expectedValue.toString()); } return expectedValue; } protected final Object getActualValue(RestTestExecutionContext executionContext) throws IOException { - if (executionContext.stash().isStashedValue(field)) { - return executionContext.stash().unstashValue(field); + if (executionContext.stash().containsStashedValue(field)) { + return executionContext.stash().getValue(field); } return executionContext.response(field); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java index ade7fbd59ca..63f69696653 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanAssertion.java @@ -43,8 +43,10 @@ public class GreaterThanAssertion extends Assertion { @SuppressWarnings("unchecked") protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] is greater than [{}] (field: [{}])", actualValue, expectedValue, getField()); - assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class)); - assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class)); + assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", + actualValue, instanceOf(Comparable.class)); + assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", + expectedValue, instanceOf(Comparable.class)); try { assertThat(errorMessage(), (Comparable) actualValue, greaterThan((Comparable) expectedValue)); } catch (ClassCastException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java index cfdca7bc338..297eecf2d2a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/GreaterThanEqualToAssertion.java @@ -43,8 +43,10 @@ public class GreaterThanEqualToAssertion extends Assertion { @Override protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] is greater than or equal to [{}] (field: [{}])", actualValue, expectedValue, getField()); - assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class)); - assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class)); + assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", + actualValue, instanceOf(Comparable.class)); + assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", + expectedValue, instanceOf(Comparable.class)); try { assertThat(errorMessage(), (Comparable) actualValue, greaterThanOrEqualTo((Comparable) expectedValue)); } catch (ClassCastException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java index 265487a0388..eb28ba01a94 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LengthAssertion.java @@ -44,7 +44,8 @@ public class LengthAssertion extends Assertion { @Override protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] has length [{}] (field: [{}])", actualValue, expectedValue, getField()); - assertThat("expected value of [" + getField() + "] is not numeric (got [" + expectedValue.getClass() + "]", expectedValue, instanceOf(Number.class)); + assertThat("expected value of [" + getField() + "] is not numeric (got [" + expectedValue.getClass() + "]", + expectedValue, instanceOf(Number.class)); int length = ((Number) expectedValue).intValue(); if (actualValue instanceof String) { assertThat(errorMessage(), ((String) actualValue).length(), equalTo(length)); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java index 89387ff8952..153a7824569 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanAssertion.java @@ -44,8 +44,10 @@ public class LessThanAssertion extends Assertion { @SuppressWarnings("unchecked") protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField()); - assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class)); - assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class)); + assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", + actualValue, instanceOf(Comparable.class)); + assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", + expectedValue, instanceOf(Comparable.class)); try { assertThat(errorMessage(), (Comparable) actualValue, lessThan((Comparable) expectedValue)); } catch (ClassCastException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java index 99cbf1155d5..1eb3a9fc2b2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/LessThanOrEqualToAssertion.java @@ -43,8 +43,10 @@ public class LessThanOrEqualToAssertion extends Assertion { @Override protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] is less than or equal to [{}] (field: [{}])", actualValue, expectedValue, getField()); - assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", actualValue, instanceOf(Comparable.class)); - assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", expectedValue, instanceOf(Comparable.class)); + assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", + actualValue, instanceOf(Comparable.class)); + assertThat("expected value of [" + getField() + "] is not comparable (got [" + expectedValue.getClass() + "])", + expectedValue, instanceOf(Comparable.class)); try { assertThat(errorMessage(), (Comparable) actualValue, lessThanOrEqualTo((Comparable) expectedValue)); } catch (ClassCastException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java index e00fbbea01c..3a96d4532a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/section/MatchAssertion.java @@ -18,15 +18,21 @@ */ package org.elasticsearch.test.rest.section; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; import java.util.regex.Pattern; import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; /** @@ -45,12 +51,12 @@ public class MatchAssertion extends Assertion { @Override protected void doAssert(Object actualValue, Object expectedValue) { - //if the value is wrapped into / it is a regexp (e.g. /s+d+/) if (expectedValue instanceof String) { String expValue = ((String) expectedValue).trim(); if (expValue.length() > 2 && expValue.startsWith("/") && expValue.endsWith("/")) { - assertThat("field [" + getField() + "] was expected to be of type String but is an instanceof [" + safeClass(actualValue) + "]", actualValue, instanceOf(String.class)); + assertThat("field [" + getField() + "] was expected to be of type String but is an instanceof [" + + safeClass(actualValue) + "]", actualValue, instanceOf(String.class)); String stringValue = (String) actualValue; String regex = expValue.substring(1, expValue.length() - 1); logger.trace("assert that [{}] matches [{}]", stringValue, regex); @@ -60,20 +66,131 @@ public class MatchAssertion extends Assertion { } } - assertThat(errorMessage(), actualValue, notNullValue()); + assertNotNull("field [" + getField() + "] is null", actualValue); logger.trace("assert that [{}] matches [{}] (field [{}])", actualValue, expectedValue, getField()); - if (!actualValue.getClass().equals(safeClass(expectedValue))) { + if (actualValue.getClass().equals(safeClass(expectedValue)) == false) { if (actualValue instanceof Number && expectedValue instanceof Number) { //Double 1.0 is equal to Integer 1 - assertThat(errorMessage(), ((Number) actualValue).doubleValue(), equalTo(((Number) expectedValue).doubleValue())); + assertThat("field [" + getField() + "] doesn't match the expected value", + ((Number) actualValue).doubleValue(), equalTo(((Number) expectedValue).doubleValue())); return; } } - assertThat(errorMessage(), actualValue, equalTo(expectedValue)); + if (expectedValue.equals(actualValue) == false) { + FailureMessage message = new FailureMessage(getField()); + message.compare(getField(), actualValue, expectedValue); + throw new AssertionError(message.message); + } } - private String errorMessage() { - return "field [" + getField() + "] doesn't match the expected value"; + private static class FailureMessage { + private final StringBuilder message; + private int indent = 0; + + private FailureMessage(String field) { + this.message = new StringBuilder(field + " didn't match the expected value:\n"); + } + + private void compareMaps(Map actual, Map expected) { + actual = new TreeMap<>(actual); + expected = new TreeMap<>(expected); + for (Map.Entry expectedEntry : expected.entrySet()) { + compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue()); + } + for (Map.Entry unmatchedEntry : actual.entrySet()) { + field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]"); + } + } + + private void compareLists(List actual, List expected) { + int i = 0; + while (i < actual.size() && i < expected.size()) { + compare(Integer.toString(i), actual.get(i), expected.get(i)); + i++; + } + if (actual.size() == expected.size()) { + return; + } + indent(); + if (actual.size() < expected.size()) { + message.append("expected [").append(expected.size() - i).append("] more entries\n"); + return; + } + message.append("received [").append(actual.size() - i).append("] more entries than expected\n"); + } + + private void compare(String field, @Nullable Object actual, Object expected) { + if (expected instanceof Map) { + if (actual == null) { + field(field, "expected map but not found"); + return; + } + if (false == actual instanceof Map) { + field(field, "expected map but found [" + actual + "]"); + return; + } + @SuppressWarnings("unchecked") + Map expectedMap = (Map) expected; + @SuppressWarnings("unchecked") + Map actualMap = (Map) actual; + if (expectedMap.isEmpty() && actualMap.isEmpty()) { + field(field, "same [empty map]"); + return; + } + field(field, null); + indent += 1; + compareMaps(actualMap, expectedMap); + indent -= 1; + return; + } + if (expected instanceof List) { + if (actual == null) { + field(field, "expected list but not found"); + return; + } + if (false == actual instanceof List) { + field(field, "expected list but found [" + actual + "]"); + return; + } + @SuppressWarnings("unchecked") + List expectedList = (List) expected; + @SuppressWarnings("unchecked") + List actualList = (List) actual; + if (expectedList.isEmpty() && actualList.isEmpty()) { + field(field, "same [empty list]"); + return; + } + field(field, null); + indent += 1; + compareLists(actualList, expectedList); + indent -= 1; + return; + } + if (actual == null) { + field(field, "expected [" + expected + "] but not found"); + return; + } + if (Objects.equals(expected, actual)) { + field(field, "same [" + expected + "]"); + return; + } + field(field, "expected [" + expected + "] but was [" + actual + "]"); + } + + private void indent() { + for (int i = 0; i < indent; i++) { + message.append(" "); + } + } + + private void field(Object name, String info) { + indent(); + message.append(String.format(Locale.ROOT, "%30s: ", name)); + if (info != null) { + message.append(info); + } + message.append('\n'); + } } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java deleted file mode 100644 index 3ead65a2111..00000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/section/ResponseBodyAssertion.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.section; - -import java.io.IOException; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.TreeMap; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.test.rest.parser.RestTestFragmentParser; -import org.elasticsearch.test.rest.parser.RestTestParseException; -import org.elasticsearch.test.rest.parser.RestTestSuiteParseContext; - -/** - * Checks that the response body matches some text. - */ -public class ResponseBodyAssertion extends Assertion { - public static final RestTestFragmentParser PARSER = new RestTestFragmentParser() { - @Override - public ResponseBodyAssertion parse(RestTestSuiteParseContext parseContext) throws IOException, RestTestParseException { - try (XContentParser parser = JsonXContent.jsonXContent.createParser(parseContext.parseField())) { - return new ResponseBodyAssertion("$body", parser.map()); - } - } - }; - - private ResponseBodyAssertion(String field, Map expectedValue) { - super(field, expectedValue); - } - - @Override - protected void doAssert(Object actualValue, Object expectedValue) { - if (false == expectedValue.equals(actualValue)) { - @SuppressWarnings("unchecked") - Map actual = (Map) actualValue; - @SuppressWarnings("unchecked") - Map expected = (Map) expectedValue; - FailureMessage message = new FailureMessage(); - message.compareMaps(actual, expected); - throw new AssertionError(message.message); - } - } - - private class FailureMessage { - private final StringBuilder message = new StringBuilder("body didn't match the expected value:\n"); - private int indent = 0; - - private void compareMaps(Map actual, Map expected) { - actual = new TreeMap<>(actual); - expected = new TreeMap<>(expected); - for (Map.Entry expectedEntry : expected.entrySet()) { - compare(expectedEntry.getKey(), actual.remove(expectedEntry.getKey()), expectedEntry.getValue()); - } - for (Map.Entry unmatchedEntry : actual.entrySet()) { - field(unmatchedEntry.getKey(), "unexpected but found [" + unmatchedEntry.getValue() + "]"); - } - } - - private void compareLists(List actual, List expected) { - int i = 0; - while (i < actual.size() && i < expected.size()) { - compare(i, actual.get(i), expected.get(i)); - i++; - } - if (actual.size() == expected.size()) { - return; - } - indent(); - if (actual.size() < expected.size()) { - message.append("expected [").append(expected.size() - i).append("] more entries\n"); - return; - } - message.append("received [").append(actual.size() - i).append("] more entries than expected\n"); - } - - private void compare(Object field, @Nullable Object actual, Object expected) { - if (expected instanceof Map) { - if (actual == null) { - field(field, "expected map but not found"); - return; - } - if (false == actual instanceof Map) { - field(field, "expected map but found [" + actual + "]"); - return; - } - @SuppressWarnings("unchecked") - Map expectedMap = (Map) expected; - @SuppressWarnings("unchecked") - Map actualMap = (Map) actual; - if (expectedMap.isEmpty() && actualMap.isEmpty()) { - field(field, "same [empty map]"); - return; - } - field(field, null); - indent += 1; - compareMaps(actualMap, expectedMap); - indent -= 1; - return; - } - if (expected instanceof List) { - if (actual == null) { - field(field, "expected list but not found"); - return; - } - if (false == actual instanceof List) { - field(field, "expected list but found [" + actual + "]"); - return; - } - @SuppressWarnings("unchecked") - List expectedList = (List) expected; - @SuppressWarnings("unchecked") - List actualList = (List) actual; - if (expectedList.isEmpty() && actualList.isEmpty()) { - field(field, "same [empty list]"); - return; - } - field(field, null); - indent += 1; - compareLists(actualList, expectedList); - indent -= 1; - return; - } - if (actual == null) { - field(field, "expected [" + expected + "] but not found"); - return; - } - if (Objects.equals(expected, actual)) { - field(field, "same [" + expected + "]"); - return; - } - field(field, "expected [" + expected + "] but was [" + actual + "]"); - } - - private void indent() { - for (int i = 0; i < indent; i++) { - message.append(" "); - } - } - - private void field(Object name, String info) { - indent(); - message.append(String.format(Locale.ROOT, "%30s: ", name)); - if (info != null) { - message.append(info); - } - message.append('\n'); - } - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java index 66ba1528b90..cf33570ef28 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/Features.java @@ -34,7 +34,7 @@ import java.util.List; */ public final class Features { - private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "yaml"); + private static final List SUPPORTED = Arrays.asList("stash_in_path", "groovy_scripting", "headers", "embedded_stash_key"); private Features() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java index 69acae55fdc..b32308f8cd8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/support/FileUtils.java @@ -78,7 +78,8 @@ public final class FileUtils { * Each input path can either be a single file (the .yaml suffix is optional) or a directory. * Each path is looked up in the classpath, or optionally from {@code fileSystem} if its not null. */ - public static Map> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths) throws IOException { + public static Map> findYamlSuites(FileSystem fileSystem, String optionalPathPrefix, final String... paths) + throws IOException { Map> yamlSuites = new HashMap<>(); for (String path : paths) { collectFiles(resolveFile(fileSystem, optionalPathPrefix, path, YAML_SUFFIX), YAML_SUFFIX, yamlSuites); @@ -86,7 +87,8 @@ public final class FileUtils { return yamlSuites; } - private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix) throws IOException { + private static Path resolveFile(FileSystem fileSystem, String optionalPathPrefix, String path, String optionalFileSuffix) + throws IOException { if (fileSystem != null) { Path file = findFile(fileSystem, path, optionalFileSuffix); if (!lenientExists(file)) { @@ -94,7 +96,8 @@ public final class FileUtils { String newPath = optionalPathPrefix + "/" + path; file = findFile(fileSystem, newPath, optionalFileSuffix); if (!lenientExists(file)) { - throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: " + optionalFileSuffix); + throw new NoSuchFileException("path prefix: " + optionalPathPrefix + ", path: " + path + ", file suffix: " + + optionalFileSuffix); } } return file; diff --git a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java index 7b1c3fd936b..1b99d2f32cb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/store/MockFSDirectoryService.java @@ -131,11 +131,11 @@ public class MockFSDirectoryService extends FsDirectoryService { ESTestCase.checkIndexFailed = true; logger.warn("check index [failure] index files={}\n{}", Arrays.toString(dir.listAll()), - new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); + os.bytes().utf8ToString()); throw new IOException("index check failure"); } else { if (logger.isDebugEnabled()) { - logger.debug("check index [success]\n{}", new String(os.bytes().toBytes(), StandardCharsets.UTF_8)); + logger.debug("check index [success]\n{}", os.bytes().utf8ToString()); } } } catch (LockObtainFailedException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java index 654a1c971ca..94c30f496dd 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/CapturingTransport.java @@ -51,11 +51,11 @@ public class CapturingTransport implements Transport { private TransportServiceAdapter adapter; - static public class CapturedRequest { - final public DiscoveryNode node; - final public long requestId; - final public String action; - final public TransportRequest request; + public static class CapturedRequest { + public final DiscoveryNode node; + public final long requestId; + public final String action; + public final TransportRequest request; public CapturedRequest(DiscoveryNode node, long requestId, String action, TransportRequest request) { this.node = node; diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index 7688e2842e6..7bc7f0d5254 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -383,7 +383,7 @@ public class MockTransportService extends TransportService { BytesStreamOutput bStream = new BytesStreamOutput(); request.writeTo(bStream); final TransportRequest clonedRequest = reg.newRequest(); - clonedRequest.readFrom(StreamInput.wrap(bStream.bytes())); + clonedRequest.readFrom(bStream.bytes().streamInput()); threadPool.schedule(delay, ThreadPool.Names.GENERIC, new AbstractRunnable() { @Override diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java index 128cb862e57..db41c42e16a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/FileUtilsTests.java @@ -84,7 +84,8 @@ public class FileUtilsTests extends ESTestCase { assertSingleFile(yamlSuites.get(dir.getFileName().toString()), dir.getFileName().toString(), file.getFileName().toString()); //load from external file (optional extension) - yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test", dir.resolve("test_loading").toAbsolutePath().toString()); + yamlSuites = FileUtils.findYamlSuites(dir.getFileSystem(), "/rest-api-spec/test", + dir.resolve("test_loading").toAbsolutePath().toString()); assertThat(yamlSuites, notNullValue()); assertThat(yamlSuites.size(), equalTo(1)); assertThat(yamlSuites.containsKey(dir.getFileName().toString()), equalTo(true)); diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java deleted file mode 100644 index fefcd57af79..00000000000 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/JsonPathTests.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.test; - -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.Stash; -import org.elasticsearch.test.rest.json.JsonPath; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; - -public class JsonPathTests extends ESTestCase { - public void testEvaluateObjectPathEscape() throws Exception { - String json = "{ \"field1\": { \"field2.field3\" : \"value2\" } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.field2\\.field3"); - assertThat(object, instanceOf(String.class)); - assertThat((String)object, equalTo("value2")); - } - - public void testEvaluateObjectPathWithDoubleDot() throws Exception { - String json = "{ \"field1\": { \"field2\" : \"value2\" } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1..field2"); - assertThat(object, instanceOf(String.class)); - assertThat((String)object, equalTo("value2")); - } - - public void testEvaluateObjectPathEndsWithDot() throws Exception { - String json = "{ \"field1\": { \"field2\" : \"value2\" } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.field2."); - assertThat(object, instanceOf(String.class)); - assertThat((String)object, equalTo("value2")); - } - - public void testEvaluateString() throws Exception { - String json = "{ \"field1\": { \"field2\" : \"value2\" } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.field2"); - assertThat(object, instanceOf(String.class)); - assertThat((String)object, equalTo("value2")); - } - - public void testEvaluateInteger() throws Exception { - String json = "{ \"field1\": { \"field2\" : 333 } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.field2"); - assertThat(object, instanceOf(Integer.class)); - assertThat((Integer)object, equalTo(333)); - } - - public void testEvaluateDouble() throws Exception { - String json = "{ \"field1\": { \"field2\" : 3.55 } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.field2"); - assertThat(object, instanceOf(Double.class)); - assertThat((Double)object, equalTo(3.55)); - } - - public void testEvaluateArray() throws Exception { - String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.array1"); - assertThat(object, instanceOf(List.class)); - List list = (List) object; - assertThat(list.size(), equalTo(2)); - assertThat(list.get(0), instanceOf(String.class)); - assertThat((String)list.get(0), equalTo("value1")); - assertThat(list.get(1), instanceOf(String.class)); - assertThat((String)list.get(1), equalTo("value2")); - } - - public void testEvaluateArrayElement() throws Exception { - String json = "{ \"field1\": { \"array1\" : [ \"value1\", \"value2\" ] } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.array1.1"); - assertThat(object, instanceOf(String.class)); - assertThat((String)object, equalTo("value2")); - } - - public void testEvaluateArrayElementObject() throws Exception { - String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.array1.1.element"); - assertThat(object, instanceOf(String.class)); - assertThat((String)object, equalTo("value2")); - } - - public void testEvaluateArrayElementObjectWrongPath() throws Exception { - String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("field1.array2.1.element"); - assertThat(object, nullValue()); - } - - @SuppressWarnings("unchecked") - public void testEvaluateObjectKeys() throws Exception { - String json = "{ \"metadata\": { \"templates\" : {\"template_1\": { \"field\" : \"value\"}, \"template_2\": { \"field\" : \"value\"} } } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate("metadata.templates"); - assertThat(object, instanceOf(Map.class)); - Map map = (Map)object; - assertThat(map.size(), equalTo(2)); - Set strings = map.keySet(); - assertThat(strings, contains("template_1", "template_2")); - } - - @SuppressWarnings("unchecked") - public void testEvaluateEmptyPath() throws Exception { - String json = "{ \"field1\": { \"array1\" : [ {\"element\": \"value1\"}, {\"element\":\"value2\"} ] } }"; - JsonPath jsonPath = new JsonPath(json); - Object object = jsonPath.evaluate(""); - assertThat(object, notNullValue()); - assertThat(object, instanceOf(Map.class)); - assertThat(((Map)object).containsKey("field1"), equalTo(true)); - } - - public void testEvaluateStashInPropertyName() throws Exception { - String json = "{ \"field1\": { \"elements\" : {\"element1\": \"value1\"}}}"; - JsonPath jsonPath = new JsonPath(json); - try { - jsonPath.evaluate("field1.$placeholder.element1"); - fail("evaluate should have failed due to unresolved placeholder"); - } catch(IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("stashed value not found for key [$placeholder]")); - } - - Stash stash = new Stash(); - stash.stashValue("placeholder", "elements"); - Object object = jsonPath.evaluate("field1.$placeholder.element1", stash); - assertThat(object, notNullValue()); - assertThat(object.toString(), equalTo("value1")); - } -} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java new file mode 100644 index 00000000000..1d99a73c767 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/ObjectPathTests.java @@ -0,0 +1,265 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test.rest.test; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.test.rest.Stash; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class ObjectPathTests extends ESTestCase { + + private static XContentBuilder randomXContentBuilder() throws IOException { + //only string based formats are supported, no cbor nor smile + XContentType xContentType = randomFrom(XContentType.JSON, XContentType.YAML); + return XContentBuilder.builder(XContentFactory.xContent(xContentType)); + } + + public void testEvaluateObjectPathEscape() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.field("field2.field3", "value2"); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("field1.field2\\.field3"); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value2")); + } + + public void testEvaluateObjectPathWithDots() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.field("field2", "value2"); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("field1..field2"); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value2")); + object = objectPath.evaluate("field1.field2."); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value2")); + object = objectPath.evaluate("field1.field2"); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value2")); + } + + public void testEvaluateInteger() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.field("field2", 333); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("field1.field2"); + assertThat(object, instanceOf(Integer.class)); + assertThat(object, equalTo(333)); + } + + public void testEvaluateDouble() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.field("field2", 3.55); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("field1.field2"); + assertThat(object, instanceOf(Double.class)); + assertThat(object, equalTo(3.55)); + } + + public void testEvaluateArray() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.array("array1", "value1", "value2"); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("field1.array1"); + assertThat(object, instanceOf(List.class)); + List list = (List) object; + assertThat(list.size(), equalTo(2)); + assertThat(list.get(0), instanceOf(String.class)); + assertThat(list.get(0), equalTo("value1")); + assertThat(list.get(1), instanceOf(String.class)); + assertThat(list.get(1), equalTo("value2")); + object = objectPath.evaluate("field1.array1.1"); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value2")); + } + + @SuppressWarnings("unchecked") + public void testEvaluateArrayElementObject() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.startArray("array1"); + xContentBuilder.startObject(); + xContentBuilder.field("element", "value1"); + xContentBuilder.endObject(); + xContentBuilder.startObject(); + xContentBuilder.field("element", "value2"); + xContentBuilder.endObject(); + xContentBuilder.endArray(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("field1.array1.1.element"); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("value2")); + object = objectPath.evaluate(""); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Map.class)); + assertThat(((Map)object).containsKey("field1"), equalTo(true)); + object = objectPath.evaluate("field1.array2.1.element"); + assertThat(object, nullValue()); + } + + @SuppressWarnings("unchecked") + public void testEvaluateObjectKeys() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("metadata"); + xContentBuilder.startObject("templates"); + xContentBuilder.startObject("template_1"); + xContentBuilder.field("field", "value"); + xContentBuilder.endObject(); + xContentBuilder.startObject("template_2"); + xContentBuilder.field("field", "value"); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + Object object = objectPath.evaluate("metadata.templates"); + assertThat(object, instanceOf(Map.class)); + Map map = (Map)object; + assertThat(map.size(), equalTo(2)); + Set strings = map.keySet(); + assertThat(strings, contains("template_1", "template_2")); + } + + public void testEvaluateStashInPropertyName() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startObject(); + xContentBuilder.startObject("field1"); + xContentBuilder.startObject("elements"); + xContentBuilder.field("element1", "value1"); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + xContentBuilder.endObject(); + ObjectPath objectPath = ObjectPath.createFromXContent(xContentBuilder.contentType().xContent(), xContentBuilder.string()); + try { + objectPath.evaluate("field1.$placeholder.element1"); + fail("evaluate should have failed due to unresolved placeholder"); + } catch(IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("stashed value not found for key [placeholder]")); + } + + // Stashed value is whole property name + Stash stash = new Stash(); + stash.stashValue("placeholder", "elements"); + Object object = objectPath.evaluate("field1.$placeholder.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Stash key has dots + Map stashedObject = new HashMap<>(); + stashedObject.put("subobject", "elements"); + stash.stashValue("object", stashedObject); + object = objectPath.evaluate("field1.$object\\.subobject.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Stashed value is part of property name + stash.stashValue("placeholder", "ele"); + object = objectPath.evaluate("field1.${placeholder}ments.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Stashed value is inside of property name + stash.stashValue("placeholder", "le"); + object = objectPath.evaluate("field1.e${placeholder}ments.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Multiple stashed values in property name + stash.stashValue("placeholder", "le"); + stash.stashValue("placeholder2", "nts"); + object = objectPath.evaluate("field1.e${placeholder}me${placeholder2}.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + + // Stashed value is part of property name and has dots + stashedObject.put("subobject", "ele"); + stash.stashValue("object", stashedObject); + object = objectPath.evaluate("field1.${object\\.subobject}ments.element1", stash); + assertThat(object, notNullValue()); + assertThat(object.toString(), equalTo("value1")); + } + + @SuppressWarnings("unchecked") + public void testEvaluateArrayAsRoot() throws Exception { + XContentBuilder xContentBuilder = randomXContentBuilder(); + xContentBuilder.startArray(); + xContentBuilder.startObject(); + xContentBuilder.field("alias", "test_alias1"); + xContentBuilder.field("index", "test1"); + xContentBuilder.endObject(); + xContentBuilder.startObject(); + xContentBuilder.field("alias", "test_alias2"); + xContentBuilder.field("index", "test2"); + xContentBuilder.endObject(); + xContentBuilder.endArray(); + ObjectPath objectPath = ObjectPath.createFromXContent(XContentFactory.xContent(XContentType.YAML), xContentBuilder.string()); + Object object = objectPath.evaluate(""); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(List.class)); + assertThat(((List)object).size(), equalTo(2)); + object = objectPath.evaluate("0"); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(Map.class)); + assertThat(((Map)object).get("alias"), equalTo("test_alias1")); + object = objectPath.evaluate("1.index"); + assertThat(object, notNullValue()); + assertThat(object, instanceOf(String.class)); + assertThat(object, equalTo("test2")); + } +} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java index 76fbe8307e6..6b5cc3defb7 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/test/RestTestParserTests.java @@ -115,11 +115,12 @@ public class RestTestParserTests extends ESTestCase { assertThat(restTestSuite.getSetupSection().getDoSections().size(), equalTo(1)); assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.create")); assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1)); - assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), equalTo("test_index")); + assertThat(restTestSuite.getSetupSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), + equalTo("test_index")); } else { assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true)); } - + assertThat(restTestSuite.getTeardownSection(), notNullValue()); if (includeTeardown) { assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false)); @@ -154,7 +155,8 @@ public class RestTestParserTests extends ESTestCase { assertThat(restTestSuite.getTestSections().get(1).getName(), equalTo("Get type mapping - pre 1.0")); assertThat(restTestSuite.getTestSections().get(1).getSkipSection().isEmpty(), equalTo(false)); - assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(), equalTo("for newer versions the index name is always returned")); + assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getReason(), + equalTo("for newer versions the index name is always returned")); assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0)); assertThat(restTestSuite.getTestSections().get(1).getSkipSection().getUpperVersion(), equalTo(Version.CURRENT)); assertThat(restTestSuite.getTestSections().get(1).getExecutableSections().size(), equalTo(3)); diff --git a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java index 63c5eb01fb0..7253c4d26e7 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java @@ -90,7 +90,7 @@ public class InternalTestClusterTests extends ESTestCase { * a set of settings that are expected to have different values betweem clusters, even they have been initialized with the same * base settins. */ - final static Set clusterUniqueSettings = new HashSet<>(); + static final Set clusterUniqueSettings = new HashSet<>(); static { clusterUniqueSettings.add(ClusterName.CLUSTER_NAME_SETTING.getKey());