honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/query-refactoring

This commit is contained in:
Christoph Büscher 2015-07-29 10:51:08 +02:00
parent f6ca11db30 5d7ed70fa0
commit a1948350a5
11 changed files with 130 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java
View File

@ -234,7 +234,9 @@ public class AwarenessAllocationDecider extends AllocationDecider {
int currentNodeCount = shardPerAttribute.get(node.node().attributes().get(awarenessAttribute));
// if we are above with leftover, then we know we are not good, even with mod
if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) {
return allocation.decision(Decision.NO, NAME, "too many shards on nodes for attribute: [%s]", awarenessAttribute);
return allocation.decision(Decision.NO, NAME,
"too many shards on node for attribute: [%s], required per attribute: [%d], node count: [%d], leftover: [%d]",
awarenessAttribute, requiredCountPerAttribute, currentNodeCount, leftoverPerAttribute);
}
// all is well, we are below or same as average
if (currentNodeCount <= requiredCountPerAttribute) {

16
core/src/main/java/org/elasticsearch/common/lucene/all/AllTermQuery.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.payloads.AveragePayloadFunction;
@ -124,14 +125,23 @@ public final class AllTermQuery extends PayloadTermQuery {
@Override
public Query rewrite(IndexReader reader) throws IOException {
boolean fieldExists = false;
boolean hasPayloads = false;
for (LeafReaderContext context : reader.leaves()) {
final Terms terms = context.reader().terms(term.field());
if (terms.hasPayloads()) {
hasPayloads = true;
break;
if (terms != null) {
fieldExists = true;
if (terms.hasPayloads()) {
hasPayloads = true;
break;
}
}
}
if (fieldExists == false) {
Query rewritten = new MatchNoDocsQuery();
rewritten.setBoost(getBoost());
return rewritten;
}
if (hasPayloads == false) {
TermQuery rewritten = new TermQuery(term);
rewritten.setBoost(getBoost());

4
core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View File

@ -72,8 +72,8 @@ public class DateFieldMapper extends NumberFieldMapper {
public static final String CONTENT_TYPE = "date";
public static class Defaults extends NumberFieldMapper.Defaults {
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime||epoch_millis", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER_BEFORE_2_0 = Joda.forPattern("dateOptionalTime", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strict_date_optional_time||epoch_millis", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER_BEFORE_2_0 = Joda.forPattern("date_optional_time", Locale.ROOT);
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final DateFieldType FIELD_TYPE = new DateFieldType();

10
core/src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/hdr/AbstractInternalHDRPercentiles.java
View File

@ -89,7 +89,10 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
keys[i] = in.readDouble();
}
long minBarForHighestToLowestValueRatio = in.readLong();
ByteBuffer stateBuffer = ByteBuffer.wrap(in.readByteArray());
final int serializedLen = in.readVInt();
byte[] bytes = new byte[serializedLen];
in.readBytes(bytes, 0, serializedLen);
ByteBuffer stateBuffer = ByteBuffer.wrap(bytes);
try {
state = DoubleHistogram.decodeFromCompressedByteBuffer(stateBuffer, minBarForHighestToLowestValueRatio);
} catch (DataFormatException e) {
@ -107,8 +110,9 @@ abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggr
}
out.writeLong(state.getHighestToLowestValueRatio());
ByteBuffer stateBuffer = ByteBuffer.allocate(state.getNeededByteBufferCapacity());
state.encodeIntoCompressedByteBuffer(stateBuffer);
out.writeByteArray(stateBuffer.array());
final int serializedLen = state.encodeIntoCompressedByteBuffer(stateBuffer);
out.writeVInt(serializedLen);
out.writeBytes(stateBuffer.array(), 0, serializedLen);
out.writeBoolean(keyed);
}

3
core/src/main/java/org/elasticsearch/search/aggregations/metrics/tophits/TopHitsAggregator.java
View File

@ -117,6 +117,9 @@ public class TopHitsAggregator extends MetricsAggregator {
if (collectors == null) {
Sort sort = subSearchContext.sort();
int topN = subSearchContext.from() + subSearchContext.size();
// In the QueryPhase we don't need this protection, because it is build into the IndexSearcher,
// but here we create collectors ourselves and we need prevent OOM because of crazy an offset and size.
topN = Math.min(topN, subSearchContext.searcher().getIndexReader().maxDoc());
TopDocsCollector<?> topLevelCollector = sort != null ? TopFieldCollector.create(sort, topN, true, subSearchContext.trackScores(), subSearchContext.trackScores()) : TopScoreDocCollector.create(topN);
collectors = new TopDocsAndLeafCollector(topLevelCollector);
collectors.leafCollector = collectors.topLevelCollector.getLeafCollector(ctx);

2
core/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
View File

@ -176,7 +176,7 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
@Override
public void search(List<LeafReaderContext> leaves, Weight weight, Collector collector) throws IOException {
final boolean timeoutSet = searchContext.timeoutInMillis() != -1;
final boolean timeoutSet = searchContext.timeoutInMillis() != SearchService.NO_TIMEOUT.millis();
final boolean terminateAfterSet = searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER;
try {
if (timeoutSet || terminateAfterSet) {

10
core/src/test/java/org/elasticsearch/index/mapper/date/SimpleDateMappingTests.java
View File

@ -495,7 +495,7 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
Version randomVersion = VersionUtils.randomVersionBetween(getRandom(), Version.V_0_90_0, Version.V_1_6_1);
IndexService index = createIndex("test", settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, randomVersion).build());
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
assertDateFormat("epoch_millis||dateOptionalTime");
assertDateFormat("epoch_millis||date_optional_time");
DocumentMapper defaultMapper = index.mapperService().documentMapper("type");
defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
@ -543,13 +543,13 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
public void testThatUpgradingAnOlderIndexToStrictDateWorks() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("date_field").field("type", "date").field("format", "dateOptionalTime").endObject().endObject()
.startObject("properties").startObject("date_field").field("type", "date").field("format", "date_optional_time").endObject().endObject()
.endObject().endObject().string();
Version randomVersion = VersionUtils.randomVersionBetween(getRandom(), Version.V_0_90_0, Version.V_1_6_1);
createIndex("test", settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, randomVersion).build());
client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
assertDateFormat("epoch_millis||dateOptionalTime");
assertDateFormat("epoch_millis||date_optional_time");
// index doc
client().prepareIndex("test", "type", "1").setSource(XContentFactory.jsonBuilder()
@ -561,12 +561,12 @@ public class SimpleDateMappingTests extends ElasticsearchSingleNodeTest {
String newMapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("properties").startObject("date_field")
.field("type", "date")
.field("format", "strictDateOptionalTime||epoch_millis")
.field("format", "strict_date_optional_time||epoch_millis")
.endObject().endObject().endObject().endObject().string();
PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(newMapping).get();
assertThat(putMappingResponse.isAcknowledged(), is(true));
assertDateFormat("strictDateOptionalTime||epoch_millis");
assertDateFormat("strict_date_optional_time||epoch_millis");
}
private void assertDateFormat(String expectedFormat) throws IOException {

17
core/src/test/java/org/elasticsearch/search/aggregations/bucket/TopHitsTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.aggregations.bucket;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchPhaseExecutionException;
import org.elasticsearch.action.search.SearchResponse;
@ -928,4 +929,20 @@ public class TopHitsTests extends ElasticsearchIntegrationTest {
}
}
}
@Test
public void testDontExplode() throws Exception {
SearchResponse response = client()
.prepareSearch("idx")
.setTypes("type")
.addAggregation(terms("terms")
.executionHint(randomExecutionHint())
.field(TERMS_AGGS_FIELD)
.subAggregation(
topHits("hits").setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1).addSort(SortBuilders.fieldSort(SORT_FIELD).order(SortOrder.DESC))
)
)
.get();
assertNoFailures(response);
}
}

18
core/src/test/java/org/elasticsearch/search/query/SearchQueryTests.java
View File

@ -1853,6 +1853,24 @@ public class SearchQueryTests extends ElasticsearchIntegrationTest {
assertHitCount(searchResponse, 1l);
}
@Test
public void testAllFieldEmptyMapping() throws Exception {
client().prepareIndex("myindex", "mytype").setId("1").setSource("{}").setRefresh(true).get();
SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
assertNoFailures(response);
}
@Test
public void testAllDisabledButQueried() throws Exception {
createIndex("myindex");
assertAcked(client().admin().indices().preparePutMapping("myindex").setType("mytype").setSource(
jsonBuilder().startObject().startObject("mytype").startObject("_all").field("enabled", false)));
client().prepareIndex("myindex", "mytype").setId("1").setSource("bar", "foo").setRefresh(true).get();
SearchResponse response = client().prepareSearch("myindex").setQuery(matchQuery("_all", "foo")).get();
assertNoFailures(response);
assertHitCount(response, 0);
}
@Test
public void testIndicesQuery() throws Exception {
createIndex("index1", "index2", "index3");

1
docs/java-api/docs/bulk.asciidoc
View File

@ -50,6 +50,7 @@ To use it, first create a `BulkProcessor` instance:
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
BulkProcessor bulkProcessor = BulkProcessor.builder(
client, <1>

60
plugins/cloud-azure/src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java
View File

@ -173,6 +173,66 @@ public class AzureSnapshotRestoreITest extends AbstractAzureTest {
assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
}
/**
* For issue #51: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/51
*/
@Test
public void testMultipleSnapshots() throws URISyntaxException, StorageException {
final String indexName = "test-idx-1";
final String typeName = "doc";
final String repositoryName = "test-repo";
final String snapshot1Name = "test-snap-1";
final String snapshot2Name = "test-snap-2";
Client client = client();
logger.info("creating index [{}]", indexName);
createIndex(indexName);
ensureGreen();
logger.info("indexing first document");
index(indexName, typeName, Integer.toString(1), "foo", "bar " + Integer.toString(1));
refresh();
assertThat(client.prepareCount(indexName).get().getCount(), equalTo(1L));
logger.info("creating Azure repository with path [{}]", getRepositoryPath());
PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository(repositoryName)
.setType("azure").setSettings(Settings.settingsBuilder()
.put(Repository.CONTAINER, getContainerName())
.put(Repository.BASE_PATH, getRepositoryPath())
.put(Repository.BASE_PATH, randomIntBetween(1000, 10000), ByteSizeUnit.BYTES)
).get();
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
logger.info("creating snapshot [{}]", snapshot1Name);
CreateSnapshotResponse createSnapshotResponse1 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).setIndices(indexName).get();
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse1.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse1.getSnapshotInfo().totalShards()));
assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot1Name).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
logger.info("indexing second document");
index(indexName, typeName, Integer.toString(2), "foo", "bar " + Integer.toString(2));
refresh();
assertThat(client.prepareCount(indexName).get().getCount(), equalTo(2L));
logger.info("creating snapshot [{}]", snapshot2Name);
CreateSnapshotResponse createSnapshotResponse2 = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshot2Name).setWaitForCompletion(true).setIndices(indexName).get();
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse2.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse2.getSnapshotInfo().totalShards()));
assertThat(client.admin().cluster().prepareGetSnapshots(repositoryName).setSnapshots(snapshot2Name).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
logger.info("closing index [{}]", indexName);
client.admin().indices().prepareClose(indexName).get();
logger.info("attempting restore from snapshot [{}]", snapshot1Name);
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot(repositoryName, snapshot1Name).setWaitForCompletion(true).execute().actionGet();
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
ensureGreen();
assertThat(client.prepareCount(indexName).get().getCount(), equalTo(1L));
}
@Test
public void testMultipleRepositories() {
Client client = client();