honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade to Lucene 3.5, closes #1502.

This commit is contained in:
Shay Banon 2011-11-27 12:57:26 +02:00
parent fd5d754fe6
commit f18ad903a9
20 changed files with 121 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.idea/libraries/lucene.xml
View File

@ -1,19 +1,19 @@
<component name="libraryTable">
<library name="lucene">
<CLASSES>
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-highlighter/jars/lucene-highlighter-3.4.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-memory/jars/lucene-memory-3.4.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-core/jars/lucene-core-3.4.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-analyzers/jars/lucene-analyzers-3.4.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-queries/jars/lucene-queries-3.4.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-highlighter/jars/lucene-highlighter-3.5.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-memory/jars/lucene-memory-3.5.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-core/jars/lucene-core-3.5.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-analyzers/jars/lucene-analyzers-3.5.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-queries/jars/lucene-queries-3.5.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>
<root url="file://$USER_HOME$/opt/lucene/3.4.0.src/contrib/memory/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.4.0.src/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.4.0.src/contrib/highlighter/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.4.0.src/contrib/queries/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.4.0.src/contrib/analyzers/common/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.5.0.src/contrib/memory/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.5.0.src/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.5.0.src/contrib/highlighter/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.5.0.src/contrib/queries/src/java" />
<root url="file://$USER_HOME$/opt/lucene/3.5.0.src/contrib/analyzers/common/src/java" />
</SOURCES>
</library>
</component>

2
.idea/modules/plugin-analysis-icu.iml
View File

@ -19,7 +19,7 @@
<library name="lucene-icu">
<CLASSES>
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-icu4j/jars/lucene-icu4j-3.2.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-icu/jars/lucene-icu-3.4.0.jar!/" />
<root url="jar://$GRADLE_REPOSITORY$/org.apache.lucene/lucene-icu/jars/lucene-icu-3.5.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES>

10
modules/elasticsearch/build.gradle
View File

@ -38,11 +38,11 @@ dependencies {
compile('net.java.dev.jna:jna:3.2.7') { transitive = false }
compile('org.apache.lucene:lucene-core:3.4.0') { transitive = false }
compile('org.apache.lucene:lucene-analyzers:3.4.0') { transitive = false }
compile('org.apache.lucene:lucene-queries:3.4.0') { transitive = false }
compile('org.apache.lucene:lucene-memory:3.4.0') { transitive = false }
compile('org.apache.lucene:lucene-highlighter:3.4.0') { transitive = false }
compile('org.apache.lucene:lucene-core:3.5.0') { transitive = false }
compile('org.apache.lucene:lucene-analyzers:3.5.0') { transitive = false }
compile('org.apache.lucene:lucene-queries:3.5.0') { transitive = false }
compile('org.apache.lucene:lucene-memory:3.5.0') { transitive = false }
compile('org.apache.lucene:lucene-highlighter:3.5.0') { transitive = false }
}
configurations {

34
modules/elasticsearch/src/main/java/org/apache/lucene/search/OpenFilterClause.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.search;
/**
* @author kimchy (shay.banon)
*/
public class OpenFilterClause extends FilterClause {
public OpenFilterClause(Filter filter, BooleanClause.Occur occur) {
super(filter, occur);
}
public void setFilter(Filter filter) {
this.filter = filter;
}
}

72
modules/elasticsearch/src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java
View File

@ -21,7 +21,14 @@ package org.apache.lucene.search.vectorhighlight;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.MultiTermQueryWrapperFilter;
import org.apache.lucene.search.PublicTermsFilter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
import org.elasticsearch.common.lucene.search.TermFilter;
@ -50,26 +57,22 @@ public class CustomFieldQuery extends FieldQuery {
}
}
// hack since flatten is called from the parent constructor, so we can't pass it
public static final ThreadLocal<IndexReader> reader = new ThreadLocal<IndexReader>();
public static final ThreadLocal<Boolean> highlightFilters = new ThreadLocal<Boolean>();
public CustomFieldQuery(Query query, FastVectorHighlighter highlighter) {
this(query, highlighter.isPhraseHighlight(), highlighter.isFieldMatch());
public CustomFieldQuery(Query query, IndexReader reader, FastVectorHighlighter highlighter) throws IOException {
this(query, reader, highlighter.isPhraseHighlight(), highlighter.isFieldMatch());
}
public CustomFieldQuery(Query query, boolean phraseHighlight, boolean fieldMatch) {
super(query, phraseHighlight, fieldMatch);
reader.remove();
public CustomFieldQuery(Query query, IndexReader reader, boolean phraseHighlight, boolean fieldMatch) throws IOException {
super(query, reader, phraseHighlight, fieldMatch);
highlightFilters.remove();
}
@Override void flatten(Query sourceQuery, Collection<Query> flatQueries) {
@Override void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQueries) throws IOException {
if (sourceQuery instanceof DisjunctionMaxQuery) {
DisjunctionMaxQuery dmq = (DisjunctionMaxQuery) sourceQuery;
for (Query query : dmq) {
flatten(query, flatQueries);
flatten(query, reader, flatQueries);
}
} else if (sourceQuery instanceof SpanTermQuery) {
TermQuery termQuery = new TermQuery(((SpanTermQuery) sourceQuery).getTerm());
@ -79,63 +82,44 @@ public class CustomFieldQuery extends FieldQuery {
} else if (sourceQuery instanceof ConstantScoreQuery) {
ConstantScoreQuery constantScoreQuery = (ConstantScoreQuery) sourceQuery;
if (constantScoreQuery.getFilter() != null) {
flatten(constantScoreQuery.getFilter(), flatQueries);
flatten(constantScoreQuery.getFilter(), reader, flatQueries);
} else {
flatten(constantScoreQuery.getQuery(), flatQueries);
flatten(constantScoreQuery.getQuery(), reader, flatQueries);
}
} else if (sourceQuery instanceof DeletionAwareConstantScoreQuery) {
flatten(((DeletionAwareConstantScoreQuery) sourceQuery).getFilter(), flatQueries);
} else if (sourceQuery instanceof FunctionScoreQuery) {
flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), flatQueries);
} else if (sourceQuery instanceof MultiTermQuery) {
MultiTermQuery multiTermQuery = (MultiTermQuery) sourceQuery;
MultiTermQuery.RewriteMethod rewriteMethod = multiTermQuery.getRewriteMethod();
// we want to rewrite a multi term query to extract the terms out of it
// LUCENE MONITOR: The regular Highlighter actually uses MemoryIndex to extract the terms
multiTermQuery.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
try {
flatten(multiTermQuery.rewrite(reader.get()), flatQueries);
} catch (IOException e) {
// ignore
} catch (BooleanQuery.TooManyClauses e) {
// ignore
} finally {
multiTermQuery.setRewriteMethod(rewriteMethod);
}
flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries);
} else if (sourceQuery instanceof FilteredQuery) {
flatten(((FilteredQuery) sourceQuery).getQuery(), flatQueries);
flatten(((FilteredQuery) sourceQuery).getFilter(), flatQueries);
flatten(((FilteredQuery) sourceQuery).getQuery(), reader, flatQueries);
flatten(((FilteredQuery) sourceQuery).getFilter(), reader, flatQueries);
} else if (sourceQuery instanceof MultiPhrasePrefixQuery) {
try {
flatten(sourceQuery.rewrite(reader.get()), flatQueries);
flatten(sourceQuery.rewrite(reader), reader, flatQueries);
} catch (IOException e) {
// ignore
}
} else if (sourceQuery instanceof FiltersFunctionScoreQuery) {
flatten(((FiltersFunctionScoreQuery) sourceQuery).getSubQuery(), flatQueries);
} else if (sourceQuery instanceof FunctionScoreQuery) {
flatten(((FunctionScoreQuery) sourceQuery).getSubQuery(), flatQueries);
flatten(((FiltersFunctionScoreQuery) sourceQuery).getSubQuery(), reader, flatQueries);
} else {
super.flatten(sourceQuery, flatQueries);
super.flatten(sourceQuery, reader, flatQueries);
}
}
void flatten(Filter sourceFilter, Collection<Query> flatQueries) {
void flatten(Filter sourceFilter, IndexReader reader, Collection<Query> flatQueries) throws IOException {
Boolean highlight = highlightFilters.get();
if (highlight == null || highlight.equals(Boolean.FALSE)) {
return;
}
if (sourceFilter instanceof TermFilter) {
flatten(new TermQuery(((TermFilter) sourceFilter).getTerm()), flatQueries);
flatten(new TermQuery(((TermFilter) sourceFilter).getTerm()), reader, flatQueries);
} else if (sourceFilter instanceof PublicTermsFilter) {
PublicTermsFilter termsFilter = (PublicTermsFilter) sourceFilter;
for (Term term : termsFilter.getTerms()) {
flatten(new TermQuery(term), flatQueries);
flatten(new TermQuery(term), reader, flatQueries);
}
} else if (sourceFilter instanceof MultiTermQueryWrapperFilter) {
if (multiTermQueryWrapperFilterQueryField != null) {
try {
flatten((Query) multiTermQueryWrapperFilterQueryField.get(sourceFilter), flatQueries);
flatten((Query) multiTermQueryWrapperFilterQueryField.get(sourceFilter), reader, flatQueries);
} catch (IllegalAccessException e) {
// ignore
}
@ -144,12 +128,12 @@ public class CustomFieldQuery extends FieldQuery {
XBooleanFilter booleanFilter = (XBooleanFilter) sourceFilter;
if (booleanFilter.getMustFilters() != null) {
for (Filter filter : booleanFilter.getMustFilters()) {
flatten(filter, flatQueries);
flatten(filter, reader, flatQueries);
}
}
if (booleanFilter.getShouldFilters() != null) {
for (Filter filter : booleanFilter.getShouldFilters()) {
flatten(filter, flatQueries);
flatten(filter, reader, flatQueries);
}
}
}

96
modules/elasticsearch/src/main/java/org/apache/lucene/search/vectorhighlight/MarginFragListBuilder.java
View File

@ -1,96 +0,0 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.lucene.search.vectorhighlight;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo;
/**
* A modification of SimpleFragListBuilder to expose the margin property. Implements FragListBuilder {@link FragListBuilder}.
*/
public class MarginFragListBuilder implements FragListBuilder {
private static final int DEFAULT_MARGIN = 6;
private int margin;
private int minFragCharSize;
public MarginFragListBuilder() {
this(DEFAULT_MARGIN);
}
public MarginFragListBuilder(int startMargin) {
margin = startMargin;
minFragCharSize = 3*margin;
}
public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) {
if( fragCharSize < minFragCharSize )
throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " +
minFragCharSize + " or higher." );
FieldFragList ffl = new FieldFragList( fragCharSize );
List<WeightedPhraseInfo> wpil = new ArrayList<WeightedPhraseInfo>();
Iterator<WeightedPhraseInfo> ite = fieldPhraseList.phraseList.iterator();
WeightedPhraseInfo phraseInfo = null;
int startOffset = 0;
boolean taken = false;
while( true ){
if( !taken ){
if( !ite.hasNext() ) break;
phraseInfo = ite.next();
}
taken = false;
if( phraseInfo == null ) break;
// if the phrase violates the border of previous fragment, discard it and try next phrase
if( phraseInfo.getStartOffset() < startOffset ) continue;
wpil.clear();
wpil.add( phraseInfo );
int st = phraseInfo.getStartOffset() - margin < startOffset ?
startOffset : phraseInfo.getStartOffset() - margin;
int en = st + fragCharSize;
if( phraseInfo.getEndOffset() > en )
en = phraseInfo.getEndOffset();
startOffset = en;
while( true ){
if( ite.hasNext() ){
phraseInfo = ite.next();
taken = true;
if( phraseInfo == null ) break;
}
else
break;
if( phraseInfo.getEndOffset() <= en )
wpil.add( phraseInfo );
else
break;
}
ffl.add( st, en, wpil );
}
return ffl;
}
}

4
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/IndexCommitDelegate.java
View File

@ -66,8 +66,8 @@ public abstract class IndexCommitDelegate extends IndexCommit {
return delegate.isDeleted();
}
@Override public boolean isOptimized() {
return delegate.isOptimized();
@Override public int getSegmentCount() {
return delegate.getSegmentCount();
}
@Override public boolean equals(Object other) {

35
modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSets.java
View File

@ -61,15 +61,7 @@ public class DocSets {
other = ((FixedBitDocSet) other).set();
}
if (other instanceof FixedBitSet) {
// copied from OpenBitSet#and
long[] intoBits = into.getBits();
long[] otherBits = ((FixedBitSet) other).getBits();
assert intoBits.length == otherBits.length;
// testing against zero can be more efficient
int pos = intoBits.length;
while (--pos >= 0) {
intoBits[pos] &= otherBits[pos];
}
into.and((FixedBitSet) other);
} else {
if (other == null) {
into.clear(0, into.length());
@ -79,16 +71,7 @@ public class DocSets {
if (disi == null) {
into.clear(0, into.length());
} else {
int numBits = into.length();
int disiDoc, bitSetDoc = into.nextSetBit(0);
while (bitSetDoc != -1 && (disiDoc = disi.advance(bitSetDoc)) < numBits) {
into.clear(bitSetDoc, disiDoc);
disiDoc++;
bitSetDoc = (disiDoc < numBits) ? into.nextSetBit(disiDoc) : -1;
}
if (bitSetDoc != -1) {
into.clear(bitSetDoc, numBits);
}
into.and(disi);
}
}
}
@ -102,22 +85,12 @@ public class DocSets {
other = ((FixedBitDocSet) other).set();
}
if (other instanceof FixedBitSet) {
// copied from OpenBitSet#andNot
long[] intoBits = into.getBits();
long[] otherBits = ((FixedBitSet) other).getBits();
assert intoBits.length == otherBits.length;
int idx = intoBits.length;
while (--idx >= 0) {
intoBits[idx] &= ~otherBits[idx];
}
into.andNot((FixedBitSet) other);
} else {
// copied from OpenBitSetDISI#inPlaceNot
DocIdSetIterator disi = other.iterator();
if (disi != null) {
int doc;
while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
into.clear(doc);
}
into.andNot(disi);
}
}
}

6
modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java
View File

@ -761,8 +761,8 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
if (dirty || refresh.force()) {
dirty = false;
AcquirableResource<ReaderSearcherHolder> current = nrtResource;
IndexReader newReader = current.resource().reader().reopen(true);
if (newReader != current.resource().reader()) {
IndexReader newReader = IndexReader.openIfChanged(current.resource().reader(), true);
if (newReader != null) {
ExtendedIndexSearcher indexSearcher = new ExtendedIndexSearcher(newReader);
indexSearcher.setSimilarity(similarityService.defaultSearchSimilarity());
nrtResource = newAcquirableResource(new ReaderSearcherHolder(indexSearcher));
@ -998,7 +998,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
indexWriter.maybeMerge();
possibleMergeNeeded = false;
} else {
indexWriter.optimize(optimize.maxNumSegments(), false);
indexWriter.forceMerge(optimize.maxNumSegments(), false);
}
} catch (OutOfMemoryError e) {
failEngine(e);

8
modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/LogByteSizeMergePolicyProvider.java
View File

@ -208,18 +208,18 @@ public class LogByteSizeMergePolicyProvider extends AbstractIndexShardComponent
return super.findMerges(infos);
}
@Override public MergeSpecification findMergesToExpungeDeletes(SegmentInfos segmentInfos) throws CorruptIndexException, IOException {
@Override public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, Map<SegmentInfo, Boolean> segmentsToMerge) throws IOException {
if (enableMerge.get() == Boolean.FALSE) {
return null;
}
return super.findMergesToExpungeDeletes(segmentInfos);
return super.findForcedMerges(infos, maxSegmentCount, segmentsToMerge);
}
@Override public MergeSpecification findMergesForOptimize(SegmentInfos infos, int maxNumSegments, Map<SegmentInfo, Boolean> segmentsToOptimize) throws IOException {
@Override public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) throws CorruptIndexException, IOException {
if (enableMerge.get() == Boolean.FALSE) {
return null;
}
return super.findMergesForOptimize(infos, maxNumSegments, segmentsToOptimize);
return super.findForcedDeletesMerges(infos);
}
}
}

8
modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/LogDocMergePolicyProvider.java
View File

@ -192,18 +192,18 @@ public class LogDocMergePolicyProvider extends AbstractIndexShardComponent imple
return super.findMerges(infos);
}
@Override public MergeSpecification findMergesToExpungeDeletes(SegmentInfos segmentInfos) throws CorruptIndexException, IOException {
@Override public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, Map<SegmentInfo, Boolean> segmentsToMerge) throws IOException {
if (enableMerge.get() == Boolean.FALSE) {
return null;
}
return super.findMergesToExpungeDeletes(segmentInfos);
return super.findForcedMerges(infos, maxSegmentCount, segmentsToMerge);
}
@Override public MergeSpecification findMergesForOptimize(SegmentInfos infos, int maxNumSegments, Map<SegmentInfo, Boolean> segmentsToOptimize) throws IOException {
@Override public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) throws CorruptIndexException, IOException {
if (enableMerge.get() == Boolean.FALSE) {
return null;
}
return super.findMergesForOptimize(infos, maxNumSegments, segmentsToOptimize);
return super.findForcedDeletesMerges(infos);
}
}
}

26
modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/TieredMergePolicyProvider.java
View File

@ -46,7 +46,7 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple
private final Set<CustomTieredMergePolicyProvider> policies = new CopyOnWriteArraySet<CustomTieredMergePolicyProvider>();
private volatile boolean compoundFormat;
private volatile double expungeDeletesPctAllowed;
private volatile double forceMergeDeletesPctAllowed;
private volatile ByteSizeValue floorSegment;
private volatile int maxMergeAtOnce;
private volatile int maxMergeAtOnceExplicit;
@ -63,7 +63,7 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple
this.compoundFormat = indexSettings.getAsBoolean("index.compound_format", store.suggestUseCompoundFile());
this.asyncMerge = indexSettings.getAsBoolean("index.merge.async", true);
this.expungeDeletesPctAllowed = componentSettings.getAsDouble("expunge_deletes_allowed", 10d); // percentage
this.forceMergeDeletesPctAllowed = componentSettings.getAsDouble("expunge_deletes_allowed", 10d); // percentage
this.floorSegment = componentSettings.getAsBytesSize("floor_segment", new ByteSizeValue(2, ByteSizeUnit.MB));
this.maxMergeAtOnce = componentSettings.getAsInt("max_merge_at_once", 10);
this.maxMergeAtOnceExplicit = componentSettings.getAsInt("max_merge_at_once_explicit", 30);
@ -73,7 +73,7 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple
this.reclaimDeletesWeight = componentSettings.getAsDouble("reclaim_deletes_weight", 2.0d);
logger.debug("using [tiered] merge policy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}], async_merge[{}]",
expungeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight, asyncMerge);
forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight, asyncMerge);
indexSettingsService.addListener(applySettings);
}
@ -87,7 +87,7 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple
mergePolicy = new CustomTieredMergePolicyProvider(this);
}
mergePolicy.setUseCompoundFile(compoundFormat);
mergePolicy.setExpungeDeletesPctAllowed(expungeDeletesPctAllowed);
mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed);
mergePolicy.setFloorSegmentMB(floorSegment.mbFrac());
mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce);
mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
@ -116,12 +116,12 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple
class ApplySettings implements IndexSettingsService.Listener {
@Override public void onRefreshSettings(Settings settings) {
double expungeDeletesPctAllowed = settings.getAsDouble("index.merge.policy.expunge_deletes_allowed", TieredMergePolicyProvider.this.expungeDeletesPctAllowed);
if (expungeDeletesPctAllowed != TieredMergePolicyProvider.this.expungeDeletesPctAllowed) {
logger.info("updating [expunge_deletes_allowed] from [{}] to [{}]", TieredMergePolicyProvider.this.expungeDeletesPctAllowed, expungeDeletesPctAllowed);
TieredMergePolicyProvider.this.expungeDeletesPctAllowed = expungeDeletesPctAllowed;
double expungeDeletesPctAllowed = settings.getAsDouble("index.merge.policy.expunge_deletes_allowed", TieredMergePolicyProvider.this.forceMergeDeletesPctAllowed);
if (expungeDeletesPctAllowed != TieredMergePolicyProvider.this.forceMergeDeletesPctAllowed) {
logger.info("updating [expunge_deletes_allowed] from [{}] to [{}]", TieredMergePolicyProvider.this.forceMergeDeletesPctAllowed, expungeDeletesPctAllowed);
TieredMergePolicyProvider.this.forceMergeDeletesPctAllowed = expungeDeletesPctAllowed;
for (CustomTieredMergePolicyProvider policy : policies) {
policy.setExpungeDeletesPctAllowed(expungeDeletesPctAllowed);
policy.setForceMergeDeletesPctAllowed(expungeDeletesPctAllowed);
}
}
@ -241,18 +241,18 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple
return super.findMerges(infos);
}
@Override public MergePolicy.MergeSpecification findMergesToExpungeDeletes(SegmentInfos segmentInfos) throws CorruptIndexException, IOException {
@Override public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount, Map<SegmentInfo, Boolean> segmentsToMerge) throws IOException {
if (enableMerge.get() == Boolean.FALSE) {
return null;
}
return super.findMergesToExpungeDeletes(segmentInfos);
return super.findForcedMerges(infos, maxSegmentCount, segmentsToMerge);
}
@Override public MergeSpecification findMergesForOptimize(SegmentInfos infos, int maxSegmentCount, Map<SegmentInfo, Boolean> segmentsToOptimize) throws IOException {
@Override public MergeSpecification findForcedDeletesMerges(SegmentInfos infos) throws CorruptIndexException, IOException {
if (enableMerge.get() == Boolean.FALSE) {
return null;
}
return super.findMergesForOptimize(infos, maxSegmentCount, segmentsToOptimize);
return super.findForcedDeletesMerges(infos);
}
}
}

23
modules/elasticsearch/src/main/java/org/elasticsearch/index/query/BoolFilterParser.java
View File

@ -21,16 +21,13 @@ package org.elasticsearch.index.query;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.Filter;
import org.apache.lucene.search.OpenFilterClause;
import org.apache.lucene.search.FilterClause;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.XBooleanFilter;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.cache.filter.support.CacheKeyFilter;
import java.io.IOException;
import java.util.List;
import static org.elasticsearch.common.collect.Lists.*;
/**
* @author kimchy (shay.banon)
@ -49,7 +46,7 @@ public class BoolFilterParser implements FilterParser {
@Override public Filter parse(QueryParseContext parseContext) throws IOException, QueryParsingException {
XContentParser parser = parseContext.parser();
List<OpenFilterClause> clauses = newArrayList();
XBooleanFilter boolFilter = new XBooleanFilter();
boolean cache = false;
CacheKeyFilter.Key cacheKey = null;
@ -62,24 +59,24 @@ public class BoolFilterParser implements FilterParser {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("must".equals(currentFieldName)) {
clauses.add(new OpenFilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST));
boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST));
} else if ("must_not".equals(currentFieldName) || "mustNot".equals(currentFieldName)) {
clauses.add(new OpenFilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST_NOT));
boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST_NOT));
} else if ("should".equals(currentFieldName)) {
clauses.add(new OpenFilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.SHOULD));
boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.SHOULD));
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("must".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
clauses.add(new OpenFilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST));
boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST));
}
} else if ("must_not".equals(currentFieldName) || "mustNot".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
clauses.add(new OpenFilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST_NOT));
boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.MUST_NOT));
}
} else if ("should".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
clauses.add(new OpenFilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.SHOULD));
boolFilter.add(new FilterClause(parseContext.parseInnerFilter(), BooleanClause.Occur.SHOULD));
}
}
} else if (token.isValue()) {
@ -93,10 +90,6 @@ public class BoolFilterParser implements FilterParser {
}
}
XBooleanFilter boolFilter = new XBooleanFilter();
for (OpenFilterClause filterClause : clauses) {
boolFilter.add(filterClause);
}
Filter filter = boolFilter;
if (cache) {
filter = parseContext.cacheFilter(filter, cacheKey);

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/BlockJoinQuery.java
View File

@ -349,10 +349,18 @@ public class BlockJoinQuery extends Query {
return parentDoc = NO_MORE_DOCS;
}
// CHANGE: Remove this and if parentTarget is 0, we can simply call nextdoc
// Every parent must have at least one child:
// assert parentTarget != 0;
if (parentTarget == 0) {
return nextDoc();
}
final int prevParentDoc = parentBits.prevSetBit(parentTarget - 1);
//System.out.println(" rolled back to prevParentDoc=" + prevParentDoc + " vs parentDoc=" + parentDoc);
assert prevParentDoc >= parentDoc;
// CHANGE: Commented out the assert because it might happen with a single nested and parent doc reader
//assert prevParentDoc >= parentDoc;
if (prevParentDoc > nextChildDoc) {
nextChildDoc = childScorer.advance(prevParentDoc);
// System.out.println(" childScorer advanced to child docID=" + nextChildDoc);

24
modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java
View File

@ -28,7 +28,15 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.highlight.*;
import org.apache.lucene.search.vectorhighlight.*;
import org.apache.lucene.search.vectorhighlight.CustomFieldQuery;
import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter;
import org.apache.lucene.search.vectorhighlight.FieldQuery;
import org.apache.lucene.search.vectorhighlight.FragListBuilder;
import org.apache.lucene.search.vectorhighlight.FragmentsBuilder;
import org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilder;
import org.apache.lucene.search.vectorhighlight.SimpleFragListBuilder;
import org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder;
import org.apache.lucene.search.vectorhighlight.SingleFragListBuilder;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.io.FastStringReader;
@ -222,7 +230,7 @@ public class HighlightPhase implements FetchSubPhase {
if (field.fragmentOffset() == -1)
fragListBuilder = new SimpleFragListBuilder();
else
fragListBuilder = new MarginFragListBuilder(field.fragmentOffset());
fragListBuilder = new SimpleFragListBuilder(field.fragmentOffset());
if (field.scoreOrdered()) {
if (mapper.stored()) {
@ -238,11 +246,11 @@ public class HighlightPhase implements FetchSubPhase {
}
}
}
FastVectorHighlighter highlighter = new FastVectorHighlighter(true, false, fragListBuilder, fragmentsBuilder);
FieldQuery fieldQuery = buildFieldQuery(highlighter, context.parsedQuery().query(), hitContext.reader(), field);
String[] fragments;
try {
FastVectorHighlighter highlighter = new FastVectorHighlighter(true, false, fragListBuilder, fragmentsBuilder);
FieldQuery fieldQuery = buildFieldQuery(highlighter, context.parsedQuery().query(), hitContext.reader(), field);
// a HACK to make highlighter do highlighting, even though its using the single frag list builder
int numberOfFragments = field.numberOfFragments() == 0 ? 1 : field.numberOfFragments();
fragments = highlighter.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments,
@ -259,14 +267,12 @@ public class HighlightPhase implements FetchSubPhase {
hitContext.hit().highlightFields(highlightFields);
} finally {
CustomFieldQuery.reader.remove();
CustomFieldQuery.highlightFilters.remove();
}
}
private FieldQuery buildFieldQuery(FastVectorHighlighter highlighter, Query query, IndexReader indexReader, SearchContextHighlight.Field field) {
CustomFieldQuery.reader.set(indexReader);
private FieldQuery buildFieldQuery(FastVectorHighlighter highlighter, Query query, IndexReader indexReader, SearchContextHighlight.Field field) throws IOException {
CustomFieldQuery.highlightFilters.set(field.highlightFilter());
return new CustomFieldQuery(query, highlighter);
return new CustomFieldQuery(query, indexReader, highlighter);
}
}

3
modules/elasticsearch/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java
View File

@ -162,7 +162,8 @@ public class ContextIndexSearcher extends ExtendedIndexSearcher {
collector = new FilteredCollector(collector, searchContext.parsedFilter());
}
if (searchContext.timeout() != null) {
collector = new TimeLimitingCollector(collector, searchContext.timeout().millis());
// TODO: change to use our own counter that uses the scheduler in ThreadPool
collector = new TimeLimitingCollector(collector, TimeLimitingCollector.getGlobalCounter(), searchContext.timeout().millis());
}
if (scopeCollectors != null) {
List<Collector> collectors = scopeCollectors.get(processingScope);

49
modules/elasticsearch/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java
View File

@ -25,6 +25,7 @@ import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@ -42,14 +43,11 @@ import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.common.collect.Lists;
import org.elasticsearch.common.lucene.Lucene;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.IdentityHashMap;
import java.util.List;
import static org.elasticsearch.common.lucene.DocumentBuilder.*;
import static org.hamcrest.MatcherAssert.*;
@ -176,43 +174,6 @@ public class SimpleLuceneTests {
indexWriter.close();
}
@Test public void testNRT() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
IndexReader reader = IndexReader.open(indexWriter, true);
List<IndexReader> readers = Lists.newArrayList();
for (int i = 0; i < 100; i++) {
readers.add(reader);
indexWriter.addDocument(doc()
.add(field("id", Integer.toString(i)))
.boost(i).build());
reader = refreshReader(reader);
}
reader.close();
// verify that all readers are closed
// also, SADLY, verifies that new readers are always created, meaning that caching based on index reader are useless
IdentityHashMap<IndexReader, Boolean> identityReaders = new IdentityHashMap<IndexReader, Boolean>();
for (IndexReader reader1 : readers) {
assertThat(reader1.getRefCount(), equalTo(0));
assertThat(identityReaders.containsKey(reader1), equalTo(false));
identityReaders.put(reader1, Boolean.TRUE);
if (reader1.getSequentialSubReaders() != null) {
for (IndexReader reader2 : reader1.getSequentialSubReaders()) {
assertThat(reader2.getRefCount(), equalTo(0));
assertThat(reader2.getSequentialSubReaders(), nullValue());
assertThat(identityReaders.containsKey(reader2), equalTo(false));
identityReaders.put(reader2, Boolean.TRUE);
}
}
}
}
@Test public void testNRTSearchOnClosedWriter() throws Exception {
Directory dir = new RAMDirectory();
IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER));
@ -242,22 +203,20 @@ public class SimpleLuceneTests {
Document doc = new Document();
NumericField field = new NumericField("int1").setIntValue(1);
field.setOmitNorms(true);
field.setOmitTermFreqAndPositions(true);
field.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS);
doc.add(field);
field = new NumericField("int1").setIntValue(1);
field.setOmitNorms(true);
field.setOmitTermFreqAndPositions(true);
doc.add(field);
field = new NumericField("int2").setIntValue(1);
field.setOmitNorms(true);
field.setOmitTermFreqAndPositions(false);
field.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS);
doc.add(field);
field = new NumericField("int2").setIntValue(1);
field.setOmitNorms(true);
field.setOmitTermFreqAndPositions(false);
field.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS);
doc.add(field);
indexWriter.addDocument(doc);

3
modules/elasticsearch/src/test/java/org/elasticsearch/deps/lucene/VectorHighlighterTests.java
View File

@ -96,8 +96,7 @@ public class VectorHighlighterTests {
// now check with the custom field query
prefixQuery = new PrefixQuery(new Term("content", "ba"));
assertThat(prefixQuery.getRewriteMethod().getClass().getName(), equalTo(PrefixQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT.getClass().getName()));
CustomFieldQuery.reader.set(reader);
fragment = highlighter.getBestFragment(new CustomFieldQuery(prefixQuery, highlighter),
fragment = highlighter.getBestFragment(new CustomFieldQuery(prefixQuery, reader, highlighter),
reader, topDocs.scoreDocs[0].doc, "content", 30);
assertThat(fragment, notNullValue());

11
modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java
View File

@ -405,7 +405,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
}
}
@Test public void testFastVectorHighlighterOffsetParameter() throws Exception {
@Test public void testFastVectorHighlighterOffsetParameter() throws Exception {
try {
client.admin().indices().prepareDelete("test").execute().actionGet();
} catch (Exception e) {
@ -434,7 +434,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
for (SearchHit hit : search.hits()) {
// LUCENE 3.1 UPGRADE: Caused adding the space at the end...
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("hlighting <em>bug</em> present in elasticsearch "));
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting <em>bug</em> present in elasticsearch "));
}
}
@ -464,7 +464,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
SearchResponse search = client.prepareSearch()
.setQuery(fieldQuery("title", "test")).setEncoder("html")
.addHighlightedField("title",50,1,10)
.addHighlightedField("title", 50, 1, 10)
.execute().actionGet();
@ -477,6 +477,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a html escaping highlighting <em>test</em> for *&amp;? elasticsearch"));
}
}
@Test public void testEscapeHtml_vector() throws Exception {
try {
@ -498,7 +499,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
SearchResponse search = client.prepareSearch()
.setQuery(fieldQuery("title", "test")).setEncoder("html")
.addHighlightedField("title",50,1,10)
.addHighlightedField("title", 50, 1, 10)
.execute().actionGet();
@ -508,7 +509,7 @@ public class HighlighterSearchTests extends AbstractNodesTests {
for (SearchHit hit : search.hits()) {
// LUCENE 3.1 UPGRADE: Caused adding the space at the end...
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("hlighting <em>test</em> for *&amp;? elasticsearch "));
assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting <em>test</em> for *&amp;? elasticsearch "));
}
}
}

4
plugins/analysis/icu/build.gradle
View File

@ -35,8 +35,8 @@ dependencies {
compile('org.apache.lucene:lucene-icu4j:3.2.0') { transitive = false }
distLib('org.apache.lucene:lucene-icu4j:3.2.0') { transitive = false }
compile('org.apache.lucene:lucene-icu:3.4.0') { transitive = false }
distLib('org.apache.lucene:lucene-icu:3.4.0') { transitive = false }
compile('org.apache.lucene:lucene-icu:3.5.0') { transitive = false }
distLib('org.apache.lucene:lucene-icu:3.5.0') { transitive = false }
}
task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << {