From 16cd159a381979e9d92288d6c9bada8483fea520 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 15 Apr 2012 17:39:41 +0300 Subject: [PATCH] Upgrade to Lucene 3.6, closes #1862. --- pom.xml | 2 +- .../index/memory/CustomMemoryIndex.java | 39 +- .../elasticsearch/common/lucene/Lucene.java | 5 +- .../lucene/analysis/HTMLStripCharFilter.java | 1373 ----------------- .../lucene/manager/ReferenceManager.java | 181 --- .../lucene/manager/SearcherFactory.java | 56 - .../lucene/manager/SearcherManager.java | 163 -- .../analysis/HtmlStripCharFilterFactory.java | 11 +- .../cache/bloom/simple/SimpleBloomCache.java | 13 +- .../AbstractConcurrentMapFieldDataCache.java | 9 +- .../filter/weighted/WeightedFilterCache.java | 9 +- .../index/cache/id/simple/SimpleIdCache.java | 13 +- .../index/engine/robin/RobinEngine.java | 6 +- .../analysis/IndicesAnalysisService.java | 4 +- .../highlight/HighlighterSearchTests.java | 36 +- .../deps/lucene/VectorHighlighterTests.java | 2 +- 16 files changed, 63 insertions(+), 1859 deletions(-) delete mode 100644 src/main/java/org/elasticsearch/common/lucene/analysis/HTMLStripCharFilter.java delete mode 100755 src/main/java/org/elasticsearch/common/lucene/manager/ReferenceManager.java delete mode 100644 src/main/java/org/elasticsearch/common/lucene/manager/SearcherFactory.java delete mode 100644 src/main/java/org/elasticsearch/common/lucene/manager/SearcherManager.java diff --git a/pom.xml b/pom.xml index 05808717046..ba1ec8d5ab7 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ - 3.5.0 + 3.6.0 diff --git a/src/main/java/org/apache/lucene/index/memory/CustomMemoryIndex.java b/src/main/java/org/apache/lucene/index/memory/CustomMemoryIndex.java index 3954d5d0758..01617403751 100644 --- a/src/main/java/org/apache/lucene/index/memory/CustomMemoryIndex.java +++ b/src/main/java/org/apache/lucene/index/memory/CustomMemoryIndex.java @@ -46,10 +46,10 @@ import org.apache.lucene.search.*; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Constants; +import org.elasticsearch.common.io.FastStringReader; import java.io.IOException; import java.io.Serializable; -import java.io.StringReader; import java.util.*; /** @@ -201,6 +201,8 @@ public class CustomMemoryIndex implements Serializable { private static final boolean DEBUG = false; + private final FieldInfos fieldInfos; + /** * Sorts term entries into ascending order; also works for * Arrays.binarySearch() and Arrays.sort() @@ -234,6 +236,7 @@ public class CustomMemoryIndex implements Serializable { */ private CustomMemoryIndex(boolean storeOffsets) { this.stride = storeOffsets ? 3 : 1; + fieldInfos = new FieldInfos(); } /** @@ -257,8 +260,12 @@ public class CustomMemoryIndex implements Serializable { if (analyzer == null) throw new IllegalArgumentException("analyzer must not be null"); - TokenStream stream = analyzer.tokenStream(fieldName, - new StringReader(text)); + TokenStream stream; + try { + stream = analyzer.reusableTokenStream(fieldName, new FastStringReader(text)); + } catch (IOException ex) { + throw new RuntimeException(ex); + } addField(fieldName, stream); } @@ -338,6 +345,8 @@ public class CustomMemoryIndex implements Serializable { int numOverlapTokens = 0; int pos = -1; + fieldInfos.add(fieldName, true, true); + // CHANGE if (fields.get(fieldName) != null) { Info info = fields.get(fieldName); @@ -760,13 +769,12 @@ public class CustomMemoryIndex implements Serializable { * Search support for Lucene framework integration; implements all methods * required by the Lucene IndexReader contracts. */ - private final class MemoryIndexReader extends IndexReader { + final class MemoryIndexReader extends IndexReader { private Searcher searcher; // needed to find searcher.getSimilarity() private MemoryIndexReader() { super(); // avoid as much superclass baggage as possible - readerFinishedListeners = Collections.synchronizedSet(new HashSet()); } private Info getInfo(String fieldName) { @@ -1174,12 +1182,6 @@ public class CustomMemoryIndex implements Serializable { return 1; } - @Override - public Document document(int n) { - if (DEBUG) System.err.println("MemoryIndexReader.document"); - return new Document(); // there are no stored fields - } - //When we convert to JDK 1.5 make this Set @Override public Document document(int n, FieldSelector fieldSelector) throws IOException { @@ -1219,20 +1221,9 @@ public class CustomMemoryIndex implements Serializable { if (DEBUG) System.err.println("MemoryIndexReader.doClose"); } - // lucene >= 1.9 (remove this method for lucene-1.4.3) @Override - public Collection getFieldNames(FieldOption fieldOption) { - if (DEBUG) System.err.println("MemoryIndexReader.getFieldNamesOption"); - if (fieldOption == FieldOption.UNINDEXED) - return Collections.emptySet(); - if (fieldOption == FieldOption.INDEXED_NO_TERMVECTOR) - return Collections.emptySet(); - if (fieldOption == FieldOption.TERMVECTOR_WITH_OFFSET && stride == 1) - return Collections.emptySet(); - if (fieldOption == FieldOption.TERMVECTOR_WITH_POSITION_OFFSET && stride == 1) - return Collections.emptySet(); - - return Collections.unmodifiableSet(fields.keySet()); + public FieldInfos getFieldInfos() { + return fieldInfos; } } diff --git a/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/src/main/java/org/elasticsearch/common/lucene/Lucene.java index a3f6b980bd5..be09a3bd924 100644 --- a/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -40,7 +40,7 @@ import java.lang.reflect.Field; */ public class Lucene { - public static final Version VERSION = Version.LUCENE_35; + public static final Version VERSION = Version.LUCENE_36; public static final Version ANALYZER_VERSION = VERSION; public static final Version QUERYPARSER_VERSION = VERSION; @@ -55,6 +55,9 @@ public class Lucene { if (version == null) { return defaultVersion; } + if ("3.6".equals(version)) { + return Version.LUCENE_36; + } if ("3.5".equals(version)) { return Version.LUCENE_35; } diff --git a/src/main/java/org/elasticsearch/common/lucene/analysis/HTMLStripCharFilter.java b/src/main/java/org/elasticsearch/common/lucene/analysis/HTMLStripCharFilter.java deleted file mode 100644 index d6ab8e359cb..00000000000 --- a/src/main/java/org/elasticsearch/common/lucene/analysis/HTMLStripCharFilter.java +++ /dev/null @@ -1,1373 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.lucene.analysis; - -import org.apache.lucene.analysis.BaseCharFilter; -import org.apache.lucene.analysis.CharReader; -import org.apache.lucene.analysis.CharStream; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.util.HashMap; -import java.util.Set; - -/** - * - */ -// LUCENE MONITOR: Once the next Lucene version is out, use the built in HTML filter -public class HTMLStripCharFilter extends BaseCharFilter { - private int readAheadLimit = DEFAULT_READ_AHEAD; - private int safeReadAheadLimit = readAheadLimit - 3; - private int numWhitespace = 0; - private int numRead = 0; - private int numEaten = 0; - private int numReturned = 0; - private int lastMark; - private Set escapedTags; - - // pushback buffer - private final StringBuilder pushed = new StringBuilder(); - private static final int EOF = -1; - private static final int MISMATCH = -2; - - private static final int MATCH = -3; - // temporary buffer - private final StringBuilder sb = new StringBuilder(); - public static final int DEFAULT_READ_AHEAD = 8192; - - - public static void main(String[] args) throws IOException { - Reader in = new HTMLStripCharFilter( - CharReader.get(new InputStreamReader(System.in))); - int ch; - while ((ch = in.read()) != -1) System.out.print((char) ch); - } - - public HTMLStripCharFilter(CharStream source) { - super(source.markSupported() ? source : CharReader.get(new BufferedReader(source))); - } - - public HTMLStripCharFilter(CharStream source, Set escapedTags) { - this(source); - this.escapedTags = escapedTags; - } - - public HTMLStripCharFilter(CharStream source, Set escapedTags, int readAheadLimit) { - this(source); - this.escapedTags = escapedTags; - this.readAheadLimit = readAheadLimit; - safeReadAheadLimit = readAheadLimit - 3; - } - - public int getReadAheadLimit() { - return readAheadLimit; - } - - private int next() throws IOException { - int len = pushed.length(); - if (len > 0) { - int ch = pushed.charAt(len - 1); - pushed.setLength(len - 1); - return ch; - } - numRead++; - return input.read(); - } - - private int nextSkipWS() throws IOException { - int ch = next(); - while (isSpace(ch)) ch = next(); - return ch; - } - - private int peek() throws IOException { - int len = pushed.length(); - if (len > 0) { - return pushed.charAt(len - 1); - } - int ch = input.read(); - push(ch); - return ch; - } - - private void push(int ch) { - pushed.append((char) ch); - } - - - private boolean isSpace(int ch) { - switch (ch) { - case ' ': - case '\n': - case '\r': - case '\t': - return true; - default: - return false; - } - } - - private boolean isHex(int ch) { - return (ch >= '0' && ch <= '9') || - (ch >= 'A' && ch <= 'Z') || - (ch >= 'a' && ch <= 'z'); - } - - private boolean isAlpha(int ch) { - return ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z'; - } - - private boolean isDigit(int ch) { - return ch >= '0' && ch <= '9'; - } - - /** - * From HTML 4.0 - * [4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | CombiningChar | Extender - * [5] Name ::= (Letter | '_' | ':') (NameChar)* - * [6] Names ::= Name (#x20 Name)* - * [7] Nmtoken ::= (NameChar)+ - * [8] Nmtokens ::= Nmtoken (#x20 Nmtoken)* - * * - */ - - // should I include all id chars allowable by HTML/XML here? - // including accented chars, ':', etc? - private boolean isIdChar(int ch) { - // return Character.isUnicodeIdentifierPart(ch); - // isUnicodeIdentiferPart doesn't include '-'... shoudl I still - // use it and add in '-',':',etc? - return isAlpha(ch) || isDigit(ch) || ch == '.' || - ch == '-' || ch == '_' || ch == ':' - || Character.isLetter(ch); - - } - - private boolean isFirstIdChar(int ch) { - return Character.isUnicodeIdentifierStart(ch); - // return isAlpha(ch) || ch=='_' || Character.isLetter(ch); - } - - - private void saveState() throws IOException { - lastMark = numRead; - input.mark(readAheadLimit); - } - - private void restoreState() throws IOException { - input.reset(); - pushed.setLength(0); - } - - private int readNumericEntity() throws IOException { - // "&#" has already been read at this point - int eaten = 2; - - // is this decimal, hex, or nothing at all. - int ch = next(); - int base = 10; - boolean invalid = false; - sb.setLength(0); - - if (isDigit(ch)) { - // decimal character entity - sb.append((char) ch); - for (int i = 0; i < 10; i++) { - ch = next(); - if (isDigit(ch)) { - sb.append((char) ch); - } else { - break; - } - } - } else if (ch == 'x') { - eaten++; - // hex character entity - base = 16; - sb.setLength(0); - for (int i = 0; i < 10; i++) { - ch = next(); - if (isHex(ch)) { - sb.append((char) ch); - } else { - break; - } - } - } else { - return MISMATCH; - } - - - // In older HTML, an entity may not have always been terminated - // with a semicolon. We'll also treat EOF or whitespace as terminating - // the entity. - try { - if (ch == ';' || ch == -1) { - // do not account for the eaten ";" due to the fact that we do output a char - numWhitespace = sb.length() + eaten; - return Integer.parseInt(sb.toString(), base); - } - - // if whitespace terminated the entity, we need to return - // that whitespace on the next call to read(). - if (isSpace(ch)) { - push(ch); - numWhitespace = sb.length() + eaten; - return Integer.parseInt(sb.toString(), base); - } - } catch (NumberFormatException e) { - return MISMATCH; - } - - // Not an entity... - return MISMATCH; - } - - private int readEntity() throws IOException { - int ch = next(); - if (ch == '#') return readNumericEntity(); - - //read an entity reference - - // for an entity reference, require the ';' for safety. - // otherwise we may try and convert part of some company - // names to an entity. "Alpha&Beta Corp" for instance. - // - // TODO: perhaps I should special case some of the - // more common ones like & to make the ';' optional... - - sb.setLength(0); - sb.append((char) ch); - - for (int i = 0; i < safeReadAheadLimit; i++) { - ch = next(); - if (Character.isLetter(ch)) { - sb.append((char) ch); - } else { - break; - } - } - - if (ch == ';') { - String entity = sb.toString(); - Character entityChar = entityTable.get(entity); - if (entityChar != null) { - numWhitespace = entity.length() + 1; - return entityChar.charValue(); - } - } - - return MISMATCH; - } - - /** - * valid comments according to HTML specs - * - * - * - * - * - * Hello --> - *

- * #comments inside of an entity decl: - * - *

- * Turns out, IE & mozilla don't parse comments correctly. - * Since this is meant to be a practical stripper, I'll just - * try and duplicate what the browsers do. - *

- * - * - * - *

- *

- * * - */ - - private int readBang(boolean inScript) throws IOException { - // at this point, "') { - - int ch = next(); - if (ch == '>') return MATCH; - - // if it starts with " - //since we did readComment already, it may be the case that we are already deep into the read ahead buffer - //so, we may need to abort sooner - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (ch == '>') { - return MATCH; - } else if (ch < 0) { - return MISMATCH; - } - } - } - return MISMATCH; - } - - // tries to read comments the way browsers do, not - // strictly by the standards. - // - // GRRRR. it turns out that in the wild, a - // - - private int readComment(boolean inScript) throws IOException { - // at this point "') { - push(ch); - push('-'); - continue; - } - - return MATCH; - } else if ((ch == '\'' || ch == '"') && inScript) { - push(ch); - int ret = readScriptString(); - // if this wasn't a string, there's not much we can do - // at this point without having a stack of stream states in - // order to "undo" just the latest. - } else if (ch == '<') { - eatSSI(); - } - - } - return MISMATCH; - - } - - - private int readTag() throws IOException { - // at this point '<' has already been read - int ch = next(); - if (!isAlpha(ch)) { - push(ch); - return MISMATCH; - } - - sb.setLength(0); - sb.append((char) ch); - while ((numRead - lastMark) < safeReadAheadLimit) { - - ch = next(); - if (isIdChar(ch)) { - sb.append((char) ch); - } else if (ch == '/') { - // Hmmm, a tag can close with "/>" as well as "/ >" - // read end tag '/>' or '/ >', etc - return nextSkipWS() == '>' ? MATCH : MISMATCH; - } else { - break; - } - } - if (escapedTags != null && escapedTags.contains(sb.toString())) { - //if this is a reservedTag, then keep it - return MISMATCH; - } - // After the tag id, there needs to be either whitespace or - // '>' - if (!(ch == '>' || isSpace(ch))) { - return MISMATCH; - } - - if (ch != '>') { - // process attributes - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (isSpace(ch)) { - continue; - } else if (isFirstIdChar(ch)) { - push(ch); - int ret = readAttr2(); - if (ret == MISMATCH) return ret; - } else if (ch == '/') { - // read end tag '/>' or '/ >', etc - return nextSkipWS() == '>' ? MATCH : MISMATCH; - } else if (ch == '>') { - break; - } else { - return MISMATCH; - } - - } - if ((numRead - lastMark) >= safeReadAheadLimit) { - return MISMATCH;//exit out if we exceeded the buffer - } - } - - // We only get to this point after we have read the - // entire tag. Now let's see if it's a special tag. - String name = sb.toString(); - if (name.equalsIgnoreCase("script") || name.equalsIgnoreCase("style")) { - // The content of script and style elements is - // CDATA in HTML 4 but PCDATA in XHTML. - - /* From HTML4: - Although the STYLE and SCRIPT elements use CDATA for their data model, - for these elements, CDATA must be handled differently by user agents. - Markup and entities must be treated as raw text and passed to the application - as is. The first occurrence of the character sequence "foo - // beware markup in script strings: ...document.write("")foo - // TODO: do I need to worry about CDATA sections "') return MISMATCH; - return MATCH; - } else if (ch == '\'' || ch == '"') { - // read javascript string to avoid a false match. - push(ch); - int ret = readScriptString(); - // what to do about a non-match (non-terminated string?) - // play it safe and index the rest of the data I guess... - if (ret == MISMATCH) return MISMATCH; - } else if (ch < 0) { - return MISMATCH; - } - - } - return MISMATCH; - } - - - // read a string escaped by backslashes - - private int readScriptString() throws IOException { - int quoteChar = next(); - if (quoteChar != '\'' && quoteChar != '"') return MISMATCH; - - while ((numRead - lastMark) < safeReadAheadLimit) { - int ch = next(); - if (ch == quoteChar) return MATCH; - else if (ch == '\\') { - ch = next(); - } else if (ch < 0) { - return MISMATCH; - } else if (ch == '<') { - eatSSI(); - } - - } - return MISMATCH; - } - - - private int readName(boolean checkEscaped) throws IOException { - StringBuilder builder = (checkEscaped && escapedTags != null) ? new StringBuilder() : null; - int ch = next(); - if (builder != null) builder.append((char) ch); - if (!isFirstIdChar(ch)) return MISMATCH; - ch = next(); - if (builder != null) builder.append((char) ch); - while (isIdChar(ch)) { - ch = next(); - if (builder != null) builder.append((char) ch); - } - if (ch != -1) { - push(ch); - - } - //strip off the trailing > - if (builder != null && escapedTags.contains(builder.substring(0, builder.length() - 1))) { - return MISMATCH; - } - return MATCH; - } - - /** - * [10] AttValue ::= '"' ([^<&"] | Reference)* '"' - * | "'" ([^<&'] | Reference)* "'" - *

- * need to also handle unquoted attributes, and attributes w/o values: - * - *

- * * - */ - - // This reads attributes and attempts to handle any - // embedded server side includes that would otherwise - // mess up the quote handling. - // "> - private int readAttr2() throws IOException { - if ((numRead - lastMark < safeReadAheadLimit)) { - int ch = next(); - if (!isFirstIdChar(ch)) return MISMATCH; - ch = next(); - while (isIdChar(ch) && ((numRead - lastMark) < safeReadAheadLimit)) { - ch = next(); - } - if (isSpace(ch)) ch = nextSkipWS(); - - // attributes may not have a value at all! - // if (ch != '=') return MISMATCH; - if (ch != '=') { - push(ch); - return MATCH; - } - - int quoteChar = nextSkipWS(); - - if (quoteChar == '"' || quoteChar == '\'') { - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (ch < 0) return MISMATCH; - else if (ch == '<') { - eatSSI(); - } else if (ch == quoteChar) { - return MATCH; - //} else if (ch=='<') { - // return MISMATCH; - } - - } - } else { - // unquoted attribute - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (ch < 0) return MISMATCH; - else if (isSpace(ch)) { - push(ch); - return MATCH; - } else if (ch == '>') { - push(ch); - return MATCH; - } else if (ch == '<') { - eatSSI(); - } - - } - } - } - return MISMATCH; - } - - // skip past server side include - - private int eatSSI() throws IOException { - // at this point, only a "<" was read. - // on a mismatch, push back the last char so that if it was - // a quote that closes the attribute, it will be re-read and matched. - int ch = next(); - if (ch != '!') { - push(ch); - return MISMATCH; - } - ch = next(); - if (ch != '-') { - push(ch); - return MISMATCH; - } - ch = next(); - if (ch != '-') { - push(ch); - return MISMATCH; - } - ch = next(); - if (ch != '#') { - push(ch); - return MISMATCH; - } - - push('#'); - push('-'); - push('-'); - return readComment(false); - } - - private int readProcessingInstruction() throws IOException { - // "') { - next(); - return MATCH; - } else if (ch == -1) { - return MISMATCH; - } - - } - return MISMATCH; - } - - - public int read() throws IOException { - // TODO: Do we ever want to preserve CDATA sections? - // where do we have to worry about them? - // - if (numWhitespace > 0) { - numEaten += numWhitespace; - addOffCorrectMap(numReturned, numEaten); - numWhitespace = 0; - } - numReturned++; - //do not limit this one by the READAHEAD - while (true) { - int lastNumRead = numRead; - int ch = next(); - - switch (ch) { - case '&': - saveState(); - ch = readEntity(); - if (ch >= 0) return ch; - if (ch == MISMATCH) { - restoreState(); - - return '&'; - } - break; - - case '<': - saveState(); - ch = next(); - int ret = MISMATCH; - if (ch == '!') { - ret = readBang(false); - } else if (ch == '/') { - ret = readName(true); - if (ret == MATCH) { - ch = nextSkipWS(); - ret = ch == '>' ? MATCH : MISMATCH; - } - } else if (isAlpha(ch)) { - push(ch); - ret = readTag(); - } else if (ch == '?') { - ret = readProcessingInstruction(); - } - - // matched something to be discarded, so break - // from this case and continue in the loop - if (ret == MATCH) { - //break;//was - //return whitespace from - numWhitespace = (numRead - lastNumRead) - 1;//tack on the -1 since we are returning a space right now - return ' '; - } - - // didn't match any HTML constructs, so roll back - // the stream state and just return '<' - restoreState(); - return '<'; - - default: - return ch; - } - - } - - - } - - public int read(char cbuf[], int off, int len) throws IOException { - int i = 0; - for (i = 0; i < len; i++) { - int ch = read(); - if (ch == -1) break; - cbuf[off++] = (char) ch; - } - if (i == 0) { - if (len == 0) return 0; - return -1; - } - return i; - } - - public void close() throws IOException { - input.close(); - } - - - private static final HashMap entityTable; - - static { - entityTable = new HashMap(); - // entityName and entityVal generated from the python script - // included in comments at the end of this file. - final String[] entityName = {"zwnj", "aring", "gt", "yen", "ograve", "Chi", "delta", "rang", "sup", "trade", "Ntilde", "xi", "upsih", "nbsp", "Atilde", "radic", "otimes", "aelig", "oelig", "equiv", "ni", "infin", "Psi", "auml", "cup", "Epsilon", "otilde", "lt", "Icirc", "Eacute", "Lambda", "sbquo", "Prime", "prime", "psi", "Kappa", "rsaquo", "Tau", "uacute", "ocirc", "lrm", "zwj", "cedil", "Alpha", "not", "amp", "AElig", "oslash", "acute", "lceil", "alefsym", "laquo", "shy", "loz", "ge", "Igrave", "nu", "Ograve", "lsaquo", "sube", "euro", "rarr", "sdot", "rdquo", "Yacute", "lfloor", "lArr", "Auml", "Dagger", "brvbar", "Otilde", "szlig", "clubs", "diams", "agrave", "Ocirc", "Iota", "Theta", "Pi", "zeta", "Scaron", "frac14", "egrave", "sub", "iexcl", "frac12", "ordf", "sum", "prop", "Uuml", "ntilde", "atilde", "asymp", "uml", "prod", "nsub", "reg", "rArr", "Oslash", "emsp", "THORN", "yuml", "aacute", "Mu", "hArr", "le", "thinsp", "dArr", "ecirc", "bdquo", "Sigma", "Aring", "tilde", "nabla", "mdash", "uarr", "times", "Ugrave", "Eta", "Agrave", "chi", "real", "circ", "eth", "rceil", "iuml", "gamma", "lambda", "harr", "Egrave", "frac34", "dagger", "divide", "Ouml", "image", "ndash", "hellip", "igrave", "Yuml", "ang", "alpha", "frasl", "ETH", "lowast", "Nu", "plusmn", "bull", "sup1", "sup2", "sup3", "Aacute", "cent", "oline", "Beta", "perp", "Delta", "there4", "pi", "iota", "empty", "euml", "notin", "iacute", "para", "epsilon", "weierp", "OElig", "uuml", "larr", "icirc", "Upsilon", "omicron", "upsilon", "copy", "Iuml", "Oacute", "Xi", "kappa", "ccedil", "Ucirc", "cap", "mu", "scaron", "lsquo", "isin", "Zeta", "minus", "deg", "and", "tau", "pound", "curren", "int", "ucirc", "rfloor", "ensp", "crarr", "ugrave", "exist", "cong", "theta", "oplus", "permil", "Acirc", "piv", "Euml", "Phi", "Iacute", "quot", "Uacute", "Omicron", "ne", "iquest", "eta", "rsquo", "yacute", "Rho", "darr", "Ecirc", "Omega", "acirc", "sim", "phi", "sigmaf", "macr", "thetasym", "Ccedil", "ordm", "uArr", "forall", "beta", "fnof", "rho", "micro", "eacute", "omega", "middot", "Gamma", "rlm", "lang", "spades", "supe", "thorn", "ouml", "or", "raquo", "part", "sect", "ldquo", "hearts", "sigma", "oacute"}; - final char[] entityVal = {8204, 229, 62, 165, 242, 935, 948, 9002, 8835, 8482, 209, 958, 978, 160, 195, 8730, 8855, 230, 339, 8801, 8715, 8734, 936, 228, 8746, 917, 245, 60, 206, 201, 923, 8218, 8243, 8242, 968, 922, 8250, 932, 250, 244, 8206, 8205, 184, 913, 172, 38, 198, 248, 180, 8968, 8501, 171, 173, 9674, 8805, 204, 957, 210, 8249, 8838, 8364, 8594, 8901, 8221, 221, 8970, 8656, 196, 8225, 166, 213, 223, 9827, 9830, 224, 212, 921, 920, 928, 950, 352, 188, 232, 8834, 161, 189, 170, 8721, 8733, 220, 241, 227, 8776, 168, 8719, 8836, 174, 8658, 216, 8195, 222, 255, 225, 924, 8660, 8804, 8201, 8659, 234, 8222, 931, 197, 732, 8711, 8212, 8593, 215, 217, 919, 192, 967, 8476, 710, 240, 8969, 239, 947, 955, 8596, 200, 190, 8224, 247, 214, 8465, 8211, 8230, 236, 376, 8736, 945, 8260, 208, 8727, 925, 177, 8226, 185, 178, 179, 193, 162, 8254, 914, 8869, 916, 8756, 960, 953, 8709, 235, 8713, 237, 182, 949, 8472, 338, 252, 8592, 238, 933, 959, 965, 169, 207, 211, 926, 954, 231, 219, 8745, 956, 353, 8216, 8712, 918, 8722, 176, 8743, 964, 163, 164, 8747, 251, 8971, 8194, 8629, 249, 8707, 8773, 952, 8853, 8240, 194, 982, 203, 934, 205, 34, 218, 927, 8800, 191, 951, 8217, 253, 929, 8595, 202, 937, 226, 8764, 966, 962, 175, 977, 199, 186, 8657, 8704, 946, 402, 961, 181, 233, 969, 183, 915, 8207, 9001, 9824, 8839, 254, 246, 8744, 187, 8706, 167, 8220, 9829, 963, 243}; - for (int i = 0; i < entityName.length; i++) { - entityTable.put(entityName[i], new Character(entityVal[i])); - } - // special-case nbsp to a simple space instead of 0xa0 - entityTable.put("nbsp", new Character(' ')); - } - -} - -/********************* htmlentity.py ********************** - # a simple python script to generate an HTML entity table - # from text taken from http://www.w3.org/TR/REC-html40/sgml/entities.html - - text=""" - 24 Character entity references in HTML 4 - - Contents - - 1. Introduction to character entity references - 2. Character entity references for ISO 8859-1 characters - 1. The list of characters - 3. Character entity references for symbols, mathematical symbols, and Greek letters - 1. The list of characters - 4. Character entity references for markup-significant and internationalization characters - 1. The list of characters - - 24.1 Introduction to character entity references - A character entity reference is an SGML construct that references a character of the document character set. - - This version of HTML supports several sets of character entity references: - - * ISO 8859-1 (Latin-1) characters In accordance with section 14 of [RFC1866], the set of Latin-1 entities has been extended by this specification to cover the whole right part of ISO-8859-1 (all code positions with the high-order bit set), including the already commonly used  , © and ®. The names of the entities are taken from the appendices of SGML (defined in [ISO8879]). - * symbols, mathematical symbols, and Greek letters. These characters may be represented by glyphs in the Adobe font "Symbol". - * markup-significant and internationalization characters (e.g., for bidirectional text). - - The following sections present the complete lists of character entity references. Although, by convention, [ISO10646] the comments following each entry are usually written with uppercase letters, we have converted them to lowercase in this specification for reasons of readability. - 24.2 Character entity references for ISO 8859-1 characters - - The character entity references in this section produce characters whose numeric equivalents should already be supported by conforming HTML 2.0 user agents. Thus, the character entity reference ÷ is a more convenient form than ÷ for obtaining the division sign. - - To support these named entities, user agents need only recognize the entity names and convert them to characters that lie within the repertoire of [ISO88591]. - - Character 65533 (FFFD hexadecimal) is the last valid character in UCS-2. 65534 (FFFE hexadecimal) is unassigned and reserved as the byte-swapped version of ZERO WIDTH NON-BREAKING SPACE for byte-order detection purposes. 65535 (FFFF hexadecimal) is unassigned. - 24.2.1 The list of characters - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 24.3 Character entity references for symbols, mathematical symbols, and Greek letters - - The character entity references in this section produce characters that may be represented by glyphs in the widely available Adobe Symbol font, including Greek characters, various bracketing symbols, and a selection of mathematical operators such as gradient, product, and summation symbols. - - To support these entities, user agents may support full [ISO10646] or use other means. Display of glyphs for these characters may be obtained by being able to display the relevant [ISO10646] characters or by other means, such as internally mapping the listed entities, numeric character references, and characters to the appropriate position in some font that contains the requisite glyphs. - - When to use Greek entities. This entity set contains all the letters used in modern Greek. However, it does not include Greek punctuation, precomposed accented characters nor the non-spacing accents (tonos, dialytika) required to compose them. There are no archaic letters, Coptic-unique letters, or precomposed letters for Polytonic Greek. The entities defined here are not intended for the representation of modern Greek text and would not be an efficient representation; rather, they are intended for occasional Greek letters used in technical and mathematical works. - 24.3.1 The list of characters - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 24.4 Character entity references for markup-significant and internationalization characters - - The character entity references in this section are for escaping markup-significant characters (these are the same as those in HTML 2.0 and 3.2), for denoting spaces and dashes. Other characters in this section apply to internationalization issues such as the disambiguation of bidirectional text (see the section on bidirectional text for details). - - Entities have also been added for the remaining characters occurring in CP-1252 which do not occur in the HTMLlat1 or HTMLsymbol entity sets. These all occur in the 128 to 159 range within the CP-1252 charset. These entities permit the characters to be denoted in a platform-independent manner. - - To support these entities, user agents may support full [ISO10646] or use other means. Display of glyphs for these characters may be obtained by being able to display the relevant [ISO10646] characters or by other means, such as internally mapping the listed entities, numeric character references, and characters to the appropriate position in some font that contains the requisite glyphs. - 24.4.1 The list of characters - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - """ - - codes={} - for line in text.split('\n'): - parts = line.split() - if len(parts)<3 or parts[0]!=' the concrete type that will be {@link #acquire() acquired} and - * {@link #release(Object) released}. - * @lucene.experimental - */ -// LUCENE MONITOR: 3.6 Remove this once 3.6 is out and use it -public abstract class ReferenceManager implements Closeable { - - private static final String REFERENCE_MANAGER_IS_CLOSED_MSG = "this ReferenceManager is closed"; - - protected volatile G current; - - private final Semaphore reopenLock = new Semaphore(1); - - private void ensureOpen() { - if (current == null) { - throw new AlreadyClosedException(REFERENCE_MANAGER_IS_CLOSED_MSG); - } - } - - private synchronized void swapReference(G newReference) throws IOException { - ensureOpen(); - final G oldReference = current; - current = newReference; - release(oldReference); - } - - /** - * Decrement reference counting on the given reference. - */ - protected abstract void decRef(G reference) throws IOException; - - /** - * Refresh the given reference if needed. Returns {@code null} if no refresh - * was needed, otherwise a new refreshed reference. - */ - protected abstract G refreshIfNeeded(G referenceToRefresh) throws IOException; - - /** - * Try to increment reference counting on the given reference. Return true if - * the operation was successful. - */ - protected abstract boolean tryIncRef(G reference); - - /** - * Obtain the current reference. You must match every call to acquire with one - * call to {@link #release}; it's best to do so in a finally clause, and set - * the reference to {@code null} to prevent accidental usage after it has been - * released. - */ - public final G acquire() { - G ref; - do { - if ((ref = current) == null) { - throw new AlreadyClosedException(REFERENCE_MANAGER_IS_CLOSED_MSG); - } - } while (!tryIncRef(ref)); - return ref; - } - - /** - * Close this ReferenceManager to future {@link #acquire() acquiring}. Any - * references that were previously {@link #acquire() acquired} won't be - * affected, and they should still be {@link #release released} when they are - * not needed anymore. - */ - public final synchronized void close() throws IOException { - if (current != null) { - // make sure we can call this more than once - // closeable javadoc says: - // if this is already closed then invoking this method has no effect. - swapReference(null); - afterClose(); - } - } - - /** - * Called after close(), so subclass can free any resources. - */ - protected void afterClose() throws IOException { - } - - /** - * You must call this, periodically, if you want that {@link #acquire()} will - * return refreshed instances. - *

- *

- * Threads: it's fine for more than one thread to call this at once. - * Only the first thread will attempt the refresh; subsequent threads will see - * that another thread is already handling refresh and will return - * immediately. Note that this means if another thread is already refreshing - * then subsequent threads will return right away without waiting for the - * refresh to complete. - *

- *

- * If this method returns true it means the calling thread either refreshed - * or that there were no changes to refresh. If it returns false it means another - * thread is currently refreshing. - */ - public final boolean maybeRefresh() throws IOException { - ensureOpen(); - - // Ensure only 1 thread does reopen at once; other threads just return immediately: - final boolean doTryRefresh = reopenLock.tryAcquire(); - if (doTryRefresh) { - try { - final G reference = acquire(); - try { - G newReference = refreshIfNeeded(reference); - if (newReference != null) { - assert newReference != reference : "refreshIfNeeded should return null if refresh wasn't needed"; - boolean success = false; - try { - swapReference(newReference); - success = true; - } finally { - if (!success) { - release(newReference); - } - } - } - } finally { - release(reference); - } - afterRefresh(); - } finally { - reopenLock.release(); - } - } - - return doTryRefresh; - } - - /** - * Called after swapReference has installed a new - * instance. - */ - protected void afterRefresh() throws IOException { - } - - /** - * Release the refernce previously obtained via {@link #acquire()}. - *

- * NOTE: it's safe to call this after {@link #close()}. - */ - public final void release(G reference) throws IOException { - assert reference != null; - decRef(reference); - } -} diff --git a/src/main/java/org/elasticsearch/common/lucene/manager/SearcherFactory.java b/src/main/java/org/elasticsearch/common/lucene/manager/SearcherFactory.java deleted file mode 100644 index c58249d19dc..00000000000 --- a/src/main/java/org/elasticsearch/common/lucene/manager/SearcherFactory.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.elasticsearch.common.lucene.manager; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.IndexSearcher; - -import java.io.IOException; - -/** - * Factory class used by {@link org.apache.lucene.search.SearcherManager} and {@link org.apache.lucene.search.NRTManager} to - * create new IndexSearchers. The default implementation just creates - * an IndexSearcher with no custom behavior: - *

- * 

- * public IndexSearcher newSearcher(IndexReader r) throws IOException {
- * return new IndexSearcher(r);
- * }
- *
- *

- * You can pass your own factory instead if you want custom behavior, such as: - *

    - *
  • Setting a custom scoring model: {@link org.apache.lucene.search.IndexSearcher#setSimilarity(org.apache.lucene.search.Similarity)} - *
  • Parallel per-segment search: {@link org.apache.lucene.search.IndexSearcher#IndexSearcher(org.apache.lucene.index.IndexReader, java.util.concurrent.ExecutorService)} - *
  • Return custom subclasses of IndexSearcher (for example that implement distributed scoring) - *
  • Run queries to warm your IndexSearcher before it is used. Note: when using near-realtime search - * you may want to also {@link org.apache.lucene.index.IndexWriterConfig#setMergedSegmentWarmer(org.apache.lucene.index.IndexWriter.IndexReaderWarmer)} to warm - * newly merged segments in the background, outside of the reopen path. - *
- * - * @lucene.experimental - */ -// LUCENE MONITOR: 3.6 Remove this once 3.6 is out and use it -public class SearcherFactory { - /** - * Returns a new IndexSearcher over the given reader. - */ - public IndexSearcher newSearcher(IndexReader reader) throws IOException { - return new IndexSearcher(reader); - } -} diff --git a/src/main/java/org/elasticsearch/common/lucene/manager/SearcherManager.java b/src/main/java/org/elasticsearch/common/lucene/manager/SearcherManager.java deleted file mode 100644 index 98ae8f3680f..00000000000 --- a/src/main/java/org/elasticsearch/common/lucene/manager/SearcherManager.java +++ /dev/null @@ -1,163 +0,0 @@ -package org.elasticsearch.common.lucene.manager; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.store.Directory; - -import java.io.IOException; - -/** - * Utility class to safely share {@link org.apache.lucene.search.IndexSearcher} instances across multiple - * threads, while periodically reopening. This class ensures each searcher is - * closed only once all threads have finished using it. - *

- *

- * Use {@link #acquire} to obtain the current searcher, and {@link #release} to - * release it, like this: - *

- * 

- * IndexSearcher s = manager.acquire();
- * try {
- * // Do searching, doc retrieval, etc. with s
- * } finally {
- * manager.release(s);
- * }
- * // Do not use s after this!
- * s = null;
- *
- *

- *

- * In addition you should periodically call {@link #maybeRefresh}. While it's - * possible to call this just before running each query, this is discouraged - * since it penalizes the unlucky queries that do the reopen. It's better to use - * a separate background thread, that periodically calls maybeReopen. Finally, - * be sure to call {@link #close} once you are done. - * - * @lucene.experimental - * @see SearcherFactory - */ -// LUCENE MONITOR: 3.6 Remove this once 3.6 is out and use it -public final class SearcherManager extends ReferenceManager { - - private final SearcherFactory searcherFactory; - - /** - * Creates and returns a new SearcherManager from the given {@link org.apache.lucene.index.IndexWriter}. - * - * @param writer the IndexWriter to open the IndexReader from. - * @param applyAllDeletes If true, all buffered deletes will - * be applied (made visible) in the {@link org.apache.lucene.search.IndexSearcher} / {@link org.apache.lucene.index.IndexReader}. - * If false, the deletes may or may not be applied, but remain buffered - * (in IndexWriter) so that they will be applied in the future. - * Applying deletes can be costly, so if your app can tolerate deleted documents - * being returned you might gain some performance by passing false. - * See {@link org.apache.lucene.index.IndexReader#openIfChanged(org.apache.lucene.index.IndexReader, org.apache.lucene.index.IndexWriter, boolean)}. - * @param searcherFactory An optional {@link SearcherFactory}. Pass - * null if you don't require the searcher to be warmed - * before going live or other custom behavior. - * @throws java.io.IOException - */ - public SearcherManager(IndexWriter writer, boolean applyAllDeletes, SearcherFactory searcherFactory) throws IOException { - if (searcherFactory == null) { - searcherFactory = new SearcherFactory(); - } - this.searcherFactory = searcherFactory; - current = getSearcher(searcherFactory, IndexReader.open(writer, applyAllDeletes)); - } - - /** - * Creates and returns a new SearcherManager from the given {@link org.apache.lucene.store.Directory}. - * - * @param dir the directory to open the DirectoryReader on. - * @param searcherFactory An optional {@link SearcherFactory}. Pass - * null if you don't require the searcher to be warmed - * before going live or other custom behavior. - * @throws java.io.IOException - */ - public SearcherManager(Directory dir, SearcherFactory searcherFactory) throws IOException { - if (searcherFactory == null) { - searcherFactory = new SearcherFactory(); - } - this.searcherFactory = searcherFactory; - current = getSearcher(searcherFactory, IndexReader.open(dir)); - } - - @Override - protected void decRef(IndexSearcher reference) throws IOException { - reference.getIndexReader().decRef(); - } - - @Override - protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException { - final IndexReader newReader = IndexReader.openIfChanged(referenceToRefresh.getIndexReader()); - if (newReader == null) { - return null; - } else { - return getSearcher(searcherFactory, newReader); - } - } - - @Override - protected boolean tryIncRef(IndexSearcher reference) { - return reference.getIndexReader().tryIncRef(); - } - - /** - * @deprecated see {@link #maybeRefresh()}. - */ - @Deprecated - public boolean maybeReopen() throws IOException { - return maybeRefresh(); - } - - /** - * Returns true if no changes have occured since this searcher - * ie. reader was opened, otherwise false. - * - * @see org.apache.lucene.index.IndexReader#isCurrent() - */ - public boolean isSearcherCurrent() throws IOException { - final IndexSearcher searcher = acquire(); - try { - return searcher.getIndexReader().isCurrent(); - } finally { - release(searcher); - } - } - - // NOTE: decRefs incoming reader on throwing an exception - static IndexSearcher getSearcher(SearcherFactory searcherFactory, IndexReader reader) throws IOException { - boolean success = false; - final IndexSearcher searcher; - try { - searcher = searcherFactory.newSearcher(reader); - if (searcher.getIndexReader() != reader) { - throw new IllegalStateException("SearcherFactory must wrap exactly the provided reader (got " + searcher.getIndexReader() + " but expected " + reader + ")"); - } - success = true; - } finally { - if (!success) { - reader.decRef(); - } - } - return searcher; - } -} diff --git a/src/main/java/org/elasticsearch/index/analysis/HtmlStripCharFilterFactory.java b/src/main/java/org/elasticsearch/index/analysis/HtmlStripCharFilterFactory.java index 75ae6c4567d..4cb3ea41b98 100644 --- a/src/main/java/org/elasticsearch/index/analysis/HtmlStripCharFilterFactory.java +++ b/src/main/java/org/elasticsearch/index/analysis/HtmlStripCharFilterFactory.java @@ -21,9 +21,9 @@ package org.elasticsearch.index.analysis; import com.google.common.collect.ImmutableSet; import org.apache.lucene.analysis.CharStream; +import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.lucene.analysis.HTMLStripCharFilter; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.settings.IndexSettings; @@ -35,12 +35,9 @@ public class HtmlStripCharFilterFactory extends AbstractCharFilterFactory { private final ImmutableSet escapedTags; - private final int readAheadLimit; - @Inject public HtmlStripCharFilterFactory(Index index, @IndexSettings Settings indexSettings, @Assisted String name, @Assisted Settings settings) { super(index, indexSettings, name); - this.readAheadLimit = settings.getAsInt("read_ahead", HTMLStripCharFilter.DEFAULT_READ_AHEAD); String[] escapedTags = settings.getAsArray("escaped_tags"); if (escapedTags.length > 0) { this.escapedTags = ImmutableSet.copyOf(escapedTags); @@ -53,12 +50,8 @@ public class HtmlStripCharFilterFactory extends AbstractCharFilterFactory { return escapedTags; } - public int readAheadLimit() { - return readAheadLimit; - } - @Override public CharStream create(CharStream tokenStream) { - return new HTMLStripCharFilter(tokenStream, escapedTags, readAheadLimit); + return new HTMLStripCharFilter(tokenStream, escapedTags); } } diff --git a/src/main/java/org/elasticsearch/index/cache/bloom/simple/SimpleBloomCache.java b/src/main/java/org/elasticsearch/index/cache/bloom/simple/SimpleBloomCache.java index 90e5f338d98..b2f228167ec 100644 --- a/src/main/java/org/elasticsearch/index/cache/bloom/simple/SimpleBloomCache.java +++ b/src/main/java/org/elasticsearch/index/cache/bloom/simple/SimpleBloomCache.java @@ -19,10 +19,7 @@ package org.elasticsearch.index.cache.bloom.simple; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.*; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.UnicodeUtil; @@ -48,7 +45,7 @@ import java.util.concurrent.atomic.AtomicBoolean; /** * */ -public class SimpleBloomCache extends AbstractIndexComponent implements BloomCache, IndexReader.ReaderFinishedListener { +public class SimpleBloomCache extends AbstractIndexComponent implements BloomCache, SegmentReader.CoreClosedListener { private final ThreadPool threadPool; @@ -78,8 +75,8 @@ public class SimpleBloomCache extends AbstractIndexComponent implements BloomCac } @Override - public void finished(IndexReader reader) { - clear(reader); + public void onClose(SegmentReader owner) { + clear(owner); } @Override @@ -126,7 +123,7 @@ public class SimpleBloomCache extends AbstractIndexComponent implements BloomCac synchronized (creationMutex) { fieldCache = cache.get(reader.getCoreCacheKey()); if (fieldCache == null) { - reader.addReaderFinishedListener(this); + ((SegmentReader) reader).addCoreClosedListener(this); fieldCache = ConcurrentCollections.newConcurrentMap(); cache.put(reader.getCoreCacheKey(), fieldCache); } diff --git a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java index 1bf1850f291..600df48dc2a 100644 --- a/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java +++ b/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.cache.field.data.support; import com.google.common.cache.Cache; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.SegmentReader; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -38,7 +39,7 @@ import java.util.concurrent.ConcurrentMap; /** * */ -public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexComponent implements FieldDataCache, IndexReader.ReaderFinishedListener { +public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexComponent implements FieldDataCache, SegmentReader.CoreClosedListener { private final ConcurrentMap> cache; @@ -67,8 +68,8 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC } @Override - public void finished(IndexReader reader) { - clear(reader); + public void onClose(SegmentReader owner) { + clear(owner); } @Override @@ -108,7 +109,7 @@ public abstract class AbstractConcurrentMapFieldDataCache extends AbstractIndexC fieldDataCache = cache.get(reader.getCoreCacheKey()); if (fieldDataCache == null) { fieldDataCache = buildFieldDataMap(); - reader.addReaderFinishedListener(this); + ((SegmentReader) reader).addCoreClosedListener(this); cache.put(reader.getCoreCacheKey(), fieldDataCache); } } diff --git a/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java b/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java index 20292e58829..0b11e4dd08c 100644 --- a/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java +++ b/src/main/java/org/elasticsearch/index/cache/filter/weighted/WeightedFilterCache.java @@ -24,6 +24,7 @@ import com.google.common.cache.RemovalListener; import com.google.common.cache.RemovalNotification; import com.google.common.cache.Weigher; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.SegmentReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.elasticsearch.ElasticSearchException; @@ -45,7 +46,7 @@ import org.elasticsearch.indices.cache.filter.IndicesFilterCache; import java.io.IOException; import java.util.concurrent.ConcurrentMap; -public class WeightedFilterCache extends AbstractIndexComponent implements FilterCache, IndexReader.ReaderFinishedListener, RemovalListener> { +public class WeightedFilterCache extends AbstractIndexComponent implements FilterCache, SegmentReader.CoreClosedListener, RemovalListener> { final IndicesFilterCache indicesFilterCache; @@ -91,8 +92,8 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte } @Override - public void finished(IndexReader reader) { - clear(reader); + public void onClose(SegmentReader owner) { + clear(owner); } @Override @@ -165,7 +166,7 @@ public class WeightedFilterCache extends AbstractIndexComponent implements Filte if (!cache.seenReaders.containsKey(reader.getCoreCacheKey())) { Boolean previous = cache.seenReaders.putIfAbsent(reader.getCoreCacheKey(), Boolean.TRUE); if (previous == null) { - reader.addReaderFinishedListener(cache); + ((SegmentReader) reader).addCoreClosedListener(cache); cache.seenReadersCount.inc(); } } diff --git a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java index fa432fc76ff..1fc669330ad 100644 --- a/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java +++ b/src/main/java/org/elasticsearch/index/cache/id/simple/SimpleIdCache.java @@ -20,10 +20,7 @@ package org.elasticsearch.index.cache.id.simple; import gnu.trove.impl.Constants; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermDocs; -import org.apache.lucene.index.TermEnum; +import org.apache.lucene.index.*; import org.apache.lucene.util.StringHelper; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.common.BytesWrap; @@ -50,7 +47,7 @@ import java.util.concurrent.ConcurrentMap; /** * */ -public class SimpleIdCache extends AbstractIndexComponent implements IdCache, IndexReader.ReaderFinishedListener { +public class SimpleIdCache extends AbstractIndexComponent implements IdCache, SegmentReader.CoreClosedListener { private final ConcurrentMap idReaders; @@ -71,8 +68,8 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, In } @Override - public void finished(IndexReader reader) { - clear(reader); + public void onClose(SegmentReader owner) { + clear(owner); } @Override @@ -112,7 +109,7 @@ public class SimpleIdCache extends AbstractIndexComponent implements IdCache, In continue; } - reader.addReaderFinishedListener(this); + ((SegmentReader) reader).addCoreClosedListener(this); HashMap readerBuilder = new HashMap(); builders.put(reader.getCoreCacheKey(), readerBuilder); diff --git a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index ffba421bc17..1e897ec0443 100644 --- a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -20,9 +20,7 @@ package org.elasticsearch.index.engine.robin; import org.apache.lucene.index.*; -import org.apache.lucene.search.FilteredQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; +import org.apache.lucene.search.*; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticSearchException; @@ -33,8 +31,6 @@ import org.elasticsearch.common.bloom.BloomFilter; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.manager.SearcherFactory; -import org.elasticsearch.common.lucene.manager.SearcherManager; import org.elasticsearch.common.lucene.uid.UidField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; diff --git a/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java b/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java index bb0611bddbb..57ed7cfc46a 100644 --- a/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java +++ b/src/main/java/org/elasticsearch/indices/analysis/IndicesAnalysisService.java @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.bg.BulgarianAnalyzer; import org.apache.lucene.analysis.br.BrazilianAnalyzer; import org.apache.lucene.analysis.br.BrazilianStemFilter; import org.apache.lucene.analysis.ca.CatalanAnalyzer; +import org.apache.lucene.analysis.charfilter.HTMLStripCharFilter; import org.apache.lucene.analysis.cjk.CJKAnalyzer; import org.apache.lucene.analysis.cn.ChineseAnalyzer; import org.apache.lucene.analysis.cz.CzechAnalyzer; @@ -43,6 +44,7 @@ import org.apache.lucene.analysis.fi.FinnishAnalyzer; import org.apache.lucene.analysis.fr.ElisionFilter; import org.apache.lucene.analysis.fr.FrenchAnalyzer; import org.apache.lucene.analysis.fr.FrenchStemFilter; +import org.apache.lucene.analysis.ga.IrishAnalyzer; import org.apache.lucene.analysis.gl.GalicianAnalyzer; import org.apache.lucene.analysis.hi.HindiAnalyzer; import org.apache.lucene.analysis.hu.HungarianAnalyzer; @@ -76,7 +78,6 @@ import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.lucene.analysis.HTMLStripCharFilter; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -141,6 +142,7 @@ public class IndicesAnalysisService extends AbstractComponent { analyzerProviderFactories.put("hindi", new PreBuiltAnalyzerProviderFactory("hindi", AnalyzerScope.INDICES, new HindiAnalyzer(Lucene.ANALYZER_VERSION))); analyzerProviderFactories.put("hungarian", new PreBuiltAnalyzerProviderFactory("hungarian", AnalyzerScope.INDICES, new HungarianAnalyzer(Lucene.ANALYZER_VERSION))); analyzerProviderFactories.put("indonesian", new PreBuiltAnalyzerProviderFactory("indonesian", AnalyzerScope.INDICES, new IndonesianAnalyzer(Lucene.ANALYZER_VERSION))); + analyzerProviderFactories.put("irish", new PreBuiltAnalyzerProviderFactory("irish", AnalyzerScope.INDICES, new IrishAnalyzer(Lucene.ANALYZER_VERSION))); analyzerProviderFactories.put("italian", new PreBuiltAnalyzerProviderFactory("italian", AnalyzerScope.INDICES, new ItalianAnalyzer(Lucene.ANALYZER_VERSION))); analyzerProviderFactories.put("latvian", new PreBuiltAnalyzerProviderFactory("latvian", AnalyzerScope.INDICES, new LatvianAnalyzer(Lucene.ANALYZER_VERSION))); analyzerProviderFactories.put("norwegian", new PreBuiltAnalyzerProviderFactory("norwegian", AnalyzerScope.INDICES, new NorwegianAnalyzer(Lucene.ANALYZER_VERSION))); diff --git a/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java index 21811ecee0e..b1fb300077d 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java @@ -162,7 +162,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(search.hits().hits().length, equalTo(5)); for (SearchHit hit : search.hits()) { - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting bug present in elasticsearch ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting bug present in elasticsearch")); } search = client.prepareSearch() @@ -176,7 +176,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(search.hits().hits().length, equalTo(5)); for (SearchHit hit : search.hits()) { - assertThat(hit.highlightFields().get("attachments.body").fragments()[0], equalTo("attachment 1 attachment 2 ")); + assertThat(hit.highlightFields().get("attachments.body").fragments()[0], equalTo("attachment 1 attachment 2")); } } @@ -267,8 +267,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat("Failures " + Arrays.toString(searchResponse.shardFailures()), searchResponse.shardFailures().length, equalTo(0)); assertThat(searchResponse.hits().totalHits(), equalTo(1l)); - // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a test ")); + assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a test")); logger.info("--> searching on _all, highlighting on field1"); source = searchSource() @@ -281,7 +280,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo(1l)); // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a test ")); + assertThat(searchResponse.hits().getAt(0).highlightFields().get("field1").fragments()[0], equalTo("this is a test")); logger.info("--> searching on _all, highlighting on field2"); source = searchSource() @@ -294,7 +293,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo(1l)); // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The quick brown fox jumps over the lazy dog ")); + assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The quick brown fox jumps over the lazy dog")); logger.info("--> searching on _all, highlighting on field2"); source = searchSource() @@ -307,7 +306,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo(1l)); // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The quick brown fox jumps over the lazy dog ")); + assertThat(searchResponse.hits().getAt(0).highlightFields().get("field2").fragments()[0], equalTo("The quick brown fox jumps over the lazy dog")); } @Test @@ -342,7 +341,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().hits().length, equalTo(COUNT)); for (SearchHit hit : searchResponse.hits()) { // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(hit.highlightFields().get("field1").fragments()[0], equalTo("test " + hit.id() + " ")); + assertThat(hit.highlightFields().get("field1").fragments()[0], equalTo("test " + hit.id())); } logger.info("--> searching explicitly on field1 and highlighting on it, with DFS"); @@ -355,8 +354,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT)); assertThat(searchResponse.hits().hits().length, equalTo(COUNT)); for (SearchHit hit : searchResponse.hits()) { - // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(hit.highlightFields().get("field1").fragments()[0], equalTo("test " + hit.id() + " ")); + assertThat(hit.highlightFields().get("field1").fragments()[0], equalTo("test " + hit.id())); } logger.info("--> searching explicitly _all and highlighting on _all"); @@ -368,8 +366,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(searchResponse.hits().totalHits(), equalTo((long) COUNT)); assertThat(searchResponse.hits().hits().length, equalTo(COUNT)); for (SearchHit hit : searchResponse.hits()) { - // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(hit.highlightFields().get("_all").fragments()[0], equalTo("test " + hit.id() + " ")); + assertThat(hit.highlightFields().get("_all").fragments()[0], equalTo("test " + hit.id() + " ")); } } @@ -413,7 +410,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { for (SearchHit hit : search.hits()) { // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting bug present in elasticsearch ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("This is a test on the highlighting bug present in elasticsearch")); } } @@ -448,7 +445,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { for (SearchHit hit : search.hits()) { // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting bug present in elasticsearch ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting bug present in elasticsearch")); } } @@ -523,8 +520,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(search.hits().hits().length, equalTo(5)); for (SearchHit hit : search.hits()) { - // LUCENE 3.1 UPGRADE: Caused adding the space at the end... - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting test for *&? elasticsearch ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("highlighting test for *&? elasticsearch")); } } @@ -555,7 +551,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); SearchHit hit = search.hits().getAt(0); - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test")); // search on title.key and highlight on title search = client.prepareSearch() @@ -566,7 +562,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); hit = search.hits().getAt(0); - assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test ")); + assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test")); } @Test @@ -596,7 +592,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); SearchHit hit = search.hits().getAt(0); - assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test ")); + assertThat(hit.highlightFields().get("title").fragments()[0], equalTo("this is a test")); // search on title.key and highlight on title.key search = client.prepareSearch() @@ -607,7 +603,7 @@ public class HighlighterSearchTests extends AbstractNodesTests { assertThat(Arrays.toString(search.shardFailures()), search.failedShards(), equalTo(0)); hit = search.hits().getAt(0); - assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test ")); + assertThat(hit.highlightFields().get("title.key").fragments()[0], equalTo("this is a test")); } @Test diff --git a/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java b/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java index fde13951bf8..09ba8e1dd42 100644 --- a/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java +++ b/src/test/java/org/elasticsearch/test/unit/deps/lucene/VectorHighlighterTests.java @@ -60,7 +60,7 @@ public class VectorHighlighterTests { String fragment = highlighter.getBestFragment(highlighter.getFieldQuery(new TermQuery(new Term("content", "bad"))), reader, topDocs.scoreDocs[0].doc, "content", 30); assertThat(fragment, notNullValue()); - assertThat(fragment, equalTo("e big bad dog ")); + assertThat(fragment, equalTo("the big bad dog")); } @Test