From 9c742d4de4817c49f3b51d635a02953d8a0dc391 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Sun, 8 Jul 2012 17:27:56 +0000 Subject: [PATCH] LUCENE-4199: fix more locale violations git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4199@1358791 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/lucene/document/TestDateTools.java | 12 ++++++------ .../lucene/search/TestCustomSearcherSort.java | 6 +++++- .../apache/lucene/queries/ChainedFilterTest.java | 5 ++++- .../queryparser/classic/QueryParserBase.java | 15 ++++++++------- .../precedence/TestPrecedenceQueryParser.java | 8 +++++--- .../flexible/standard/TestQPHelper.java | 8 +++++--- .../queryparser/util/QueryParserTestBase.java | 8 +++++--- .../jaspell/JaspellTernarySearchTrie.java | 16 ++++++++++++---- .../lucene/index/AlcoholicMergePolicy.java | 3 ++- 9 files changed, 52 insertions(+), 29 deletions(-) diff --git a/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java b/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java index c42388980da..779f58e1157 100644 --- a/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java +++ b/lucene/core/src/test/org/apache/lucene/document/TestDateTools.java @@ -61,12 +61,12 @@ public class TestDateTools extends LuceneTestCase { public void testStringtoTime() throws ParseException { long time = DateTools.stringToTime("197001010000"); - Calendar cal = new GregorianCalendar(); + // we use default locale since LuceneTestCase randomizes it + Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("GMT"), Locale.getDefault()); cal.clear(); cal.set(1970, 0, 1, // year=1970, month=january, day=1 0, 0, 0); // hour, minute, second cal.set(Calendar.MILLISECOND, 0); - cal.setTimeZone(TimeZone.getTimeZone("GMT")); assertEquals(cal.getTime().getTime(), time); cal.set(1980, 1, 2, // year=1980, month=february, day=2 11, 5, 0); // hour, minute, second @@ -76,9 +76,9 @@ public class TestDateTools extends LuceneTestCase { } public void testDateAndTimetoString() throws ParseException { - Calendar cal = new GregorianCalendar(); + // we use default locale since LuceneTestCase randomizes it + Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("GMT"), Locale.getDefault()); cal.clear(); - cal.setTimeZone(TimeZone.getTimeZone("GMT")); cal.set(2004, 1, 3, // year=2004, month=february(!), day=3 22, 8, 56); // hour, minute, second cal.set(Calendar.MILLISECOND, 333); @@ -141,9 +141,9 @@ public class TestDateTools extends LuceneTestCase { } public void testRound() { - Calendar cal = new GregorianCalendar(); + // we use default locale since LuceneTestCase randomizes it + Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("GMT"), Locale.getDefault()); cal.clear(); - cal.setTimeZone(TimeZone.getTimeZone("GMT")); cal.set(2004, 1, 3, // year=2004, month=february(!), day=3 22, 8, 56); // hour, minute, second cal.set(Calendar.MILLISECOND, 333); diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCustomSearcherSort.java b/lucene/core/src/test/org/apache/lucene/search/TestCustomSearcherSort.java index 19e38bbbd5e..a5404ff5ca3 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestCustomSearcherSort.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestCustomSearcherSort.java @@ -19,8 +19,10 @@ package org.apache.lucene.search; import java.io.IOException; import java.util.Calendar; import java.util.GregorianCalendar; +import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.TimeZone; import java.util.TreeMap; import org.apache.lucene.document.DateTools; @@ -230,10 +232,12 @@ public class TestCustomSearcherSort extends LuceneTestCase { private class RandomGen { RandomGen(Random random) { this.random = random; + base.set(1980, 1, 1); } private Random random; - private Calendar base = new GregorianCalendar(1980, 1, 1); + // we use the default Locale/TZ since LuceneTestCase randomizes it + private Calendar base = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); // Just to generate some different Lucene Date strings private String getLuceneDate() { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java b/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java index 2b588b737ba..714d2a7cd3f 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/ChainedFilterTest.java @@ -19,6 +19,8 @@ package org.apache.lucene.queries; import java.util.Calendar; import java.util.GregorianCalendar; +import java.util.Locale; +import java.util.TimeZone; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -56,7 +58,8 @@ public class ChainedFilterTest extends LuceneTestCase { super.setUp(); directory = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), directory); - Calendar cal = new GregorianCalendar(); + // we use the default Locale/TZ since LuceneTestCase randomizes it + Calendar cal = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); cal.clear(); cal.setTimeInMillis(1041397200000L); // 2003 January 01 diff --git a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java index 3e62f786936..5ba1d07a416 100644 --- a/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java +++ b/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java @@ -316,7 +316,8 @@ public abstract class QueryParserBase { } /** - * Set locale used by date range parsing. + * Set locale used by date range parsing, lowercasing, and other + * locale-sensitive operations. */ public void setLocale(Locale locale) { this.locale = locale; @@ -658,8 +659,8 @@ public abstract class QueryParserBase { boolean endInclusive) throws ParseException { if (lowercaseExpandedTerms) { - part1 = part1==null ? null : part1.toLowerCase(); - part2 = part2==null ? null : part2.toLowerCase(); + part1 = part1==null ? null : part1.toLowerCase(locale); + part2 = part2==null ? null : part2.toLowerCase(locale); } @@ -935,7 +936,7 @@ public abstract class QueryParserBase { if (!allowLeadingWildcard && (termStr.startsWith("*") || termStr.startsWith("?"))) throw new ParseException("'*' or '?' not allowed as first character in WildcardQuery"); if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(); + termStr = termStr.toLowerCase(locale); } Term t = new Term(field, termStr); return newWildcardQuery(t); @@ -964,7 +965,7 @@ public abstract class QueryParserBase { protected Query getRegexpQuery(String field, String termStr) throws ParseException { if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(); + termStr = termStr.toLowerCase(locale); } Term t = new Term(field, termStr); return newRegexpQuery(t); @@ -998,7 +999,7 @@ public abstract class QueryParserBase { if (!allowLeadingWildcard && termStr.startsWith("*")) throw new ParseException("'*' not allowed as first character in PrefixQuery"); if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(); + termStr = termStr.toLowerCase(locale); } Term t = new Term(field, termStr); return newPrefixQuery(t); @@ -1018,7 +1019,7 @@ public abstract class QueryParserBase { protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException { if (lowercaseExpandedTerms) { - termStr = termStr.toLowerCase(); + termStr = termStr.toLowerCase(locale); } Term t = new Term(field, termStr); return newFuzzyQuery(t, minSimilarity, fuzzyPrefixLength); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java index b542aa2b831..7664b0c770d 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java @@ -26,6 +26,7 @@ import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.TimeZone; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; @@ -393,9 +394,9 @@ public class TestPrecedenceQueryParser extends LuceneTestCase { private String getLocalizedDate(int year, int month, int day, boolean extendLastDate) { - // we use the default Locale since LuceneTestCase randomizes it + // we use the default Locale/TZ since LuceneTestCase randomizes it DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault()); - Calendar calendar = new GregorianCalendar(); + Calendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); calendar.set(year, month, day); if (extendLastDate) { calendar.set(Calendar.HOUR_OF_DAY, 23); @@ -409,7 +410,8 @@ public class TestPrecedenceQueryParser extends LuceneTestCase { public void testDateRange() throws Exception { String startDate = getLocalizedDate(2002, 1, 1, false); String endDate = getLocalizedDate(2002, 1, 4, false); - Calendar endDateExpected = new GregorianCalendar(); + // we use the default Locale/TZ since LuceneTestCase randomizes it + Calendar endDateExpected = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); endDateExpected.set(2002, 1, 4, 23, 59, 59); endDateExpected.set(Calendar.MILLISECOND, 999); final String defaultField = "default"; diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java index dcc6dd79a27..3088b63c375 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.TimeZone; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -694,9 +695,9 @@ public class TestQPHelper extends LuceneTestCase { } private String getLocalizedDate(int year, int month, int day) { - // we use the default Locale since LuceneTestCase randomizes it + // we use the default Locale/TZ since LuceneTestCase randomizes it DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault()); - Calendar calendar = new GregorianCalendar(); + Calendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); calendar.clear(); calendar.set(year, month, day); calendar.set(Calendar.HOUR_OF_DAY, 23); @@ -709,7 +710,8 @@ public class TestQPHelper extends LuceneTestCase { public void testDateRange() throws Exception { String startDate = getLocalizedDate(2002, 1, 1); String endDate = getLocalizedDate(2002, 1, 4); - Calendar endDateExpected = new GregorianCalendar(); + // we use the default Locale/TZ since LuceneTestCase randomizes it + Calendar endDateExpected = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); endDateExpected.clear(); endDateExpected.set(2002, 1, 4, 23, 59, 59); endDateExpected.set(Calendar.MILLISECOND, 999); diff --git a/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java b/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java index e72d91f9da0..aba324e782a 100644 --- a/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java +++ b/lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java @@ -24,6 +24,7 @@ import java.util.Calendar; import java.util.Date; import java.util.GregorianCalendar; import java.util.Locale; +import java.util.TimeZone; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -592,9 +593,9 @@ public abstract class QueryParserTestBase extends LuceneTestCase { } private String getLocalizedDate(int year, int month, int day) { - // we use the default Locale since LuceneTestCase randomizes it + // we use the default Locale/TZ since LuceneTestCase randomizes it DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault()); - Calendar calendar = new GregorianCalendar(); + Calendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); calendar.clear(); calendar.set(year, month, day); calendar.set(Calendar.HOUR_OF_DAY, 23); @@ -607,7 +608,8 @@ public abstract class QueryParserTestBase extends LuceneTestCase { public void testDateRange() throws Exception { String startDate = getLocalizedDate(2002, 1, 1); String endDate = getLocalizedDate(2002, 1, 4); - Calendar endDateExpected = new GregorianCalendar(); + // we use the default Locale/TZ since LuceneTestCase randomizes it + Calendar endDateExpected = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault()); endDateExpected.clear(); endDateExpected.set(2002, 1, 4, 23, 59, 59); endDateExpected.set(Calendar.MILLISECOND, 999); diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java index 5bfede8666f..3031552eb18 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java @@ -35,6 +35,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.List; +import java.util.Locale; import java.util.Vector; import java.util.zip.GZIPInputStream; @@ -149,11 +150,18 @@ public class JaspellTernarySearchTrie { /** The base node in the trie. */ private TSTNode rootNode; + + private final Locale locale; /** * Constructs an empty Ternary Search Trie. */ public JaspellTernarySearchTrie() { + this(Locale.ROOT); + } + + public JaspellTernarySearchTrie(Locale locale) { + this.locale = locale; } // for loading @@ -214,7 +222,7 @@ public class JaspellTernarySearchTrie { occur = Float.parseFloat(word.substring(pos + 1).trim()); word = word.substring(0, pos); } - String key = word.toLowerCase(); + String key = word.toLowerCase(locale); if (rootNode == null) { rootNode = new TSTNode(key.charAt(0), null); } @@ -244,7 +252,7 @@ public class JaspellTernarySearchTrie { if (occur2 != null) { occur += occur2.floatValue(); } - currentNode = getOrCreateNode(word.trim().toLowerCase()); + currentNode = getOrCreateNode(word.trim().toLowerCase(locale)); currentNode.data = occur; } } @@ -384,7 +392,7 @@ public class JaspellTernarySearchTrie { *@return The Float retrieved from the Ternary Search Trie. */ public Float getAndIncrement(String key) { - String key2 = key.trim().toLowerCase(); + String key2 = key.trim().toLowerCase(locale); TSTNode node = getNode(key2); if (node == null) { return null; @@ -765,7 +773,7 @@ public class JaspellTernarySearchTrie { * the Trie. */ public void remove(String key) { - deleteNode(getNode(key.trim().toLowerCase())); + deleteNode(getNode(key.trim().toLowerCase(locale))); } /** diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java b/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java index fa205503fb4..64d54ecb80a 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/AlcoholicMergePolicy.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Calendar; import java.util.GregorianCalendar; +import java.util.Locale; import java.util.Random; import java.util.TimeZone; @@ -45,7 +46,7 @@ public class AlcoholicMergePolicy extends LogMergePolicy { private final Calendar calendar; public AlcoholicMergePolicy(TimeZone tz, Random random) { - this.calendar = new GregorianCalendar(tz); + this.calendar = new GregorianCalendar(tz, Locale.ROOT); this.random = random; maxMergeSize = _TestUtil.nextInt(random, 1024*1024, Integer.MAX_VALUE); }