LUCENE-4199: more default locale/encoding fixes

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4199@1358635 13f79535-47bb-0310-9956-ffa450edef68

lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java

@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.hunspell;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -330,7 +331,7 @@ public class HunspellStemmer {
     
     HunspellStemmer stemmer = new HunspellStemmer(dictionary);
 
-    Scanner scanner = new Scanner(System.in);
+    Scanner scanner = new Scanner(System.in, Charset.defaultCharset().name());
     
     System.out.print("> ");
     while (scanner.hasNextLine()) {

lucene/analysis/stempel/src/test/org/egothor/stemmer/TestCompile.java

@@ -60,13 +60,14 @@ import java.io.BufferedReader;
 import java.io.DataInputStream;
 import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.net.URI;
 import java.util.Locale;
 import java.util.StringTokenizer;
 
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 
 public class TestCompile extends LuceneTestCase {
@@ -121,7 +122,7 @@ public class TestCompile extends LuceneTestCase {
   private static void assertTrie(Trie trie, String file, boolean usefull,
       boolean storeorig) throws Exception {
     LineNumberReader in = new LineNumberReader(new BufferedReader(
-        new FileReader(file)));
+        new InputStreamReader(new FileInputStream(file), IOUtils.CHARSET_UTF_8)));
     
     for (String line = in.readLine(); line != null; line = in.readLine()) {
       try {

lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/utils/StreamUtilsTest.java

@@ -31,6 +31,7 @@ import java.io.OutputStreamWriter;
 import org.apache.commons.compress.compressors.CompressorStreamFactory;
 import org.apache.lucene.benchmark.BenchmarkTestCase;
 import org.apache.lucene.benchmark.byTask.utils.StreamUtils;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util._TestUtil;
 import org.junit.After;
 import org.junit.Before;
@@ -88,7 +89,7 @@ public class StreamUtilsTest extends BenchmarkTestCase {
   
   private File rawTextFile(String ext) throws Exception {
     File f = new File(testDir,"testfile." +	ext);
-    BufferedWriter w = new BufferedWriter(new FileWriter(f));
+    BufferedWriter w = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), IOUtils.CHARSET_UTF_8));
     w.write(TEXT);
     w.newLine();
     w.close();
@@ -117,7 +118,7 @@ public class StreamUtilsTest extends BenchmarkTestCase {
   }
 
 	private void writeText(OutputStream os) throws IOException {
-		BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));
+		BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os, IOUtils.CHARSET_UTF_8));
   	w.write(TEXT);
   	w.newLine();
   	w.close();
@@ -125,7 +126,7 @@ public class StreamUtilsTest extends BenchmarkTestCase {
 
   private void assertReadText(File f) throws Exception {
     InputStream ir = StreamUtils.inputStream(f);
-    InputStreamReader in = new InputStreamReader(ir);
+    InputStreamReader in = new InputStreamReader(ir, IOUtils.CHARSET_UTF_8);
     BufferedReader r = new BufferedReader(in);
     String line = r.readLine();
     assertEquals("Wrong text found in "+f.getName(), TEXT, line);

lucene/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java

@@ -31,7 +31,9 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
+import java.nio.charset.Charset;
 
 /**
  * Test that quality run does its job.
@@ -55,7 +57,7 @@ public class TestQualityRun extends BenchmarkTestCase {
     int maxResults = 1000;
     String docNameField = "doctitle"; // orig docID is in the linedoc format title 
     
-    PrintWriter logger = VERBOSE ? new PrintWriter(System.out,true) : null;
+    PrintWriter logger = VERBOSE ? new PrintWriter(new OutputStreamWriter(System.out, Charset.defaultCharset()),true) : null;
    
     // prepare topics
     InputStream topics = getClass().getResourceAsStream("trecTopics.txt");

lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java

@@ -20,6 +20,7 @@ package org.apache.lucene.codecs;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
 import java.util.Comparator;
 import java.util.Iterator;
 import java.util.Locale;
@@ -346,7 +347,12 @@ public class BlockTreeTermsReader extends FieldsProducer {
     @Override
     public String toString() {
       final ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
-      final PrintStream out = new PrintStream(bos);
+      PrintStream out;
+      try {
+        out = new PrintStream(bos, false, "UTF-8");
+      } catch (UnsupportedEncodingException bogus) {
+        throw new RuntimeException(bogus);
+      }
       
       out.println("  index FST:");
       out.println("    " + indexNodeCount + " nodes");
@@ -379,7 +385,11 @@ public class BlockTreeTermsReader extends FieldsProducer {
         assert totalBlockCount == total;
       }
 
-      return bos.toString();
+      try {
+        return bos.toString("UTF-8");
+      } catch (UnsupportedEncodingException bogus) {
+        throw new RuntimeException(bogus);
+      }
     }
   }
 

lucene/demo/src/java/org/apache/lucene/demo/xmlparser/FormBasedXmlQueryDemo.java

@@ -48,6 +48,7 @@ import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.Version;
 
 /**
@@ -131,7 +132,7 @@ public class FormBasedXmlQueryDemo extends HttpServlet {
     IndexWriterConfig iwConfig = new IndexWriterConfig(Version.LUCENE_40, analyzer);
     IndexWriter writer = new IndexWriter(rd, iwConfig);
     InputStream dataIn = getServletContext().getResourceAsStream("/WEB-INF/data.tsv");
-    BufferedReader br = new BufferedReader(new InputStreamReader(dataIn));
+    BufferedReader br = new BufferedReader(new InputStreamReader(dataIn, IOUtils.CHARSET_UTF_8));
     String line = br.readLine();
     final FieldType textNoNorms = new FieldType(TextField.TYPE_STORED);
     textNoNorms.setOmitNorms(true);

lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java

@@ -32,6 +32,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Locale;
 
 public class TestBlockJoin extends LuceneTestCase {
 
@@ -638,7 +639,7 @@ public class TestBlockJoin extends LuceneTestCase {
           int childId = Integer.parseInt(document.get("childID"));
           assertTrue(explanation.isMatch());
           assertEquals(hit.score, explanation.getValue(), 0.0f);
-          assertEquals(String.format("Score based on child doc range from %d to %d", hit.doc - 1 - childId, hit.doc - 1), explanation.getDescription());
+          assertEquals(String.format(Locale.ROOT, "Score based on child doc range from %d to %d", hit.doc - 1 - childId, hit.doc - 1), explanation.getDescription());
         }
       }