Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4096: FileDataSource & FieldReaderDataSource to default to UTF-8 charset 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1411812 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
James Dyer 2012-11-20 19:38:52 +00:00
parent 105babd077
commit 1f47a886a3
5 changed files with 20 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
solr/CHANGES.txt
View File

@ -258,10 +258,14 @@ Other Changes
* SOLR-3602: Update ZooKeeper to 3.4.5 (Mark Miller)
* SOLR-4095 DIH NumberFormatTransformer & DateFormatTransformer default to the
* SOLR-4095: DIH NumberFormatTransformer & DateFormatTransformer default to the
ROOT Locale if none is specified. These previously used the machine's default.
(James Dyer)
* SOLR-4096: DIH FileDataSource & FieldReaderDataSource default to UTF-8 encoding
if none is specified. These previously used the machine's default.
(James Dyer)
================== 4.0.0 ==================
Versions of Major Components

2
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FieldReaderDataSource.java
View File

@ -106,7 +106,7 @@ public class FieldReaderDataSource extends DataSource<Reader> {
private Reader getReader(Blob blob)
throws SQLException, UnsupportedEncodingException {
if (encoding == null) {
return (new InputStreamReader(blob.getBinaryStream()));
return (new InputStreamReader(blob.getBinaryStream(), "UTF-8"));
} else {
return (new InputStreamReader(blob.getBinaryStream(), encoding));
}

2
solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/FileDataSource.java
View File

@ -125,7 +125,7 @@ public class FileDataSource extends DataSource<Reader> {
protected Reader openStream(File file) throws FileNotFoundException,
UnsupportedEncodingException {
if (encoding == null) {
return new InputStreamReader(new FileInputStream(file));
return new InputStreamReader(new FileInputStream(file), "UTF-8");
} else {
return new InputStreamReader(new FileInputStream(file), encoding);
}

20
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListEntityProcessor.java
View File

@ -41,9 +41,9 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
tmpdir.delete();
tmpdir.mkdir();
tmpdir.deleteOnExit();
createFile(tmpdir, "a.xml", "a.xml".getBytes(), false);
createFile(tmpdir, "b.xml", "b.xml".getBytes(), false);
createFile(tmpdir, "c.props", "c.props".getBytes(), false);
createFile(tmpdir, "a.xml", "a.xml".getBytes("UTF-8"), false);
createFile(tmpdir, "b.xml", "b.xml".getBytes("UTF-8"), false);
createFile(tmpdir, "c.props", "c.props".getBytes("UTF-8"), false);
Map attrs = createMap(
FileListEntityProcessor.FILE_NAME, "xml$",
FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath());
@ -138,9 +138,9 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
tmpdir.delete();
tmpdir.mkdir();
tmpdir.deleteOnExit();
createFile(tmpdir, "a.xml", "a.xml".getBytes(), true);
createFile(tmpdir, "b.xml", "b.xml".getBytes(), true);
createFile(tmpdir, "c.props", "c.props".getBytes(), true);
createFile(tmpdir, "a.xml", "a.xml".getBytes("UTF-8"), true);
createFile(tmpdir, "b.xml", "b.xml".getBytes("UTF-8"), true);
createFile(tmpdir, "c.props", "c.props".getBytes("UTF-8"), true);
Map attrs = createMap(
FileListEntityProcessor.FILE_NAME, "xml$",
FileListEntityProcessor.BASE_DIR, tmpdir.getAbsolutePath(),
@ -162,7 +162,7 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
VariableResolver resolver = new VariableResolver();
String lastMod = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date(System.currentTimeMillis() - 50000));
resolver.addNamespace("a", createMap("x", lastMod));
createFile(tmpdir, "t.xml", "t.xml".getBytes(), false);
createFile(tmpdir, "t.xml", "t.xml".getBytes("UTF-8"), false);
fList = getFiles(resolver, attrs);
assertEquals(1, fList.size());
assertEquals("File name must be t.xml", new File(tmpdir, "t.xml").getAbsolutePath(), fList.get(0));
@ -177,9 +177,9 @@ public class TestFileListEntityProcessor extends AbstractDataImportHandlerTestCa
File childdir = new File(tmpdir + "/child" );
childdir.mkdirs();
childdir.deleteOnExit();
createFile(childdir, "a.xml", "a.xml".getBytes(), true);
createFile(childdir, "b.xml", "b.xml".getBytes(), true);
createFile(childdir, "c.props", "c.props".getBytes(), true);
createFile(childdir, "a.xml", "a.xml".getBytes("UTF-8"), true);
createFile(childdir, "b.xml", "b.xml".getBytes("UTF-8"), true);
createFile(childdir, "c.props", "c.props".getBytes("UTF-8"), true);
Map attrs = createMap(
FileListEntityProcessor.FILE_NAME, "^.*\\.xml$",
FileListEntityProcessor.BASE_DIR, childdir.getAbsolutePath(),

6
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestFileListWithLineEntityProcessor.java
View File

@ -33,9 +33,9 @@ public class TestFileListWithLineEntityProcessor extends AbstractDataImportHandl
tmpdir.delete();
tmpdir.mkdir();
tmpdir.deleteOnExit();
createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes(), false);
createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes(), false);
createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes(), false);
createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes("UTF-8"), false);
createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes("UTF-8"), false);
createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes("UTF-8"), false);
String config = generateConfig(tmpdir);
LocalSolrQueryRequest request = lrf.makeRequest(