Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix lots of default charset violations in Solr caused by String.getBytes() and IOUtils.toString() [and others] 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1067160 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2011-02-04 12:01:49 +00:00
parent 6f31407109
commit 0188da6a73
12 changed files with 47 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDataConfig.java
View File

@ -19,9 +19,10 @@ package org.apache.solr.handler.dataimport;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;
import org.xml.sax.InputSource;
import javax.xml.parsers.DocumentBuilderFactory;
import java.io.ByteArrayInputStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
@ -55,7 +56,7 @@ public class TestDataConfig extends AbstractDataImportHandlerTestCase {
public void testBasic() throws Exception {
javax.xml.parsers.DocumentBuilder builder = DocumentBuilderFactory
.newInstance().newDocumentBuilder();
Document doc = builder.parse(new ByteArrayInputStream(xml.getBytes()));
Document doc = builder.parse(new InputSource(new StringReader(xml)));
DataConfig dc = new DataConfig();
dc.readFromXml(doc.getDocumentElement());

2
solr/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -232,7 +232,7 @@ public class CoreContainer
cores.load(solrHome, fconf);
} else {
log.info("no solr.xml file found - using default");
cores.load(solrHome, new ByteArrayInputStream(DEF_SOLR_XML.getBytes()));
cores.load(solrHome, new ByteArrayInputStream(DEF_SOLR_XML.getBytes("UTF-8")));
cores.configFile = fconf;
}

8
solr/src/java/org/apache/solr/handler/DumpRequestHandler.java
View File

@ -18,7 +18,7 @@
package org.apache.solr.handler;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.util.ArrayList;
import org.apache.commons.io.IOUtils;
@ -46,11 +46,11 @@ public class DumpRequestHandler extends RequestHandlerBase
stream.add( "sourceInfo", content.getSourceInfo() );
stream.add( "size", content.getSize() );
stream.add( "contentType", content.getContentType() );
InputStream is = content.getStream();
Reader reader = content.getReader();
try {
stream.add( "stream", IOUtils.toString(is) );
stream.add( "stream", IOUtils.toString(reader) );
} finally {
is.close();
reader.close();
}
streams.add( stream );
}

2
solr/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java
View File

@ -210,7 +210,7 @@ public class ShowFileRequestHandler extends RequestHandlerBase
}
try {
InputStream input = core.getResourceLoader().openResource(path);
return IOUtils.toString( input );
return IOUtils.toString( input, "UTF-8" );
}
catch( Exception ex ) {} // ignore it
return "";

1
solr/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java
View File

@ -185,6 +185,7 @@ public class SystemInfoHandler extends RequestHandlerBase
try {
Process process = Runtime.getRuntime().exec(cmd);
in = new DataInputStream( process.getInputStream() );
// use default charset from locale here, because the command invoked also uses the default locale:
return IOUtils.toString( in );
}
catch( Exception ex ) {

20
solr/src/test/org/apache/solr/client/solrj/embedded/JettyWebappTest.java
View File

@ -86,23 +86,23 @@ public class JettyWebappTest extends LuceneTestCase
// sure they compile ok
String adminPath = "http://localhost:"+port+context+"/";
String html = IOUtils.toString( new URL(adminPath).openStream() );
assertNotNull( html ); // real error will be an exception
byte[] bytes = IOUtils.toByteArray( new URL(adminPath).openStream() );
assertNotNull( bytes ); // real error will be an exception
adminPath += "admin/";
html = IOUtils.toString( new URL(adminPath).openStream() );
assertNotNull( html ); // real error will be an exception
bytes = IOUtils.toByteArray( new URL(adminPath).openStream() );
assertNotNull( bytes ); // real error will be an exception
// analysis
html = IOUtils.toString( new URL(adminPath+"analysis.jsp").openStream() );
assertNotNull( html ); // real error will be an exception
bytes = IOUtils.toByteArray( new URL(adminPath+"analysis.jsp").openStream() );
assertNotNull( bytes ); // real error will be an exception
// schema browser
html = IOUtils.toString( new URL(adminPath+"schema.jsp").openStream() );
assertNotNull( html ); // real error will be an exception
bytes = IOUtils.toByteArray( new URL(adminPath+"schema.jsp").openStream() );
assertNotNull( bytes ); // real error will be an exception
// schema browser
html = IOUtils.toString( new URL(adminPath+"threaddump.jsp").openStream() );
assertNotNull( html ); // real error will be an exception
bytes = IOUtils.toByteArray( new URL(adminPath+"threaddump.jsp").openStream() );
assertNotNull( bytes ); // real error will be an exception
}
}

2
solr/src/test/org/apache/solr/cloud/ZkTestServer.java
View File

@ -279,7 +279,7 @@ public class ZkTestServer {
BufferedReader reader = null;
try {
OutputStream outstream = sock.getOutputStream();
outstream.write(cmd.getBytes());
outstream.write(cmd.getBytes("US-ASCII"));
outstream.flush();
// this replicates NC - close the output stream before reading
sock.shutdownOutput();

21
solr/src/test/org/apache/solr/common/util/ContentStreamTest.java
View File

@ -27,6 +27,7 @@ import java.io.InputStream;
import java.io.StringReader;
import java.net.ConnectException;
import java.net.URL;
import java.net.URLConnection;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -64,12 +65,15 @@ public class ContentStreamTest extends LuceneTestCase
public void testURLStream() throws IOException
{
String content = null;
byte[] content = null;
String contentType = null;
URL url = new URL( "http://svn.apache.org/repos/asf/lucene/dev/trunk/" );
InputStream in = null;
try {
in = url.openStream();
content = IOUtils.toString( in );
URLConnection conn = url.openConnection();
in = conn.getInputStream();
contentType = conn.getContentType();
content = IOUtils.toByteArray(in);
} catch (ConnectException ex) {
assumeNoException("Unable to connect to " + url + " to run the test.", ex);
}finally {
@ -78,23 +82,26 @@ public class ContentStreamTest extends LuceneTestCase
}
}
assertTrue( content.length() > 10 ); // found something...
assertTrue( content.length > 10 ); // found something...
ContentStreamBase stream = new ContentStreamBase.URLStream( url );
assertEquals( content.length(), stream.getSize().intValue() );
assertEquals( content.length, stream.getSize().intValue() );
// Test the stream
in = stream.getStream();
try {
assertTrue( IOUtils.contentEquals(
new ByteArrayInputStream( content.getBytes() ), in ) );
new ByteArrayInputStream(content), in ) );
}
finally {
IOUtils.closeQuietly(in);
}
String charset = ContentStreamBase.getCharsetFromContentType(contentType);
if (charset == null)
charset = ContentStreamBase.DEFAULT_CHARSET;
// Re-open the stream and this time use a reader
stream = new ContentStreamBase.URLStream( url );
assertTrue( IOUtils.contentEquals( new StringReader( content ), stream.getReader() ) );
assertTrue( IOUtils.contentEquals( new StringReader(new String(content, charset)), stream.getReader() ) );
}
}

5
solr/src/test/org/apache/solr/common/util/DOMUtilTest.java
View File

@ -17,7 +17,7 @@
package org.apache.solr.common.util;
import java.io.ByteArrayInputStream;
import java.io.StringReader;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@ -27,6 +27,7 @@ import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.apache.lucene.util.LuceneTestCase;
@ -85,6 +86,6 @@ public class DOMUtilTest extends LuceneTestCase {
}
public Document getDocument( String xml ) throws Exception {
return builder.parse( new ByteArrayInputStream( xml.getBytes() ) );
return builder.parse(new InputSource(new StringReader(xml)));
}
}

2
solr/src/test/org/apache/solr/handler/DocumentAnalysisRequestHandlerTest.java
View File

@ -177,7 +177,7 @@ public class DocumentAnalysisRequestHandlerTest extends AnalysisRequestHandlerTe
"</docs>"
).getBytes("ISO-8859-1");
// we declare a content stream without charset:
// we declare a content stream with charset:
final ContentStream cs = new ByteStream(xmlBytes, "application/xml; charset=ISO-8859-1");
ModifiableSolrParams params = new ModifiableSolrParams();

14
solr/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
View File

@ -74,7 +74,7 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
List<ContentStream> streams = new ArrayList<ContentStream>();
SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 1, streams.size() );
assertEquals( body1, IOUtils.toString( streams.get(0).getStream() ) );
assertEquals( body1, IOUtils.toString( streams.get(0).getReader() ) );
req.close();
// Now add three and make sure they come out ok
@ -87,9 +87,9 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
input.add( body1 );
input.add( body2 );
input.add( body3 );
output.add( IOUtils.toString( streams.get(0).getStream() ) );
output.add( IOUtils.toString( streams.get(1).getStream() ) );
output.add( IOUtils.toString( streams.get(2).getStream() ) );
output.add( IOUtils.toString( streams.get(0).getReader() ) );
output.add( IOUtils.toString( streams.get(1).getReader() ) );
output.add( IOUtils.toString( streams.get(2).getReader() ) );
// sort them so the output is consistent
Collections.sort( input );
Collections.sort( output );
@ -112,13 +112,13 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
{
boolean ok = false;
String url = "http://www.apache.org/dist/lucene/solr/";
String txt = null;
byte[] bytes = null;
try {
URLConnection connection = new URL(url).openConnection();
connection.setConnectTimeout(5000);
connection.setReadTimeout(5000);
connection.connect();
txt = IOUtils.toString( connection.getInputStream());
bytes = IOUtils.toByteArray( connection.getInputStream());
}
catch( Exception ex ) {
assumeNoException("Unable to connect to " + url + " to run the test.", ex);
@ -134,7 +134,7 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
List<ContentStream> streams = new ArrayList<ContentStream>();
SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 1, streams.size() );
assertEquals( txt, IOUtils.toString( streams.get(0).getStream() ) );
assertArrayEquals( bytes, IOUtils.toByteArray( streams.get(0).getStream() ) );
req.close();
}

2
solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -65,7 +65,7 @@ public class SolrDispatchFilter implements Filter
public SolrDispatchFilter() {
try {
adminRequestParser = new SolrRequestParsers(new Config(null,"solr",new ByteArrayInputStream("<root/>".getBytes()),"") );
adminRequestParser = new SolrRequestParsers(new Config(null,"solr",new ByteArrayInputStream("<root/>".getBytes("UTF-8")),"") );
} catch (Exception e) {
//unlikely
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,e);