Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

speed up response writers: SOLR-377 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@584574 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2007-10-14 18:38:54 +00:00
parent cc005c832e
commit df97bd903c
7 changed files with 191 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -150,6 +150,8 @@ Optimizations
4. SOLR-354: Optimize removing all documents. Now when a delete by query
of *:* is issued, the current index is removed. (yonik)
5. SOLR-377: Speed up response writers. (yonik)
Bug Fixes
1. Make TextField respect sortMissingFirst and sortMissingLast fields.
(J.J. Larrea via yonik)

128
src/java/org/apache/solr/common/util/FastWriter.java Executable file
View File

@ -0,0 +1,128 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.util;
import java.io.Writer;
import java.io.IOException;
/** Single threaded BufferedWriter
* Internal Solr use only, subject to change.
*/
public class FastWriter extends Writer {
// use default BUFSIZE of BufferedWriter so if we wrap that
// it won't cause double buffering.
private static final int BUFSIZE = 8192;
private final Writer sink;
private final char[] buf;
private int pos;
public FastWriter(Writer w) {
this(w, new char[BUFSIZE], 0);
}
public FastWriter(Writer sink, char[] tempBuffer, int start) {
this.sink = sink;
this.buf = tempBuffer;
this.pos = start;
}
public static FastWriter wrap(Writer sink) {
return (sink instanceof FastWriter) ? (FastWriter)sink : new FastWriter(sink);
}
@Override
public void write(int c) throws IOException {
write((char)c);
}
public void write(char c) throws IOException {
if (pos >= buf.length) {
sink.write(buf,0,pos);
pos=0;
}
buf[pos++] = (char)c;
}
@Override
public FastWriter append(char c) throws IOException {
if (pos >= buf.length) {
sink.write(buf,0,pos);
pos=0;
}
buf[pos++] = (char)c;
return this;
}
@Override
public void write(char cbuf[], int off, int len) throws IOException {
int space = buf.length - pos;
if (len < space) {
System.arraycopy(cbuf, off, buf, pos, len);
pos += len;
} else if (len<BUFSIZE) {
// if the data to write is small enough, buffer it.
System.arraycopy(cbuf, off, buf, pos, space);
sink.write(buf, 0, buf.length);
pos = len-space;
System.arraycopy(cbuf, off+space, buf, 0, pos);
} else {
sink.write(buf,0,pos); // flush
pos=0;
// don't buffer, just write to sink
sink.write(cbuf, off, len);
}
}
@Override
public void write(String str, int off, int len) throws IOException {
int space = buf.length - pos;
if (len < space) {
str.getChars(off, off+len, buf, pos);
pos += len;
} else if (len<BUFSIZE) {
// if the data to write is small enough, buffer it.
str.getChars(off, off+space, buf, pos);
sink.write(buf, 0, buf.length);
str.getChars(off+space, off+len, buf, 0);
pos = len-space;
} else {
sink.write(buf,0,pos); // flush
pos=0;
// don't buffer, just write to sink
sink.write(str, off, len);
}
}
@Override
public void flush() throws IOException {
sink.write(buf,0,pos);
pos=0;
sink.flush();
}
@Override
public void close() throws IOException {
flush();
sink.close();
}
public void flushBuffer() throws IOException {
sink.write(buf, 0, pos);
pos=0;
}
}

65
src/java/org/apache/solr/common/util/XML.java
View File

@ -87,11 +87,13 @@ public class XML {
out.write('<');
out.write(tag);
if (val == null) {
out.write("/>");
out.write('/');
out.write('>');
} else {
out.write('>');
escapeCharData(val,out);
out.write("</");
out.write('<');
out.write('/');
out.write(tag);
out.write('>');
}
@ -104,16 +106,19 @@ public class XML {
for (int i=0; i<attrs.length; i++) {
out.write(' ');
out.write(attrs[i++].toString());
out.write("=\"");
out.write('=');
out.write('"');
out.write(attrs[i].toString());
out.write("\"");
out.write('"');
}
if (val == null) {
out.write("/>");
out.write('/');
out.write('>');
} else {
out.write('>');
out.write(val);
out.write("</");
out.write('<');
out.write('/');
out.write(tag);
out.write('>');
}
@ -126,16 +131,19 @@ public class XML {
for (int i=0; i<attrs.length; i++) {
out.write(' ');
out.write(attrs[i++].toString());
out.write("=\"");
out.write('=');
out.write('"');
escapeAttributeValue(attrs[i].toString(), out);
out.write("\"");
out.write('"');
}
if (val == null) {
out.write("/>");
out.write('/');
out.write('>');
} else {
out.write('>');
escapeCharData(val,out);
out.write("</");
out.write('<');
out.write('/');
out.write(tag);
out.write('>');
}
@ -143,41 +151,16 @@ public class XML {
private static void escape(String str, Writer out, String[] escapes) throws IOException {
int start=0;
// "n" was used for counting the chars added to out...
// removed cause it wasn't really useful so far.
// int n=0;
for (int i=start; i<str.length(); i++) {
for (int i=0; i<str.length(); i++) {
char ch = str.charAt(i);
// since I already received the char, what if I put it into
// a char array and wrote that to the stream instead of the
// string? (would cause extra GC though)
String subst=null;
if (ch<escapes.length) {
subst=escapes[ch];
}
if (subst != null) {
if (start<i) {
out.write(str.substring(start,i));
// write(str,off,len) causes problems for Jetty with chars > 127
//out.write(str, start, i-start);
// n+=i-start;
}
out.write(subst);
// n+=subst.length();
start=i+1;
String replacement = escapes[ch];
if (replacement != null) {
out.write(replacement);
continue;
}
}
if (start==0) {
out.write(str);
// n += str.length();
} else if (start<str.length()) {
out.write(str.substring(start));
// write(str,off,len) causes problems for Jetty with chars > 127
// out.write(str, start, str.length()-start);
// n += str.length()-start;
out.write(ch);
}
// return n;
}
}

49
src/java/org/apache/solr/request/JSONResponseWriter.java
View File

@ -81,8 +81,9 @@ class JSONWriter extends TextResponseWriter {
}
writeNamedList(null, rsp.getValues());
if(wrapperFunction!=null) {
writer.write(")");
writer.write(')');
}
writer.flushBuffer();
}
protected void writeKey(String fname, boolean needsEscaping) throws IOException {
@ -496,36 +497,37 @@ class JSONWriter extends TextResponseWriter {
escaped: quotation mark, reverse solidus, and the control
characters (U+0000 through U+001F).
*/
StringBuilder sb = new StringBuilder(val.length()+8);
sb.append('"');
writer.write('"');
for (int i=0; i<val.length(); i++) {
char ch = val.charAt(i);
if ((ch > '#' && ch != '\\') || ch==' ') { // fast path
writer.write(ch);
continue;
}
switch(ch) {
case '"':
case '\\':
sb.append('\\');
sb.append(ch);
writer.write('\\');
writer.write(ch);
break;
case '\r': sb.append('\\').append('r'); break;
case '\n': sb.append('\\').append('n'); break;
case '\t': sb.append('\\').append('t'); break;
case '\b': sb.append('\\').append('b'); break;
case '\f': sb.append('\\').append('f'); break;
case '\r': writer.write('\\'); writer.write('r'); break;
case '\n': writer.write('\\'); writer.write('n'); break;
case '\t': writer.write('\\'); writer.write('t'); break;
case '\b': writer.write('\\'); writer.write('b'); break;
case '\f': writer.write('\\'); writer.write('f'); break;
// case '/':
default: {
if (ch <= 0x1F) {
unicodeEscape(sb,ch);
unicodeEscape(writer,ch);
} else {
sb.append(ch);
writer.write(ch);
}
}
}
}
sb.append('"');
writer.append(sb);
writer.write('"');
} else {
writer.write('"');
writer.write(val);
@ -673,15 +675,14 @@ class JSONWriter extends TextResponseWriter {
writeStr(name, val, false);
}
protected static void unicodeEscape(Appendable sb, int ch) throws IOException {
String str = Integer.toHexString(ch & 0xffff);
switch (str.length()) {
case 1: sb.append("\\u000"); break;
case 2: sb.append("\\u00"); break;
case 3: sb.append("\\u0"); break;
default: sb.append("\\u"); break;
}
sb.append(str);
private static char[] hexdigits = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
protected static void unicodeEscape(Appendable out, int ch) throws IOException {
out.append('\\');
out.append('u');
out.append(hexdigits[(ch>>>12) ]);
out.append(hexdigits[(ch>>>8) & 0xf]);
out.append(hexdigits[(ch>>>4) & 0xf]);
out.append(hexdigits[(ch) & 0xf]);
}
}

5
src/java/org/apache/solr/request/PythonResponseWriter.java
View File

@ -95,7 +95,10 @@ class PythonWriter extends JSONWriter {
}
}
writer.write( needUnicode ? "u'" : "'");
if (needUnicode) {
writer.write('u');
}
writer.write('\'');
writer.append(sb);
writer.write('\'');
}

12
src/java/org/apache/solr/request/RubyResponseWriter.java
View File

@ -51,7 +51,8 @@ class RubyWriter extends JSONWriter {
@Override
protected void writeKey(String fname, boolean needsEscaping) throws IOException {
writeStr(null, fname, needsEscaping);
writer.write("=>");
writer.write('=');
writer.write('>');
}
@Override
@ -63,16 +64,13 @@ class RubyWriter extends JSONWriter {
// Also, there are very few escapes recognized in a single quoted string, so
// only escape the backslash and single quote.
writer.write('\'');
// it might be more efficient to use a stringbuilder or write substrings
// if writing chars to the stream is slow.
if (needsEscaping) {
for (int i=0; i<val.length(); i++) {
char ch = val.charAt(i);
switch(ch) {
case '\'':
case '\\': writer.write('\\'); writer.write(ch); break;
default: writer.write(ch); break;
if (ch=='\'' || ch=='\\') {
writer.write('\\');
}
writer.write(ch);
}
} else {
writer.write(val);

5
src/java/org/apache/solr/request/TextResponseWriter.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.request;
import org.apache.lucene.document.Document;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.FastWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.DocList;
import java.io.IOException;
@ -34,7 +35,7 @@ import java.util.Set;
*/
public abstract class TextResponseWriter {
protected final Writer writer;
protected final FastWriter writer;
protected final IndexSchema schema;
protected final SolrQueryRequest req;
protected final SolrQueryResponse rsp;
@ -47,7 +48,7 @@ public abstract class TextResponseWriter {
public TextResponseWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
this.writer = writer;
this.writer = FastWriter.wrap(writer);
this.schema = req.getSchema();
this.req = req;
this.rsp = rsp;