merge trunk (1233476:1237067)

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3661@1237068 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-01-28 15:04:12 +00:00
commit df204bc54b
14 changed files with 246 additions and 130 deletions

View File

@ -36,6 +36,11 @@
<build-directory>../build/solr-solrj</build-directory>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-noggit</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>

View File

@ -22,6 +22,7 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.lucene.codecs.SegmentInfosWriter;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.store.ChecksumIndexOutput;
@ -63,6 +64,9 @@ public class PreFlexRWSegmentInfosWriter extends SegmentInfosWriter {
/** Save a single segment's info. */
private void writeInfo(IndexOutput output, SegmentInfo si) throws IOException {
// we are about to write this SI in 3.x format, dropping all codec information, etc.
// so it had better be a 3.x segment or you will get very confusing errors later.
assert si.getCodec() instanceof Lucene3xCodec : "broken test, trying to mix preflex with other codecs";
assert si.getDelCount() <= si.docCount: "delCount=" + si.getDelCount() + " docCount=" + si.docCount + " segment=" + si.name;
// Write the Lucene version that created this segment, since 3.1
output.writeString(si.getVersion());

View File

@ -95,9 +95,9 @@ public abstract class BaseCharFilter extends CharFilter {
diffs = ArrayUtil.grow(diffs);
}
assert (size == 0 || off >= offsets[size])
assert (size == 0 || off >= offsets[size - 1])
: "Offset #" + size + "(" + off + ") is less than the last recorded offset "
+ offsets[size] + "\n" + Arrays.toString(offsets) + "\n" + Arrays.toString(diffs);
+ offsets[size - 1] + "\n" + Arrays.toString(offsets) + "\n" + Arrays.toString(diffs);
if (size == 0 || off != offsets[size - 1]) {
offsets[size] = off;

View File

@ -433,9 +433,9 @@ public class CoreContainer
try {
for (SolrCore core : cores.values()) {
try {
if (!core.isClosed()) {
core.close();
}
core.close();
// make sure we wait for any recoveries to stop
core.getUpdateHandler().getSolrCoreState().cancelRecovery();
} catch (Throwable t) {
SolrException.log(log, "Error shutting down core", t);
}
@ -491,6 +491,9 @@ public class CoreContainer
SolrCore old = null;
synchronized (cores) {
if (isShutDown) {
throw new IllegalStateException("This CoreContainer has been shutdown");
}
old = cores.put(name, core);
/*
* set both the name of the descriptor and the name of the

View File

@ -23,8 +23,10 @@ import java.util.*;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.FastWriter;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
@ -130,20 +132,29 @@ public abstract class TextResponseWriter {
else {
writeStr(name, f.stringValue(), true);
}
} else if (val instanceof Integer) {
writeInt(name, val.toString());
} else if (val instanceof Number) {
if (val instanceof Integer) {
writeInt(name, val.toString());
} else if (val instanceof Long) {
writeLong(name, val.toString());
} else if (val instanceof Float) {
// we pass the float instead of using toString() because
// it may need special formatting. same for double.
writeFloat(name, ((Float)val).floatValue());
} else if (val instanceof Double) {
writeDouble(name, ((Double)val).doubleValue());
} else if (val instanceof Short) {
writeInt(name, val.toString());
} else if (val instanceof Byte) {
writeInt(name, val.toString());
} else {
// default... for debugging only
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
}
} else if (val instanceof Boolean) {
writeBool(name, val.toString());
} else if (val instanceof Long) {
writeLong(name, val.toString());
} else if (val instanceof Date) {
writeDate(name,(Date)val);
} else if (val instanceof Float) {
// we pass the float instead of using toString() because
// it may need special formatting. same for double.
writeFloat(name, ((Float)val).floatValue());
} else if (val instanceof Double) {
writeDouble(name, ((Double)val).doubleValue());
} else if (val instanceof Document) {
SolrDocument doc = toSolrDocument( (Document)val );
DocTransformer transformer = returnFields.getTransformer();
@ -181,6 +192,12 @@ public abstract class TextResponseWriter {
writeArray(name,(Object[])val);
} else if (val instanceof Iterator) {
writeArray(name,(Iterator)val);
} else if (val instanceof byte[]) {
byte[] arr = (byte[])val;
writeByteArr(name, arr, 0, arr.length);
} else if (val instanceof BytesRef) {
BytesRef arr = (BytesRef)val;
writeByteArr(name, arr.bytes, arr.offset, arr.length);
} else {
// default... for debugging only
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
@ -334,4 +351,7 @@ public abstract class TextResponseWriter {
/** if this form of the method is called, val is the Solr ISO8601 based date format */
public abstract void writeDate(String name, String val) throws IOException;
public void writeByteArr(String name, byte[] buf, int offset, int len) throws IOException {
writeStr(name, Base64.byteArrayToBase64(buf, offset, len), false);
}
}

View File

@ -30,7 +30,9 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
@Ignore
public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest {
@BeforeClass

View File

@ -55,8 +55,7 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
@AfterClass
public static void afterClass() throws InterruptedException {
// wait just a bit for any zk client threads to outlast timeout
Thread.sleep(2000);
}
@Override
@ -87,10 +86,18 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
public ClientThread(int nodeNumber) throws Exception {
super("Thread-" + nodeNumber);
boolean created = false;
this.zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
this.zkStateReader = new ZkStateReader(zkClient);
this.nodeNumber = nodeNumber;
props = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, Integer.toString(nodeNumber), ZkStateReader.CORE_NAME_PROP, "");
try {
this.zkStateReader = new ZkStateReader(zkClient);
this.nodeNumber = nodeNumber;
props = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, Integer.toString(nodeNumber), ZkStateReader.CORE_NAME_PROP, "");
created = true;
} finally {
if (!created) {
zkClient.close();
}
}
}
@Override
@ -281,13 +288,13 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
threads.add(thread1);
scheduler.schedule(thread1, 0, TimeUnit.MILLISECONDS);
Thread.sleep(4000);
Thread.sleep(2000);
Thread scheduleThread = new Thread() {
@Override
public void run() {
for (int i = 1; i < atLeast(15); i++) {
int count = atLeast(5);
for (int i = 1; i < count; i++) {
int launchIn = random.nextInt(500);
ClientThread thread = null;
try {
@ -362,7 +369,7 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
connLossThread.start();
killThread.start();
Thread.sleep(6000);
Thread.sleep(4000);
stopStress = true;
@ -371,14 +378,14 @@ public class LeaderElectionTest extends SolrTestCaseJ4 {
killThread.interrupt();
scheduleThread.join();
scheduler.shutdownNow();
connLossThread.join();
killThread.join();
scheduler.shutdownNow();
int seq = threads.get(getLeaderThread()).getSeq();
assertFalse("seq is -1 and we may have a zombie leader", seq == -1);
// we have a leader we know, TODO: lets check some other things
// cleanup any threads still running
for (ClientThread thread : threads) {

View File

@ -41,7 +41,7 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
}
@Test
public void testNaNInf() throws IOException {
public void testTypes() throws IOException {
SolrQueryRequest req = req("dummy");
SolrQueryResponse rsp = new SolrQueryResponse();
QueryResponseWriter w = new PythonResponseWriter();
@ -77,8 +77,12 @@ public class JSONWriterTest extends SolrTestCaseJ4 {
nl.add(null, 42);
rsp.add("nl", nl);
rsp.add("byte", Byte.valueOf((byte)-3));
rsp.add("short", Short.valueOf((short)-4));
rsp.add("bytes", "abc".getBytes("UTF-8"));
w.write(buf, req, rsp);
assertEquals("{\"nl\":[[\"data1\",\"he\\u2028llo\\u2029!\"],[null,42]]}", buf.toString());
assertEquals("{\"nl\":[[\"data1\",\"he\\u2028llo\\u2029!\"],[null,42]],\"byte\":-3,\"short\":-4,\"bytes\":\"YWJj\"}", buf.toString());
req.close();
}

View File

@ -31,6 +31,7 @@
<mkdir dir="${dist}/solrj-lib" />
<copy todir="${dist}/solrj-lib">
<fileset dir="${common-solr.dir}/lib">
<include name="apache-solr-noggit-*.jar"/>
<include name="commons-codec-*.jar"/>
<include name="commons-io-*.jar"/>
<include name="commons-httpclient-*.jar"/>

View File

@ -118,7 +118,6 @@ public class SolrZkClient {
public SolrZkClient(String zkServerAddress, int zkClientTimeout,
ZkClientConnectionStrategy strat, final OnReconnect onReconnect, int clientConnectTimeout) throws InterruptedException,
TimeoutException, IOException {
numOpens.incrementAndGet();
connManager = new ConnectionManager("ZooKeeperConnection Watcher:"
+ zkServerAddress, this, zkServerAddress, zkClientTimeout, strat, onReconnect);
strat.connect(zkServerAddress, zkClientTimeout, connManager,
@ -141,6 +140,7 @@ public class SolrZkClient {
}
});
connManager.waitForConnected(clientConnectTimeout);
numOpens.incrementAndGet();
}
/**

View File

@ -31,15 +31,14 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.noggit.CharArr;
import org.apache.noggit.JSONParser;
import org.apache.noggit.JSONWriter;
import org.apache.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.ByteUtils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@ -90,17 +89,18 @@ public class ZkStateReader {
}
public static byte[] toUTF8(CharArr out) {
BytesRef br = new BytesRef(out);
return Arrays.copyOf(br.bytes, br.length);
byte[] arr = new byte[out.size() << 2]; // is 4x the real worst-case upper-bound?
int nBytes = ByteUtils.UTF16toUTF8(out, 0, out.size(), arr, 0);
return Arrays.copyOf(arr, nBytes);
}
public static Object fromJSON(byte[] utf8) {
// convert directly from bytes to chars
// and parse directly from that instead of going through
// intermediate strings or readers
CharsRef chars = new CharsRef();
UnicodeUtil.UTF8toUTF16(utf8, 0, utf8.length, chars); // TODO: this method currently oversizes the array
JSONParser parser = new JSONParser(chars.chars, chars.offset, chars.length);
CharArr chars = new CharArr();
ByteUtils.UTF8toUTF16(utf8, 0, utf8.length, chars);
JSONParser parser = new JSONParser(chars.getArray(), chars.getStart(), chars.length());
try {
return ObjectBuilder.getVal(parser);
} catch (IOException e) {

View File

@ -0,0 +1,126 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.common.util;
import org.apache.noggit.CharArr;
public class ByteUtils {
/** Converts utf8 to utf16 and returns the number of 16 bit Java chars written.
* Full characters are read, even if this reads past the length passed (and can result in
* an ArrayOutOfBoundsException if invalid UTF8 is passed). Explicit checks for valid UTF8 are not performed.
* The char[] out should probably have enough room to hold the worst case of each byte becoming a Java char.
*/
public static int UTF8toUTF16(byte[] utf8, int offset, int len, char[] out, int out_offset) {
int out_start = out_offset;
final int limit = offset + len;
while (offset < limit) {
int b = utf8[offset++]&0xff;
if (b < 0xc0) {
assert b < 0x80;
out[out_offset++] = (char)b;
} else if (b < 0xe0) {
out[out_offset++] = (char)(((b&0x1f)<<6) + (utf8[offset++]&0x3f));
} else if (b < 0xf0) {
out[out_offset++] = (char)(((b&0xf)<<12) + ((utf8[offset]&0x3f)<<6) + (utf8[offset+1]&0x3f));
offset += 2;
} else {
assert b < 0xf8;
int ch = ((b&0x7)<<18) + ((utf8[offset]&0x3f)<<12) + ((utf8[offset+1]&0x3f)<<6) + (utf8[offset+2]&0x3f);
offset += 3;
if (ch < 0xffff) {
out[out_offset++] = (char)ch;
} else {
int chHalf = ch - 0x0010000;
out[out_offset++] = (char) ((chHalf >> 10) + 0xD800);
out[out_offset++] = (char) ((chHalf & 0x3FFL) + 0xDC00);
}
}
}
return out_offset - out_start;
}
/** Convert UTF8 bytes into UTF16 characters. */
public static void UTF8toUTF16(byte[] utf8, int offset, int len, CharArr out) {
// TODO: do in chunks if the input is large
out.reserve(len);
int n = UTF8toUTF16(utf8, offset, len, out.getArray(), out.getEnd());
out.setEnd(out.getEnd() + n);
}
/** Convert UTF8 bytes into a String */
public static String UTF8toUTF16(byte[] utf8, int offset, int len) {
char[] out = new char[len];
int n = UTF8toUTF16(utf8, offset, len, out, 0);
return new String(out,0,n);
}
/** Writes UTF8 into the byte array, starting at offset. The caller should ensure that
* there is enough space for the worst-case scenario.
* @return the number of bytes written
*/
public static int UTF16toUTF8(CharSequence s, int offset, int len, byte[] result, int resultOffset) {
final int end = offset + len;
int upto = resultOffset;
for(int i=offset;i<end;i++) {
final int code = (int) s.charAt(i);
if (code < 0x80)
result[upto++] = (byte) code;
else if (code < 0x800) {
result[upto++] = (byte) (0xC0 | (code >> 6));
result[upto++] = (byte)(0x80 | (code & 0x3F));
} else if (code < 0xD800 || code > 0xDFFF) {
result[upto++] = (byte)(0xE0 | (code >> 12));
result[upto++] = (byte)(0x80 | ((code >> 6) & 0x3F));
result[upto++] = (byte)(0x80 | (code & 0x3F));
} else {
// surrogate pair
// confirm valid high surrogate
if (code < 0xDC00 && (i < end-1)) {
int utf32 = (int) s.charAt(i+1);
// confirm valid low surrogate and write pair
if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) {
utf32 = ((code - 0xD7C0) << 10) + (utf32 & 0x3FF);
i++;
result[upto++] = (byte)(0xF0 | (utf32 >> 18));
result[upto++] = (byte)(0x80 | ((utf32 >> 12) & 0x3F));
result[upto++] = (byte)(0x80 | ((utf32 >> 6) & 0x3F));
result[upto++] = (byte)(0x80 | (utf32 & 0x3F));
continue;
}
}
// replace unpaired surrogate or out-of-order low surrogate
// with substitution character
result[upto++] = (byte) 0xEF;
result[upto++] = (byte) 0xBF;
result[upto++] = (byte) 0xBD;
}
}
return upto - resultOffset;
}
}

View File

@ -16,6 +16,7 @@
*/
package org.apache.solr.common.util;
import org.apache.noggit.CharArr;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
@ -463,80 +464,23 @@ public class JavaBinCodec {
int end = s.length();
int maxSize = end * 4;
if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize];
int upto = 0;
for(int i=0;i<end;i++) {
final int code = (int) s.charAt(i);
int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0);
if (code < 0x80)
bytes[upto++] = (byte) code;
else if (code < 0x800) {
bytes[upto++] = (byte) (0xC0 | (code >> 6));
bytes[upto++] = (byte)(0x80 | (code & 0x3F));
} else if (code < 0xD800 || code > 0xDFFF) {
bytes[upto++] = (byte)(0xE0 | (code >> 12));
bytes[upto++] = (byte)(0x80 | ((code >> 6) & 0x3F));
bytes[upto++] = (byte)(0x80 | (code & 0x3F));
} else {
// surrogate pair
// confirm valid high surrogate
if (code < 0xDC00 && (i < end-1)) {
int utf32 = (int) s.charAt(i+1);
// confirm valid low surrogate and write pair
if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) {
utf32 = ((code - 0xD7C0) << 10) + (utf32 & 0x3FF);
i++;
bytes[upto++] = (byte)(0xF0 | (utf32 >> 18));
bytes[upto++] = (byte)(0x80 | ((utf32 >> 12) & 0x3F));
bytes[upto++] = (byte)(0x80 | ((utf32 >> 6) & 0x3F));
bytes[upto++] = (byte)(0x80 | (utf32 & 0x3F));
continue;
}
}
// replace unpaired surrogate or out-of-order low surrogate
// with substitution character
bytes[upto++] = (byte) 0xEF;
bytes[upto++] = (byte) 0xBF;
bytes[upto++] = (byte) 0xBD;
}
}
writeTag(STR, upto);
daos.write(bytes, 0, upto);
writeTag(STR, sz);
daos.write(bytes, 0, sz);
}
byte[] bytes;
char[] chars;
CharArr arr = new CharArr();
public String readStr(FastInputStream dis) throws IOException {
int sz = readSize(dis);
if (chars == null || chars.length < sz) chars = new char[sz];
if (bytes == null || bytes.length < sz) bytes = new byte[sz];
dis.readFully(bytes, 0, sz);
int outUpto=0;
for (int i = 0; i < sz;) {
final int b = bytes[i++]&0xff;
final int ch;
if (b < 0xc0) {
assert b < 0x80;
ch = b;
} else if (b < 0xe0) {
ch = ((b&0x1f)<<6) + (bytes[i++]&0x3f);
} else if (b < 0xf0) {
ch = ((b&0xf)<<12) + ((bytes[i++]&0x3f)<<6) + (bytes[i++]&0x3f);
} else {
assert b < 0xf8;
ch = ((b&0x7)<<18) + ((bytes[i++]&0x3f)<<12) + ((bytes[i++]&0x3f)<<6) + (bytes[i++]&0x3f);
}
if (ch <= 0xFFFF) {
// target is a character <= 0xFFFF
chars[outUpto++] = (char) ch;
} else {
// target is a character in range 0xFFFF - 0x10FFFF
final int chHalf = ch - 0x10000;
chars[outUpto++] = (char) ((chHalf >> 0xA) + 0xD800);
chars[outUpto++] = (char) ((chHalf & 0x3FF) + 0xDC00);
}
}
return new String(chars, 0, outUpto);
arr.reset();
ByteUtils.UTF8toUTF16(bytes, 0, sz, arr);
return arr.toString();
}
public void writeInt(int val) throws IOException {
@ -600,15 +544,32 @@ public class JavaBinCodec {
} else if (val instanceof String) {
writeStr((String) val);
return true;
} else if (val instanceof Integer) {
writeInt(((Integer) val).intValue());
return true;
} else if (val instanceof Long) {
writeLong(((Long) val).longValue());
return true;
} else if (val instanceof Float) {
writeFloat(((Float) val).floatValue());
return true;
} else if (val instanceof Number) {
if (val instanceof Integer) {
writeInt(((Integer) val).intValue());
return true;
} else if (val instanceof Long) {
writeLong(((Long) val).longValue());
return true;
} else if (val instanceof Float) {
writeFloat(((Float) val).floatValue());
return true;
} else if (val instanceof Double) {
daos.writeByte(DOUBLE);
daos.writeDouble(((Double) val).doubleValue());
return true;
} else if (val instanceof Byte) {
daos.writeByte(BYTE);
daos.writeByte(((Byte) val).intValue());
return true;
} else if (val instanceof Short) {
daos.writeByte(SHORT);
daos.writeShort(((Short) val).intValue());
return true;
}
return false;
} else if (val instanceof Date) {
daos.writeByte(DATE);
daos.writeLong(((Date) val).getTime());
@ -617,18 +578,6 @@ public class JavaBinCodec {
if ((Boolean) val) daos.writeByte(BOOL_TRUE);
else daos.writeByte(BOOL_FALSE);
return true;
} else if (val instanceof Double) {
daos.writeByte(DOUBLE);
daos.writeDouble(((Double) val).doubleValue());
return true;
} else if (val instanceof Byte) {
daos.writeByte(BYTE);
daos.writeByte(((Byte) val).intValue());
return true;
} else if (val instanceof Short) {
daos.writeByte(SHORT);
daos.writeShort(((Short) val).intValue());
return true;
} else if (val instanceof byte[]) {
writeByteArray((byte[]) val, 0, ((byte[]) val).length);
return true;

View File

@ -115,11 +115,6 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
if (numOpens != 0 || numCloses != 0) {
// NOTE: some other tests don't use this base class and hence won't reset the counts.
log.warn("startTrackingSearchers: numOpens="+numOpens+" numCloses="+numCloses);
try {
throw new RuntimeException();
} catch (Exception e) {
log.error("",e);
}
numOpens = numCloses = 0;
}
}
@ -137,7 +132,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
// wait a bit in case any ending threads have anything to release
int retries = 0;
while (endNumOpens - numOpens != endNumCloses - numCloses) {
if (retries++ > 30) {
if (retries++ > 60) {
break;
}
try {