mirror of https://github.com/apache/lucene.git
SOLR-8453: test HTTP client error responses
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1723613 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley
parent
96360c78c4
commit
fa640e6b69
|
|
@ -0,0 +1,385 @@
|
||||||
|
package org.apache.solr.client.solrj;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import java.io.BufferedInputStream;
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.lang.invoke.MethodHandles;
|
||||||
|
import java.lang.reflect.Array;
|
||||||
|
import java.net.HttpURLConnection;
|
||||||
|
import java.net.Socket;
|
||||||
|
import java.net.URL;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList;
|
||||||
|
import java.util.concurrent.ThreadPoolExecutor;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.solr.SolrJettyTestBase;
|
||||||
|
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
|
||||||
|
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
|
||||||
|
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||||
|
import org.apache.solr.client.solrj.request.RequestWriter;
|
||||||
|
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||||
|
import org.apache.solr.common.SolrDocument;
|
||||||
|
import org.apache.solr.common.SolrInputDocument;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
|
||||||
|
public class TestSolrJErrorHandling extends SolrJettyTestBase {
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||||
|
|
||||||
|
List<Throwable> unexpected = new CopyOnWriteArrayList<>();
|
||||||
|
|
||||||
|
|
||||||
|
@BeforeClass
|
||||||
|
public static void beforeTest() throws Exception {
|
||||||
|
createJetty(legacyExampleCollection1SolrHome());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
unexpected.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getChain(Throwable th) {
|
||||||
|
StringBuilder sb = new StringBuilder(40);
|
||||||
|
Throwable lastCause = null;
|
||||||
|
do {
|
||||||
|
if (lastCause != null) sb.append("->");
|
||||||
|
sb.append(th.getClass().getSimpleName());
|
||||||
|
lastCause = th;
|
||||||
|
th = th.getCause();
|
||||||
|
} while(th != null);
|
||||||
|
sb.append("(" + lastCause.getMessage() + ")");
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void showExceptions() throws Exception {
|
||||||
|
if (unexpected.isEmpty()) return;
|
||||||
|
|
||||||
|
Map<String,Integer> counts = new HashMap<>();
|
||||||
|
|
||||||
|
// dedup in case there are many clients or many exceptions
|
||||||
|
for (Throwable e : unexpected) {
|
||||||
|
String chain = getChain(e);
|
||||||
|
Integer prev = counts.put(chain, 1);
|
||||||
|
if (prev != null) {
|
||||||
|
counts.put(chain, prev+1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
StringBuilder sb = new StringBuilder("EXCEPTION LIST:");
|
||||||
|
for (Map.Entry<String,Integer> entry : counts.entrySet()) {
|
||||||
|
sb.append("\n\t").append(entry.getValue()).append(") ").append(entry.getKey());
|
||||||
|
}
|
||||||
|
|
||||||
|
log.error(sb.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWithXml() throws Exception {
|
||||||
|
HttpSolrClient client = (HttpSolrClient) getSolrClient();
|
||||||
|
client.setRequestWriter(new RequestWriter());
|
||||||
|
client.deleteByQuery("*:*"); // delete everything!
|
||||||
|
doIt(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testWithBinary() throws Exception {
|
||||||
|
HttpSolrClient client = (HttpSolrClient) getSolrClient();
|
||||||
|
client.setRequestWriter(new BinaryRequestWriter());
|
||||||
|
client.deleteByQuery("*:*"); // delete everything!
|
||||||
|
doIt(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
Iterator<SolrInputDocument> manyDocs(final int base, final int numDocs) {
|
||||||
|
return new Iterator<SolrInputDocument>() {
|
||||||
|
int count = 0;
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return count < numDocs;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SolrInputDocument next() {
|
||||||
|
int id = base + count++;
|
||||||
|
if (count == 1) { // first doc is legit, and will increment a counter
|
||||||
|
return sdoc("id","test", "count_i", map("inc",1));
|
||||||
|
}
|
||||||
|
// include "ignore_exception" so the log doesn't fill up with known exceptions, and change the values for each doc
|
||||||
|
// so binary format won't compress too much
|
||||||
|
return sdoc("id",Integer.toString(id),"ignore_exception_field_does_not_exist_"+id,"fieldval"+id);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void remove() {
|
||||||
|
}
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
void doThreads(final HttpSolrClient client, final int numThreads, final int numRequests) throws Exception {
|
||||||
|
final AtomicInteger tries = new AtomicInteger(0);
|
||||||
|
|
||||||
|
List<Thread> threads = new ArrayList<>();
|
||||||
|
|
||||||
|
for (int i=0; i<numThreads; i++) {
|
||||||
|
final int threadNum = i;
|
||||||
|
threads.add( new Thread() {
|
||||||
|
int reqLeft = numRequests;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run() {
|
||||||
|
try {
|
||||||
|
while (--reqLeft >= 0) {
|
||||||
|
tries.incrementAndGet();
|
||||||
|
doSingle(client, threadNum);
|
||||||
|
}
|
||||||
|
} catch (Throwable e) {
|
||||||
|
// Allow thread to exit, we should have already recorded the exception.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Thread thread : threads) {
|
||||||
|
thread.start();
|
||||||
|
}
|
||||||
|
for (Thread thread : threads) {
|
||||||
|
thread.join();
|
||||||
|
}
|
||||||
|
|
||||||
|
showExceptions();
|
||||||
|
|
||||||
|
int count = getCount(client);
|
||||||
|
if (count > tries.get()) {
|
||||||
|
fail("Number of requests was " + tries.get() + " but final count was " + count);
|
||||||
|
}
|
||||||
|
|
||||||
|
assertEquals(tries.get(), getCount(client));
|
||||||
|
|
||||||
|
assertTrue("got unexpected exceptions. ", unexpected.isEmpty() );
|
||||||
|
}
|
||||||
|
|
||||||
|
int getCount(HttpSolrClient client) throws IOException, SolrServerException {
|
||||||
|
client.commit();
|
||||||
|
QueryResponse rsp = client.query(params("q", "id:test", "fl", "count_i", "wt", "json"));
|
||||||
|
int count = ((Number)rsp.getResults().get(0).get("count_i")).intValue();
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// this always failed with the Jetty 9.3 snapshot
|
||||||
|
void doIt(HttpSolrClient client) throws Exception {
|
||||||
|
client.deleteByQuery("*:*");
|
||||||
|
doThreads(client,10,100);
|
||||||
|
// doSingle(client, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
void doSingle(HttpSolrClient client, int threadNum) {
|
||||||
|
try {
|
||||||
|
client.add(manyDocs(threadNum*1000000, 1000));
|
||||||
|
}
|
||||||
|
catch (HttpSolrClient.RemoteSolrException e) {
|
||||||
|
String msg = e.getMessage();
|
||||||
|
assertTrue(msg, msg.contains("field_does_not_exist"));
|
||||||
|
}
|
||||||
|
catch (Throwable e) {
|
||||||
|
unexpected.add(e);
|
||||||
|
log.error("unexpected exception:", e);
|
||||||
|
fail("FAILING unexpected exception: " + e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/***
|
||||||
|
@Test
|
||||||
|
public void testLive() throws Exception {
|
||||||
|
HttpSolrClient client = new HttpSolrClient("http://localhost:8983/techproducts/solr/");
|
||||||
|
client.add( sdoc() );
|
||||||
|
doiIt(client);
|
||||||
|
}
|
||||||
|
***/
|
||||||
|
|
||||||
|
String getJsonDocs(int numDocs) {
|
||||||
|
StringBuilder sb = new StringBuilder(numDocs * 20);
|
||||||
|
sb.append("[");
|
||||||
|
for (int i = 0; i < numDocs; i++) {
|
||||||
|
sb.append("{ id : '" + i + "' , unknown_field_" + i + " : 'unknown field value' }");
|
||||||
|
}
|
||||||
|
sb.append("]");
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
char[] whitespace(int n) {
|
||||||
|
char[] arr = new char[n];
|
||||||
|
Arrays.fill(arr, ' ');
|
||||||
|
return arr;
|
||||||
|
}
|
||||||
|
|
||||||
|
String getResponse(InputStream is) throws Exception {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
byte[] buf = new byte[100000];
|
||||||
|
for (;;) {
|
||||||
|
int n = 0;
|
||||||
|
try {
|
||||||
|
n = is.read(buf);
|
||||||
|
} catch (IOException e) {
|
||||||
|
// a real HTTP client probably wouldn't try to read past the end and would thus
|
||||||
|
// not get an exception until the *next* http request.
|
||||||
|
log.error("CAUGHT IOException, but already read " + sb.length() + " : " + getChain(e));
|
||||||
|
}
|
||||||
|
if (n <= 0) break;
|
||||||
|
sb.append(new String(buf, 0, n, StandardCharsets.UTF_8));
|
||||||
|
log.info("BUFFER=" + sb.toString());
|
||||||
|
break; // for now, assume we got whole response in one read... otherwise we could block when trying to read again
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testHttpURLConnection() throws Exception {
|
||||||
|
|
||||||
|
String bodyString = getJsonDocs(200000); // sometimes succeeds with this size, but larger can cause OOM from command line
|
||||||
|
|
||||||
|
HttpSolrClient client = (HttpSolrClient) getSolrClient();
|
||||||
|
|
||||||
|
String urlString = client.getBaseURL() + "/update";
|
||||||
|
|
||||||
|
HttpURLConnection conn = null;
|
||||||
|
URL url = new URL(urlString);
|
||||||
|
|
||||||
|
conn = (HttpURLConnection) url.openConnection();
|
||||||
|
conn.setRequestMethod("POST");
|
||||||
|
conn.setDoOutput(true);
|
||||||
|
conn.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
|
||||||
|
|
||||||
|
OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream(), "UTF-8");
|
||||||
|
writer.write(bodyString);
|
||||||
|
writer.flush();
|
||||||
|
|
||||||
|
int code = 1;
|
||||||
|
try {
|
||||||
|
code = conn.getResponseCode();
|
||||||
|
} catch (Throwable th) {
|
||||||
|
log.error("ERROR DURING conn.getResponseCode():",th);
|
||||||
|
}
|
||||||
|
|
||||||
|
/***
|
||||||
|
java.io.IOException: Error writing to server
|
||||||
|
at __randomizedtesting.SeedInfo.seed([2928C6EE314CD076:947A81A74F582526]:0)
|
||||||
|
at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:665)
|
||||||
|
at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:677)
|
||||||
|
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1533)
|
||||||
|
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1440)
|
||||||
|
at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480)
|
||||||
|
*/
|
||||||
|
|
||||||
|
log.info("CODE=" + code);
|
||||||
|
InputStream is;
|
||||||
|
if (code == 200) {
|
||||||
|
is = conn.getInputStream();
|
||||||
|
} else {
|
||||||
|
log.info("Attempting to get error stream.");
|
||||||
|
is = conn.getErrorStream();
|
||||||
|
if (is == null) {
|
||||||
|
log.info("Can't get error stream... try input stream?");
|
||||||
|
is = conn.getInputStream();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
String rbody = IOUtils.toString(is, StandardCharsets.UTF_8);
|
||||||
|
log.info("RESPONSE BODY:" + rbody);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testRawSocket() throws Exception {
|
||||||
|
|
||||||
|
String hostName = "127.0.0.1";
|
||||||
|
int port = jetty.getLocalPort();
|
||||||
|
|
||||||
|
Socket socket = new Socket(hostName, port);
|
||||||
|
PrintWriter out = new PrintWriter(socket.getOutputStream(), true);
|
||||||
|
// BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream()));
|
||||||
|
InputStream in = socket.getInputStream();
|
||||||
|
|
||||||
|
String body = getJsonDocs(100000);
|
||||||
|
int bodyLen = body.length();
|
||||||
|
|
||||||
|
// bodyLen *= 10; // make server wait for more
|
||||||
|
|
||||||
|
char[] whitespace = whitespace(1000000);
|
||||||
|
bodyLen += whitespace.length;
|
||||||
|
|
||||||
|
String headers = "POST /solr/collection1/update HTTP/1.1\n" +
|
||||||
|
"Host: localhost:" + port + "\n" +
|
||||||
|
// "User-Agent: curl/7.43.0\n" +
|
||||||
|
"Accept: */*\n" +
|
||||||
|
"Content-type:application/json\n" +
|
||||||
|
"Content-Length: " + bodyLen + "\n" +
|
||||||
|
"Connection: Keep-Alive\n";
|
||||||
|
|
||||||
|
out.write(headers);
|
||||||
|
out.write("\n"); // extra newline separates headers from body
|
||||||
|
out.write(body);
|
||||||
|
out.flush();
|
||||||
|
|
||||||
|
// Now what if I try to write more? This doesn't seem to throw an exception!
|
||||||
|
Thread.sleep(1000);
|
||||||
|
out.write(whitespace); // whitespace
|
||||||
|
out.flush();
|
||||||
|
|
||||||
|
String rbody = getResponse(in); // This will throw a connection reset exception if you try to read past the end of the HTTP response
|
||||||
|
log.info("RESPONSE BODY:" + rbody);
|
||||||
|
assertTrue(rbody.contains("unknown_field"));
|
||||||
|
|
||||||
|
/***
|
||||||
|
// can I reuse now?
|
||||||
|
// writing another request doesn't actually throw an exception, but the following read does
|
||||||
|
out.write(headers);
|
||||||
|
out.write("\n"); // extra newline separates headers from body
|
||||||
|
out.write(body);
|
||||||
|
out.flush();
|
||||||
|
|
||||||
|
rbody = getResponse(in);
|
||||||
|
log.info("RESPONSE BODY:" + rbody);
|
||||||
|
assertTrue(rbody.contains("unknown_field"));
|
||||||
|
***/
|
||||||
|
|
||||||
|
|
||||||
|
IOUtils.closeQuietly(out);
|
||||||
|
IOUtils.closeQuietly(in);
|
||||||
|
IOUtils.closeQuietly(socket);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue