diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java b/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java new file mode 100644 index 00000000000..7051df2feff --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/TestSolrJErrorHandling.java @@ -0,0 +1,385 @@ +package org.apache.solr.client.solrj; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Array; +import java.net.HttpURLConnection; +import java.net.Socket; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.io.IOUtils; +import org.apache.solr.SolrJettyTestBase; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.impl.BinaryRequestWriter; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrInputDocument; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") +public class TestSolrJErrorHandling extends SolrJettyTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + List unexpected = new CopyOnWriteArrayList<>(); + + + @BeforeClass + public static void beforeTest() throws Exception { + createJetty(legacyExampleCollection1SolrHome()); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + unexpected.clear(); + } + + public String getChain(Throwable th) { + StringBuilder sb = new StringBuilder(40); + Throwable lastCause = null; + do { + if (lastCause != null) sb.append("->"); + sb.append(th.getClass().getSimpleName()); + lastCause = th; + th = th.getCause(); + } while(th != null); + sb.append("(" + lastCause.getMessage() + ")"); + return sb.toString(); + } + + public void showExceptions() throws Exception { + if (unexpected.isEmpty()) return; + + Map counts = new HashMap<>(); + + // dedup in case there are many clients or many exceptions + for (Throwable e : unexpected) { + String chain = getChain(e); + Integer prev = counts.put(chain, 1); + if (prev != null) { + counts.put(chain, prev+1); + } + } + + StringBuilder sb = new StringBuilder("EXCEPTION LIST:"); + for (Map.Entry entry : counts.entrySet()) { + sb.append("\n\t").append(entry.getValue()).append(") ").append(entry.getKey()); + } + + log.error(sb.toString()); + } + + @Test + public void testWithXml() throws Exception { + HttpSolrClient client = (HttpSolrClient) getSolrClient(); + client.setRequestWriter(new RequestWriter()); + client.deleteByQuery("*:*"); // delete everything! + doIt(client); + } + + @Test + public void testWithBinary() throws Exception { + HttpSolrClient client = (HttpSolrClient) getSolrClient(); + client.setRequestWriter(new BinaryRequestWriter()); + client.deleteByQuery("*:*"); // delete everything! + doIt(client); + } + + Iterator manyDocs(final int base, final int numDocs) { + return new Iterator() { + int count = 0; + @Override + public boolean hasNext() { + return count < numDocs; + } + + @Override + public SolrInputDocument next() { + int id = base + count++; + if (count == 1) { // first doc is legit, and will increment a counter + return sdoc("id","test", "count_i", map("inc",1)); + } + // include "ignore_exception" so the log doesn't fill up with known exceptions, and change the values for each doc + // so binary format won't compress too much + return sdoc("id",Integer.toString(id),"ignore_exception_field_does_not_exist_"+id,"fieldval"+id); + } + + @Override + public void remove() { + } + }; + }; + + void doThreads(final HttpSolrClient client, final int numThreads, final int numRequests) throws Exception { + final AtomicInteger tries = new AtomicInteger(0); + + List threads = new ArrayList<>(); + + for (int i=0; i= 0) { + tries.incrementAndGet(); + doSingle(client, threadNum); + } + } catch (Throwable e) { + // Allow thread to exit, we should have already recorded the exception. + } + } + } + ); + } + + for (Thread thread : threads) { + thread.start(); + } + for (Thread thread : threads) { + thread.join(); + } + + showExceptions(); + + int count = getCount(client); + if (count > tries.get()) { + fail("Number of requests was " + tries.get() + " but final count was " + count); + } + + assertEquals(tries.get(), getCount(client)); + + assertTrue("got unexpected exceptions. ", unexpected.isEmpty() ); + } + + int getCount(HttpSolrClient client) throws IOException, SolrServerException { + client.commit(); + QueryResponse rsp = client.query(params("q", "id:test", "fl", "count_i", "wt", "json")); + int count = ((Number)rsp.getResults().get(0).get("count_i")).intValue(); + return count; + } + + // this always failed with the Jetty 9.3 snapshot + void doIt(HttpSolrClient client) throws Exception { + client.deleteByQuery("*:*"); + doThreads(client,10,100); + // doSingle(client, 1); + } + + void doSingle(HttpSolrClient client, int threadNum) { + try { + client.add(manyDocs(threadNum*1000000, 1000)); + } + catch (HttpSolrClient.RemoteSolrException e) { + String msg = e.getMessage(); + assertTrue(msg, msg.contains("field_does_not_exist")); + } + catch (Throwable e) { + unexpected.add(e); + log.error("unexpected exception:", e); + fail("FAILING unexpected exception: " + e); + } + } + + /*** + @Test + public void testLive() throws Exception { + HttpSolrClient client = new HttpSolrClient("http://localhost:8983/techproducts/solr/"); + client.add( sdoc() ); + doiIt(client); + } + ***/ + + String getJsonDocs(int numDocs) { + StringBuilder sb = new StringBuilder(numDocs * 20); + sb.append("["); + for (int i = 0; i < numDocs; i++) { + sb.append("{ id : '" + i + "' , unknown_field_" + i + " : 'unknown field value' }"); + } + sb.append("]"); + return sb.toString(); + } + + char[] whitespace(int n) { + char[] arr = new char[n]; + Arrays.fill(arr, ' '); + return arr; + } + + String getResponse(InputStream is) throws Exception { + StringBuilder sb = new StringBuilder(); + byte[] buf = new byte[100000]; + for (;;) { + int n = 0; + try { + n = is.read(buf); + } catch (IOException e) { + // a real HTTP client probably wouldn't try to read past the end and would thus + // not get an exception until the *next* http request. + log.error("CAUGHT IOException, but already read " + sb.length() + " : " + getChain(e)); + } + if (n <= 0) break; + sb.append(new String(buf, 0, n, StandardCharsets.UTF_8)); + log.info("BUFFER=" + sb.toString()); + break; // for now, assume we got whole response in one read... otherwise we could block when trying to read again + } + return sb.toString(); + } + + @Test + public void testHttpURLConnection() throws Exception { + + String bodyString = getJsonDocs(200000); // sometimes succeeds with this size, but larger can cause OOM from command line + + HttpSolrClient client = (HttpSolrClient) getSolrClient(); + + String urlString = client.getBaseURL() + "/update"; + + HttpURLConnection conn = null; + URL url = new URL(urlString); + + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); + + OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream(), "UTF-8"); + writer.write(bodyString); + writer.flush(); + + int code = 1; + try { + code = conn.getResponseCode(); + } catch (Throwable th) { + log.error("ERROR DURING conn.getResponseCode():",th); + } + +/*** + java.io.IOException: Error writing to server + at __randomizedtesting.SeedInfo.seed([2928C6EE314CD076:947A81A74F582526]:0) + at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:665) + at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:677) + at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1533) + at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1440) + at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480) + */ + + log.info("CODE=" + code); + InputStream is; + if (code == 200) { + is = conn.getInputStream(); + } else { + log.info("Attempting to get error stream."); + is = conn.getErrorStream(); + if (is == null) { + log.info("Can't get error stream... try input stream?"); + is = conn.getInputStream(); + } + } + + String rbody = IOUtils.toString(is, StandardCharsets.UTF_8); + log.info("RESPONSE BODY:" + rbody); + } + + @Test + public void testRawSocket() throws Exception { + + String hostName = "127.0.0.1"; + int port = jetty.getLocalPort(); + + Socket socket = new Socket(hostName, port); + PrintWriter out = new PrintWriter(socket.getOutputStream(), true); + // BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream())); + InputStream in = socket.getInputStream(); + + String body = getJsonDocs(100000); + int bodyLen = body.length(); + + // bodyLen *= 10; // make server wait for more + + char[] whitespace = whitespace(1000000); + bodyLen += whitespace.length; + + String headers = "POST /solr/collection1/update HTTP/1.1\n" + + "Host: localhost:" + port + "\n" + +// "User-Agent: curl/7.43.0\n" + + "Accept: */*\n" + + "Content-type:application/json\n" + + "Content-Length: " + bodyLen + "\n" + + "Connection: Keep-Alive\n"; + + out.write(headers); + out.write("\n"); // extra newline separates headers from body + out.write(body); + out.flush(); + + // Now what if I try to write more? This doesn't seem to throw an exception! + Thread.sleep(1000); + out.write(whitespace); // whitespace + out.flush(); + + String rbody = getResponse(in); // This will throw a connection reset exception if you try to read past the end of the HTTP response + log.info("RESPONSE BODY:" + rbody); + assertTrue(rbody.contains("unknown_field")); + + /*** + // can I reuse now? + // writing another request doesn't actually throw an exception, but the following read does + out.write(headers); + out.write("\n"); // extra newline separates headers from body + out.write(body); + out.flush(); + + rbody = getResponse(in); + log.info("RESPONSE BODY:" + rbody); + assertTrue(rbody.contains("unknown_field")); + ***/ + + + IOUtils.closeQuietly(out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(socket); + } + + +}