This commit is contained in:
Karl Wright 2018-04-13 12:28:03 -04:00
commit 8462b134ea
10 changed files with 239 additions and 89 deletions

View File

@ -114,4 +114,12 @@ public final class LuceneLevenshteinDistance implements StringDistance {
}
return ref;
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (null == obj) return false;
return (getClass() == obj.getClass());
}
}

View File

@ -96,6 +96,9 @@ New Features
* SOLR-11336: DocBasedVersionConstraintsProcessorFactory is more extensible and now supports a list of versioned fields.
(versionField config may now be a comma-delimited list). (Michael Braun via David Smiley)
* SOLR-11913: SolrJ SolrParams now implements Iterable<Map.Entry<String, String[]>> and also has a stream() method
using it for convenience. (David Smiley, Tapan Vaishnav)
Bug Fixes
----------------------
@ -151,6 +154,8 @@ Bug Fixes
(Rohit, Varun Thacker)
* SOLR-12150: Fix a test bug in CdcrBidirectionalTest.testBiDir (Steve Rowe, Amrit Sarkar via Varun Thacker)
* SOLR-10513: ConjunctionSolrSpellChecker did not work with LuceneLevenshteinDistance (Amrit Sarkar via James Dyer)
Optimizations
----------------------

View File

@ -16,7 +16,11 @@
*/
package org.apache.solr.handler.dataimport;
import static org.apache.solr.handler.dataimport.DataImporter.IMPORT_CMD;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@ -24,29 +28,26 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.util.plugin.SolrCoreAware;
import java.util.*;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.handler.dataimport.DataImporter.IMPORT_CMD;
/**
* <p>
* Solr Request Handler for data import from databases and REST data sources.
@ -210,18 +211,18 @@ public class DataImportHandler extends RequestHandlerBase implements
rsp.add("statusMessages", importer.getStatusMessages());
}
/** The value is converted to a String or {@code List<String>} if multi-valued. */
private Map<String, Object> getParamsMap(SolrParams params) {
Iterator<String> names = params.getParameterNamesIterator();
Map<String, Object> result = new HashMap<>();
while (names.hasNext()) {
String s = names.next();
String[] val = params.getParams(s);
if (val == null || val.length < 1)
continue;
if (val.length == 1)
result.put(s, val[0]);
else
result.put(s, Arrays.asList(val));
for (Map.Entry<String, String[]> pair : params){
String s = pair.getKey();
String[] val = pair.getValue();
if (val == null || val.length < 1)
continue;
if (val.length == 1)
result.put(s, val[0]);
else
result.put(s, Arrays.asList(val));
}
return result;
}

View File

@ -36,10 +36,10 @@ import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.NamedList;
@ -127,8 +127,9 @@ public abstract class TextResponseWriter implements PushWriter {
// if there get to be enough types, perhaps hashing on the type
// to get a handler might be faster (but types must be exact to do that...)
// (see a patch on LUCENE-3041 for inspiration)
// go in order of most common to least common
// go in order of most common to least common, however some of the more general types like Map belong towards the end
if (val == null) {
writeNull(name);
} else if (val instanceof String) {
@ -170,20 +171,25 @@ public abstract class TextResponseWriter implements PushWriter {
// restricts the fields to write...?
} else if (val instanceof SolrDocumentList) {
writeSolrDocumentList(name, (SolrDocumentList)val, returnFields);
} else if (val instanceof Map) {
writeMap(name, (Map)val, false, true);
} else if (val instanceof NamedList) {
writeNamedList(name, (NamedList)val);
} else if (val instanceof Path) {
writeStr(name, ((Path) val).toAbsolutePath().toString(), true);
} else if (val instanceof IteratorWriter) {
writeIterator((IteratorWriter) val);
} else if (val instanceof Iterable) {
} else if (val instanceof MapWriter) {
writeMap((MapWriter) val);
} else if (val instanceof MapSerializable) {
//todo find a better way to reuse the map more efficiently
writeMap(name, ((MapSerializable) val).toMap(new LinkedHashMap<>()), false, true);
} else if (val instanceof Map) {
writeMap(name, (Map)val, false, true);
} else if (val instanceof Iterator) { // very generic; keep towards the end
writeArray(name, (Iterator) val);
} else if (val instanceof Iterable) { // very generic; keep towards the end
writeArray(name,((Iterable)val).iterator());
} else if (val instanceof Object[]) {
writeArray(name,(Object[])val);
} else if (val instanceof Iterator) {
writeArray(name, (Iterator) val);
} else if (val instanceof byte[]) {
byte[] arr = (byte[])val;
writeByteArr(name, arr, 0, arr.length);
@ -194,13 +200,8 @@ public abstract class TextResponseWriter implements PushWriter {
writeStr(name, val.toString(), true);
} else if (val instanceof WriteableValue) {
((WriteableValue)val).write(name, this);
} else if (val instanceof MapWriter) {
writeMap((MapWriter) val);
} else if (val instanceof MapSerializable) {
//todo find a better way to reuse the map more efficiently
writeMap(name, ((MapSerializable) val).toMap(new LinkedHashMap<>()), false, true);
} else {
// default... for debugging only
// default... for debugging only. Would be nice to "assert false" ?
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
}
}

View File

@ -81,9 +81,8 @@ public class ConjunctionSolrSpellChecker extends SolrSpellChecker {
checkers.add(checker);
}
@SuppressWarnings("unchecked")
@Override
public String init(NamedList config, SolrCore core) {
public String init(@SuppressWarnings("rawtypes") NamedList config, SolrCore core) {
for (int i = 0; i < checkers.size(); i++) {
SolrSpellChecker c = checkers.get(i);
String dn = c.init(config, core);

View File

@ -18,27 +18,50 @@ package org.apache.solr.spelling;
import java.io.IOException;
import org.apache.lucene.search.spell.JaroWinklerDistance;
import org.apache.lucene.search.spell.LevenshteinDistance;
import org.apache.lucene.search.spell.LuceneLevenshteinDistance;
import org.apache.lucene.search.spell.NGramDistance;
import org.apache.lucene.search.spell.StringDistance;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.core.SolrCore;
import org.apache.solr.search.SolrIndexSearcher;
import org.junit.Assert;
import org.junit.Test;
public class ConjunctionSolrSpellCheckerTest extends LuceneTestCase {
public static final Class<?>[] AVAILABLE_DISTANCES = {LevenshteinDistance.class, LuceneLevenshteinDistance.class,
JaroWinklerDistance.class, NGramDistance.class};
@Test
public void test() throws Exception {
ConjunctionSolrSpellChecker cssc = new ConjunctionSolrSpellChecker();
MockSolrSpellChecker levenstein1 = new MockSolrSpellChecker(new LevenshteinDistance());
MockSolrSpellChecker levenstein2 = new MockSolrSpellChecker(new LevenshteinDistance());
MockSolrSpellChecker ngram = new MockSolrSpellChecker(new NGramDistance());
@SuppressWarnings("unchecked")
Class<StringDistance> sameDistance = (Class<StringDistance>) AVAILABLE_DISTANCES[random().nextInt(AVAILABLE_DISTANCES.length)];
cssc.addChecker(levenstein1);
cssc.addChecker(levenstein2);
StringDistance sameDistance1 = sameDistance.newInstance();
StringDistance sameDistance2 = sameDistance.newInstance();
//NGramDistance defaults to 2, so we'll try 3 or 4 to ensure we have one that is not-equal.
StringDistance differentDistance = new NGramDistance(3);
if(sameDistance1.equals(differentDistance)) {
differentDistance = new NGramDistance(4);
if(sameDistance1.equals(differentDistance)) {
fail("Cannot set up test. 2 NGramDistances with different gram sizes should not be equal.");
}
}
Assert.assertEquals("The distance " + sameDistance + " does not properly implement equals.", sameDistance1, sameDistance2);
MockSolrSpellChecker checker1 = new MockSolrSpellChecker(sameDistance1);
MockSolrSpellChecker checker2 = new MockSolrSpellChecker(sameDistance2);
MockSolrSpellChecker checker3 = new MockSolrSpellChecker(differentDistance);
cssc.addChecker(checker1);
cssc.addChecker(checker2);
try {
cssc.addChecker(ngram);
cssc.addChecker(checker3);
fail("ConjunctionSolrSpellChecker should have thrown an exception about non-identical StringDistances.");
} catch (IllegalArgumentException iae) {
// correct behavior

View File

@ -23,7 +23,7 @@ import java.util.Set;
/**
* This class is similar to MultiMapSolrParams except you can edit the
* This class is similar to {@link MultiMapSolrParams} except you can edit the
* parameters after it is initialized. It has helper functions to set/add
* integer and boolean param values.
*
@ -132,12 +132,13 @@ public class ModifiableSolrParams extends SolrParams
return this;
}
public void add(SolrParams params)
{
Iterator<String> names = params.getParameterNamesIterator();
while (names.hasNext()) {
String name = names.next();
set(name, params.getParams(name));
/**
* Add all of the params provided in the parameter to <em>this</em> params. Any current value(s) for the same
* key will be overridden.
*/
public void add(SolrParams params) {
for (Map.Entry<String, String[]> pair: params) {
set(pair.getKey(), pair.getValue());
}
}
@ -205,4 +206,9 @@ public class ModifiableSolrParams extends SolrParams
public String[] getParams(String param) {
return vals.get( param );
}
@Override
public Iterator<Map.Entry<String, String[]>> iterator() {
return vals.entrySet().iterator();
}
}

View File

@ -74,6 +74,11 @@ public class MultiMapSolrParams extends SolrParams {
return map.keySet().iterator();
}
@Override
public Iterator<Map.Entry<String, String[]>> iterator() {
return map.entrySet().iterator();
}
public Map<String,String[]> getMap() { return map; }
/** Returns a MultiMap view of the SolrParams as efficiently as possible. The returned map may or may not be a backing implementation. */
@ -97,10 +102,8 @@ public class MultiMapSolrParams extends SolrParams {
return map;
} else {
Map<String,String[]> map = new HashMap<>();
Iterator<String> iterator = params.getParameterNamesIterator();
while (iterator.hasNext()) {
String name = iterator.next();
map.put(name, params.getParams(name));
for (Map.Entry<String, String[]> pair : params) {
map.put(pair.getKey(), pair.getValue());
}
return map;
}

View File

@ -29,6 +29,8 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.MapWriter;
@ -37,19 +39,28 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
/** SolrParams hold request parameters.
*
*
/**
* SolrParams is designed to hold parameters to Solr, often from the request coming into Solr.
* It's basically a MultiMap of String keys to one or more String values. Neither keys nor values may be null.
* Unlike a general Map/MultiMap, the size is unknown without iterating over each parameter name.
*/
public abstract class SolrParams implements Serializable, MapWriter {
public abstract class SolrParams implements Serializable, MapWriter, Iterable<Map.Entry<String, String[]>> {
/** returns the String value of a param, or null if not set */
/**
* Returns the first String value of a param, or null if not set.
* To get all, call {@link #getParams(String)} instead.
*/
public abstract String get(String param);
/** returns an array of the String values of a param, or null if none */
/** returns an array of the String values of a param, or null if no mapping for the param exists. */
public abstract String[] getParams(String param);
/** returns an Iterator over the parameter names */
/**
* Returns an Iterator over the parameter names.
* If you were to call a getter for this parameter, you should get a non-null value.
* Since you probably want the value, consider using Java 5 for-each style instead for convenience since a SolrParams
* implements {@link Iterable}.
*/
public abstract Iterator<String> getParameterNamesIterator();
/** returns the value of the param, or def if not set */
@ -58,6 +69,64 @@ public abstract class SolrParams implements Serializable, MapWriter {
return val==null ? def : val;
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
//TODO don't call toNamedList; more efficiently implement here
//note: multiple values, if present, are a String[] under 1 key
toNamedList().forEach((k, v) -> {
if (v == null || "".equals(v)) return;
try {
ew.put(k, v);
} catch (IOException e) {
throw new RuntimeException("Error serializing", e);
}
});
}
/** Returns an Iterator of {@code Map.Entry} providing a multi-map view. Treat it as read-only. */
@Override
public Iterator<Map.Entry<String, String[]>> iterator() {
Iterator<String> it = getParameterNamesIterator();
return new Iterator<Map.Entry<String, String[]>>() {
@Override
public boolean hasNext() {
return it.hasNext();
}
@Override
public Map.Entry<String, String[]> next() {
String key = it.next();
return new Map.Entry<String, String[]>() {
@Override
public String getKey() {
return key;
}
@Override
public String[] getValue() {
return getParams(key);
}
@Override
public String[] setValue(String[] newValue) {
throw new UnsupportedOperationException("read-only");
}
@Override
public String toString() {
return getKey() + "=" + Arrays.toString(getValue());
}
};
}
};
}
/** A {@link Stream} view over {@link #iterator()} -- for convenience. Treat it as read-only. */
public Stream<Map.Entry<String, String[]>> stream() {
return StreamSupport.stream(spliterator(), false);
}
// Do we add Map.forEach equivalent too? But it eager-fetches the value, and Iterable<Map.Entry> allows the user
// to only get the value when needed.
/** returns a RequiredSolrParams wrapping this */
public RequiredSolrParams required()
{
@ -439,7 +508,10 @@ public abstract class SolrParams implements Serializable, MapWriter {
return toSolrParams(nl);
}
/** Convert this to a NamedList */
/**
* Convert this to a NamedList of unique keys with either String or String[] values depending on
* how many values there are for the parameter.
*/
public NamedList<Object> toNamedList() {
final SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
@ -549,18 +621,4 @@ public abstract class SolrParams implements Serializable, MapWriter {
}
}
@Override
public void writeMap(EntryWriter ew) throws IOException {
toNamedList().forEach((k, v) -> {
if (v == null || "".equals(v)) return;
try {
ew.put(k, v);
} catch (IOException e) {
throw new RuntimeException("Error serializing", e);
}
});
}
}

View File

@ -40,15 +40,15 @@ public class SolrParamTest extends LuceneTestCase {
assertIterSize("aaa: foo", 1, aaa);
assertIterSize("required aaa: foo", 1, aaa.required());
assertEquals(new String[] { "a1", "a2" }, aaa.getParams("foo"));
assertArrayEquals(new String[] { "a1", "a2" }, aaa.getParams("foo"));
aaa.add("yak", "a3");
assertIterSize("aaa: foo & yak", 2, aaa);
assertIterSize("required aaa: foo & yak", 2, aaa.required());
assertEquals(new String[] { "a1", "a2" }, aaa.getParams("foo"));
assertEquals(new String[] { "a3" }, aaa.getParams("yak"));
assertArrayEquals(new String[] { "a1", "a2" }, aaa.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, aaa.getParams("yak"));
ModifiableSolrParams bbb = new ModifiableSolrParams();
bbb.add("foo", "b1");
@ -58,26 +58,72 @@ public class SolrParamTest extends LuceneTestCase {
assertIterSize("bbb: foo & zot", 2, bbb);
assertIterSize("required bbb: foo & zot", 2, bbb.required());
assertEquals(new String[] { "b1", "b2" }, bbb.getParams("foo"));
assertEquals(new String[] { "b3" }, bbb.getParams("zot"));
assertArrayEquals(new String[] { "b1", "b2" }, bbb.getParams("foo"));
assertArrayEquals(new String[] { "b3" }, bbb.getParams("zot"));
SolrParams def = SolrParams.wrapDefaults(aaa, bbb);
assertIterSize("def: aaa + bbb", 3, def);
assertIterSize("required def: aaa + bbb", 3, def.required());
assertEquals(new String[] { "a1", "a2" }, def.getParams("foo"));
assertEquals(new String[] { "a3" }, def.getParams("yak"));
assertEquals(new String[] { "b3" }, def.getParams("zot"));
assertArrayEquals(new String[] { "a1", "a2" }, def.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, def.getParams("yak"));
assertArrayEquals(new String[] { "b3" }, def.getParams("zot"));
SolrParams append = SolrParams.wrapAppended(aaa, bbb);
assertIterSize("append: aaa + bbb", 3, append);
assertIterSize("required appended: aaa + bbb", 3, append.required());
assertEquals(new String[] { "a1", "a2", "b1", "b2", }, append.getParams("foo"));
assertEquals(new String[] { "a3" }, append.getParams("yak"));
assertEquals(new String[] { "b3" }, append.getParams("zot"));
assertArrayEquals(new String[] { "a1", "a2", "b1", "b2", }, append.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, append.getParams("yak"));
assertArrayEquals(new String[] { "b3" }, append.getParams("zot"));
}
public void testMapEntryIterators() {
ModifiableSolrParams aaa = new ModifiableSolrParams();
aaa.add("foo", "a1");
aaa.add("foo", "a2");
assertIterSize("aaa: foo", 1, aaa);
assertIterSize("required aaa: foo", 1, aaa.required());
assertArrayEquals(new String[] { "a1", "a2" }, aaa.getParams("foo"));
aaa.add("yak", "a3");
assertIterSize("aaa: foo & yak", 2, aaa);
assertIterSize("required aaa: foo & yak", 2, aaa.required());
assertArrayEquals(new String[] { "a1", "a2" }, aaa.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, aaa.getParams("yak"));
ModifiableSolrParams bbb = new ModifiableSolrParams();
bbb.add("foo", "b1");
bbb.add("foo", "b2");
bbb.add("zot", "b3");
assertIterSize("bbb: foo & zot", 2, bbb);
assertIterSize("required bbb: foo & zot", 2, bbb.required());
assertArrayEquals(new String[] { "b1", "b2" }, bbb.getParams("foo"));
assertArrayEquals(new String[] { "b3" }, bbb.getParams("zot"));
SolrParams append = SolrParams.wrapAppended(aaa, bbb);
assertIterSize("append: aaa + bbb", 3, append);
assertIterSize("required appended: aaa + bbb", 3, append.required());
assertArrayEquals(new String[] { "a1", "a2", "b1", "b2", }, append.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, append.getParams("yak"));
assertArrayEquals(new String[] { "b3" }, append.getParams("zot"));
Iterator<Map.Entry<String, String[]>> it = append.iterator();
assertArrayEquals(new String[] { "a1", "a2", "b1", "b2", }, it.next().getValue());
assertArrayEquals(new String[] { "a3" }, it.next().getValue());
assertArrayEquals(new String[] { "b3" }, it.next().getValue());
}
@ -124,16 +170,16 @@ public class SolrParamTest extends LuceneTestCase {
bbb.add("zot", "b3");
SolrParams def = SolrParams.wrapDefaults(aaa, bbb);
assertEquals(new String[] { "a1", "a2" }, def.getParams("foo"));
assertEquals(new String[] { "a3" }, def.getParams("yak"));
assertEquals(new String[] { "b3" }, def.getParams("zot"));
assertArrayEquals(new String[] { "a1", "a2" }, def.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, def.getParams("yak"));
assertArrayEquals(new String[] { "b3" }, def.getParams("zot"));
ModifiableSolrParams combined = new ModifiableSolrParams();
combined.add(def);
assertEquals(new String[] { "a1", "a2" }, combined.getParams("foo"));
assertEquals(new String[] { "a3" }, combined.getParams("yak"));
assertEquals(new String[] { "b3" }, combined.getParams("zot"));
assertArrayEquals(new String[] { "a1", "a2" }, combined.getParams("foo"));
assertArrayEquals(new String[] { "a3" }, combined.getParams("yak"));
assertArrayEquals(new String[] { "b3" }, combined.getParams("zot"));
}