From 65c842f9faed1c9de9dce38a247a43c36b82873f Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Thu, 4 Jan 2018 13:17:25 +0000 Subject: [PATCH] SOLR-11801: Support customisation of the highlighting query response element. (Ramsey Haddad, Pranav Murugappan, David Smiley, Christine Poerschke) --- solr/CHANGES.txt | 3 + .../handler/component/HighlightComponent.java | 41 ++- .../CustomHighlightComponentTest.java | 310 ++++++++++++++++++ 3 files changed, 348 insertions(+), 6 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 7137b8079d2..61551e0983f 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -113,6 +113,9 @@ Other Changes * SOLR-11798: Formally deprecate top-level syntax in solrconfig.xml in favour of equivalent syntax. (Christine Poerschke) +* SOLR-11801: Support customisation of the "highlighting" query response element. + (Ramsey Haddad, Pranav Murugappan, David Smiley, Christine Poerschke) + ================== 7.2.0 ================== Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release. diff --git a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java index 698cc052612..1602502f492 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java @@ -187,7 +187,7 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni if(sumData != null) { // TODO ???? add this directly to the response? - rb.rsp.add("highlighting", sumData); + rb.rsp.add(highlightingResponseField(), convertHighlights(sumData)); } } } @@ -245,7 +245,8 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni public void finishStage(ResponseBuilder rb) { if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) { - NamedList.NamedListEntry[] arr = new NamedList.NamedListEntry[rb.resultIds.size()]; + final Object[] objArr = newHighlightsArray(rb.resultIds.size()); + final String highlightingResponseField = highlightingResponseField(); // TODO: make a generic routine to do automatic merging of id keyed data for (ShardRequest sreq : rb.finished) { @@ -256,13 +257,12 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni // this should only happen when using shards.tolerant=true continue; } - NamedList hl = (NamedList)srsp.getSolrResponse().getResponse().get("highlighting"); - SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(hl, rb.resultIds, arr); + Object hl = srsp.getSolrResponse().getResponse().get(highlightingResponseField); + addHighlights(objArr, hl, rb.resultIds); } } - // remove nulls in case not all docs were able to be retrieved - rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>())); + rb.rsp.add(highlightingResponseField, getAllHighlights(objArr)); } } @@ -279,4 +279,33 @@ public class HighlightComponent extends SearchComponent implements PluginInfoIni public Category getCategory() { return Category.HIGHLIGHTER; } + + //////////////////////////////////////////// + /// highlighting response collation + //////////////////////////////////////////// + + protected String highlightingResponseField() { + return "highlighting"; + } + + protected Object convertHighlights(NamedList hl) { + return hl; + } + + protected Object[] newHighlightsArray(int size) { + return new NamedList.NamedListEntry[size]; + } + + protected void addHighlights(Object[] objArr, Object obj, Map resultIds) { + Map.Entry[] arr = (Map.Entry[])objArr; + NamedList hl = (NamedList)obj; + SolrPluginUtils.copyNamedListIntoArrayByDocPosInResponse(hl, resultIds, arr); + } + + protected Object getAllHighlights(Object[] objArr) { + final Map.Entry[] arr = (Map.Entry[])objArr; + // remove nulls in case not all docs were able to be retrieved + return SolrPluginUtils.removeNulls(arr, new SimpleOrderedMap<>()); + } + } diff --git a/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java new file mode 100644 index 00000000000..5f740644d13 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.TestCloudSearcherWarming.ConfigRequest; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.highlight.SolrFragmentsBuilder; +import org.junit.BeforeClass; +import org.junit.Test; + +public class CustomHighlightComponentTest extends SolrCloudTestCase { + + public static class CustomHighlightComponent extends HighlightComponent { + + protected String id_key = "id"; + protected String snippets_key = "snippets"; + + @Override + protected String highlightingResponseField() { + return "custom_highlighting"; + } + + @Override + protected Object convertHighlights(NamedList hl) { + final ArrayList hlMaps = new ArrayList<>(); + for (int i=0; i(); + hlMap.add(id_key, hl.getName(i)); + hlMap.add(snippets_key, hl.getVal(i)); + hlMaps.add(hlMap); + } + return hlMaps; + } + + @Override + protected Object[] newHighlightsArray(int size) { + return new SimpleOrderedMap[size]; + } + + @Override + protected void addHighlights(Object[] objArr, Object obj, Map resultIds) { + SimpleOrderedMap[] mapArr = (SimpleOrderedMap[])objArr; + final ArrayList hlMaps = (ArrayList)obj; + for (SimpleOrderedMap hlMap : hlMaps) { + String id = (String)hlMap.get(id_key); + ShardDoc sdoc = resultIds.get(id); + int idx = sdoc.positionInResponse; + mapArr[idx] = hlMap; + } + } + + @Override + protected Object getAllHighlights(Object[] objArr) { + final SimpleOrderedMap[] mapArr = (SimpleOrderedMap[])objArr; + // remove nulls in case not all docs were able to be retrieved + ArrayList mapList = new ArrayList<>(); + for (SimpleOrderedMap map : mapArr) { + if (map != null) { + mapList.add((SimpleOrderedMap)map); + } + } + return mapList; + } + + } + + protected String customHighlightComponentClassName() { + return CustomHighlightComponent.class.getName(); + } + + protected String id_key = "id"; + protected String snippets_key = "snippets"; + + private static String COLLECTION; + + @BeforeClass + public static void setupCluster() throws Exception { + + // decide collection name ... + COLLECTION = "collection"+(1+random().nextInt(100)) ; + // ... and shard/replica/node numbers + final int numShards = 3; + final int numReplicas = 2; + final int maxShardsPerNode = 2; + final int nodeCount = (numShards*numReplicas + (maxShardsPerNode-1))/maxShardsPerNode; + + // create and configure cluster + configureCluster(nodeCount) + .addConfig("conf", configset("cloud-dynamic")) + .configure(); + + // create an empty collection + CollectionAdminRequest + .createCollection(COLLECTION, "conf", numShards, numReplicas) + .setMaxShardsPerNode(maxShardsPerNode) + .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); + AbstractDistribZkTestBase.waitForRecoveriesToFinish(COLLECTION, cluster.getSolrClient().getZkStateReader(), false, true, DEFAULT_TIMEOUT); + } + + @Test + public void test() throws Exception { + + // determine custom search handler name (the exact name should not matter) + final String customSearchHandlerName = "/custom_select"+random().nextInt(); + + final String defaultHighlightComponentName = HighlightComponent.COMPONENT_NAME; + final String highlightComponentName; + + // add custom component (if needed) and handler + { + if (random().nextBoolean()) { + // default component + highlightComponentName = defaultHighlightComponentName; + } else { + // custom component + highlightComponentName = "customhighlight"+random().nextInt(); + cluster.getSolrClient().request( + new ConfigRequest( + SolrRequest.METHOD.POST, + "/config", + "{\n" + + " 'add-searchcomponent': {\n" + + " 'name': '"+highlightComponentName+"',\n" + + " 'class': '"+customHighlightComponentClassName()+"'\n" + + " }\n" + + "}"), + COLLECTION); + } + // handler + cluster.getSolrClient().request( + new ConfigRequest( + SolrRequest.METHOD.POST, + "/config", + "{\n" + + " 'add-requesthandler': {\n" + + " 'name' : '"+customSearchHandlerName+"',\n" + + " 'class' : 'org.apache.solr.handler.component.SearchHandler',\n" + + " 'components' : [ '"+QueryComponent.COMPONENT_NAME+"', '"+highlightComponentName+"' ]\n" + + " }\n" + + "}"), + COLLECTION); + } + + // add some documents + final String id = "id"; + final String t1 = "a_t"; + final String t2 = "b_t"; + { + new UpdateRequest() + .add(sdoc(id, 1, t1, "bumble bee", t2, "bumble bee")) + .add(sdoc(id, 2, t1, "honey bee", t2, "honey bee")) + .add(sdoc(id, 3, t1, "solitary bee", t2, "solitary bee")) + .commit(cluster.getSolrClient(), COLLECTION); + } + + // search for the documents + { + // compose the query + final SolrQuery solrQuery = new SolrQuery(t1+":bee"); + solrQuery.setRequestHandler(customSearchHandlerName); + solrQuery.setHighlight(true); + final boolean t1Highlights = random().nextBoolean(); + if (t1Highlights) { + solrQuery.addHighlightField(t1); + } + final boolean t2Highlights = random().nextBoolean(); + if (t2Highlights) { + solrQuery.addHighlightField(t2); + } + + // make the query + final QueryResponse queryResponse = new QueryRequest(solrQuery) + .process(cluster.getSolrClient(), COLLECTION); + + // analyse the response + final Map>> highlighting = queryResponse.getHighlighting(); + final ArrayList> custom_highlighting = + (ArrayList>)queryResponse.getResponse().get("custom_highlighting"); + + if (defaultHighlightComponentName.equals(highlightComponentName)) { + // regular 'highlighting' ... + if (t1Highlights) { + checkHighlightingResponseMap(highlighting, t1); + } + if (t2Highlights) { + checkHighlightingResponseMap(highlighting, t2); + } + if (!t1Highlights && !t2Highlights) { + checkHighlightingResponseMap(highlighting, null); + } + // ... and no 'custom_highlighting' + assertNull(custom_highlighting); + } else { + // no regular 'highlighting' ... + assertNull(highlighting); + // ... but 'custom_highlighting' + assertNotNull(custom_highlighting); + if (t1Highlights) { + checkHighlightingResponseList(custom_highlighting, t1); + } + if (t2Highlights) { + checkHighlightingResponseList(custom_highlighting, t2); + } + if (!t1Highlights && !t2Highlights) { + checkHighlightingResponseList(custom_highlighting, null); + } + } + } + } + + protected void checkHighlightingResponseMap(Map>> highlightingMap, + String highlightedField) throws Exception { + assertEquals("too few or too many keys: "+highlightingMap.keySet(), + 3, highlightingMap.size()); + checkHighlightingResponseMapElement(highlightingMap.get("1"), highlightedField, "bumble ", "bee"); + checkHighlightingResponseMapElement(highlightingMap.get("2"), highlightedField, "honey ", "bee"); + checkHighlightingResponseMapElement(highlightingMap.get("3"), highlightedField, "solitary ", "bee"); + } + + protected void checkHighlightingResponseMapElement(Map> docHighlights, + String highlightedField, String preHighlightText, String highlightedText) throws Exception { + if (highlightedField == null) { + assertEquals(0, docHighlights.size()); + } else { + List docHighlightsList = docHighlights.get(highlightedField); + assertEquals(1, docHighlightsList.size()); + assertEquals(preHighlightText + + SolrFragmentsBuilder.DEFAULT_PRE_TAGS + + highlightedText + + SolrFragmentsBuilder.DEFAULT_POST_TAGS, docHighlightsList.get(0)); + } + } + + protected void checkHighlightingResponseList(ArrayList> highlightingList, + String highlightedField) throws Exception { + assertEquals("too few or too many elements: "+highlightingList.size(), + 3, highlightingList.size()); + final Set seenDocIds = new HashSet<>(); + for (SimpleOrderedMap highlightingListElementMap : highlightingList) { + final String expectedHighlightText; + final String actualHighlightText; + // two elements in total: id and snippets + assertEquals(highlightingList.toString(), 2, highlightingListElementMap.size()); + // id element + { + final String docId = (String)highlightingListElementMap.get(id_key); + seenDocIds.add(docId); + final String preHighlightText; + final String highlightedText = "bee"; + if ("1".equals(docId)) { + preHighlightText = "bumble "; + } else if ("2".equals(docId)) { + preHighlightText = "honey "; + } else if ("3".equals(docId)) { + preHighlightText = "solitary "; + } else { + preHighlightText = null; + fail("unknown docId "+docId); + } + expectedHighlightText = preHighlightText + + SolrFragmentsBuilder.DEFAULT_PRE_TAGS + + highlightedText + + SolrFragmentsBuilder.DEFAULT_POST_TAGS; + } + // snippets element + { + SimpleOrderedMap snippets = (SimpleOrderedMap)highlightingListElementMap.get(snippets_key); + if (highlightedField == null) { + assertEquals(0, snippets.size()); + } else { + ArrayList docHighlights = (ArrayList)(snippets).get(highlightedField); + assertEquals(1, docHighlights.size()); + actualHighlightText = (String)docHighlights.get(0); + assertEquals(expectedHighlightText, actualHighlightText); + } + } + } + assertEquals(3, seenDocIds.size()); + } + +}