mirror of https://github.com/apache/lucene.git
SOLR-4799 faster join using join=zipper aka merge join for nested DIH EntityProcessors
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1643097 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul
parent
854e3f930e
commit
83f0d3578c
|
|
@ -219,6 +219,9 @@ New Features
|
|||
|
||||
* SOLR-6607: Managing requesthandlers throuh API (Noble Paul)
|
||||
|
||||
* SOLR-4799: faster join using join="zipper" aka merge join for nested DIH EntityProcessors
|
||||
(Mikhail Khludnev via Noble Paul)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ public class TikaEntityProcessor extends EntityProcessorBase {
|
|||
|
||||
@Override
|
||||
protected void firstInit(Context context) {
|
||||
super.firstInit(context);
|
||||
try {
|
||||
String tikaConfigFile = context.getResolvedEntityAttribute("tikaConfig");
|
||||
if (tikaConfigFile == null) {
|
||||
|
|
|
|||
|
|
@ -44,28 +44,11 @@ public class DIHCacheSupport {
|
|||
public DIHCacheSupport(Context context, String cacheImplName) {
|
||||
this.cacheImplName = cacheImplName;
|
||||
|
||||
String where = context.getEntityAttribute("where");
|
||||
String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
|
||||
String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
|
||||
if (cacheKey != null && lookupKey == null) {
|
||||
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
|
||||
"'cacheKey' is specified for the entity "
|
||||
+ context.getEntityAttribute("name")
|
||||
+ " but 'cacheLookup' is missing");
|
||||
|
||||
}
|
||||
if (where == null && cacheKey == null) {
|
||||
cacheDoKeyLookup = false;
|
||||
} else {
|
||||
if (where != null) {
|
||||
String[] splits = where.split("=");
|
||||
cacheKey = splits[0];
|
||||
cacheForeignKey = splits[1].trim();
|
||||
} else {
|
||||
cacheForeignKey = lookupKey;
|
||||
}
|
||||
cacheDoKeyLookup = true;
|
||||
}
|
||||
Relation r = new Relation(context);
|
||||
cacheDoKeyLookup = r.doKeyLookup;
|
||||
String cacheKey = r.primaryKey;
|
||||
cacheForeignKey = r.foreignKey;
|
||||
|
||||
context.setSessionAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY, cacheKey,
|
||||
Context.SCOPE_ENTITY);
|
||||
context.setSessionAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY, cacheForeignKey,
|
||||
|
|
@ -76,6 +59,48 @@ public class DIHCacheSupport {
|
|||
Context.SCOPE_ENTITY);
|
||||
}
|
||||
|
||||
static class Relation{
|
||||
protected final boolean doKeyLookup;
|
||||
protected final String foreignKey;
|
||||
protected final String primaryKey;
|
||||
|
||||
public Relation(Context context) {
|
||||
String where = context.getEntityAttribute("where");
|
||||
String cacheKey = context.getEntityAttribute(DIHCacheSupport.CACHE_PRIMARY_KEY);
|
||||
String lookupKey = context.getEntityAttribute(DIHCacheSupport.CACHE_FOREIGN_KEY);
|
||||
if (cacheKey != null && lookupKey == null) {
|
||||
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
|
||||
"'cacheKey' is specified for the entity "
|
||||
+ context.getEntityAttribute("name")
|
||||
+ " but 'cacheLookup' is missing");
|
||||
|
||||
}
|
||||
if (where == null && cacheKey == null) {
|
||||
doKeyLookup = false;
|
||||
primaryKey = null;
|
||||
foreignKey = null;
|
||||
} else {
|
||||
if (where != null) {
|
||||
String[] splits = where.split("=");
|
||||
primaryKey = splits[0];
|
||||
foreignKey = splits[1].trim();
|
||||
} else {
|
||||
primaryKey = cacheKey;
|
||||
foreignKey = lookupKey;
|
||||
}
|
||||
doKeyLookup = true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Relation "
|
||||
+ primaryKey + "="+foreignKey ;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private DIHCache instantiateCache(Context context) {
|
||||
DIHCache cache = null;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ public class EntityProcessorBase extends EntityProcessor {
|
|||
protected String onError = ABORT;
|
||||
|
||||
protected DIHCacheSupport cacheSupport = null;
|
||||
|
||||
protected Zipper zipper;
|
||||
|
||||
|
||||
@Override
|
||||
|
|
@ -56,19 +58,30 @@ public class EntityProcessorBase extends EntityProcessor {
|
|||
if (isFirstInit) {
|
||||
firstInit(context);
|
||||
}
|
||||
if(cacheSupport!=null) {
|
||||
cacheSupport.initNewParent(context);
|
||||
}
|
||||
|
||||
if(zipper!=null){
|
||||
zipper.onNewParent(context);
|
||||
}else{
|
||||
if(cacheSupport!=null) {
|
||||
cacheSupport.initNewParent(context);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**first time init call. do one-time operations here
|
||||
/**
|
||||
* first time init call. do one-time operations here
|
||||
* it's necessary to call it from the overridden method,
|
||||
* otherwise it throws NPE on accessing zipper from nextRow()
|
||||
*/
|
||||
protected void firstInit(Context context) {
|
||||
entityName = context.getEntityAttribute("name");
|
||||
String s = context.getEntityAttribute(ON_ERROR);
|
||||
if (s != null) onError = s;
|
||||
initCache(context);
|
||||
|
||||
zipper = Zipper.createOrNull(context);
|
||||
|
||||
if(zipper==null){
|
||||
initCache(context);
|
||||
}
|
||||
isFirstInit = false;
|
||||
}
|
||||
|
||||
|
|
@ -109,25 +122,29 @@ public class EntityProcessorBase extends EntityProcessor {
|
|||
}
|
||||
|
||||
protected Map<String, Object> getNext() {
|
||||
if(cacheSupport==null) {
|
||||
try {
|
||||
if (rowIterator == null)
|
||||
if(zipper!=null){
|
||||
return zipper.supplyNextChild(rowIterator);
|
||||
}else{
|
||||
if(cacheSupport==null) {
|
||||
try {
|
||||
if (rowIterator == null)
|
||||
return null;
|
||||
if (rowIterator.hasNext())
|
||||
return rowIterator.next();
|
||||
query = null;
|
||||
rowIterator = null;
|
||||
return null;
|
||||
if (rowIterator.hasNext())
|
||||
return rowIterator.next();
|
||||
query = null;
|
||||
rowIterator = null;
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
SolrException.log(log, "getNext() failed for query '" + query + "'", e);
|
||||
query = null;
|
||||
rowIterator = null;
|
||||
wrapAndThrow(DataImportHandlerException.WARN, e);
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
return cacheSupport.getCacheData(context, query, rowIterator);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
SolrException.log(log, "getNext() failed for query '" + query + "'", e);
|
||||
query = null;
|
||||
rowIterator = null;
|
||||
wrapAndThrow(DataImportHandlerException.WARN, e);
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
return cacheSupport.getCacheData(context, query, rowIterator);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,116 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.solr.handler.dataimport.DIHCacheSupport.Relation;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.google.common.base.Supplier;
|
||||
import com.google.common.collect.Iterators;
|
||||
import com.google.common.collect.PeekingIterator;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
class Zipper {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(Zipper.class);
|
||||
private final DIHCacheSupport.Relation relation;
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
private Comparable parentId;
|
||||
@SuppressWarnings("rawtypes")
|
||||
private Comparable lastChildId;
|
||||
|
||||
private Iterator<Map<String,Object>> rowIterator;
|
||||
private PeekingIterator<Map<String,Object>> peeker;
|
||||
|
||||
/** @return initialized zipper or null */
|
||||
public static Zipper createOrNull(Context context){
|
||||
if("zipper".equals(context.getEntityAttribute("join"))){
|
||||
DIHCacheSupport.Relation r = new DIHCacheSupport.Relation(context);
|
||||
if(r.doKeyLookup){
|
||||
return new Zipper(r);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
private Zipper(Relation relation) {
|
||||
this.relation = relation;
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
public Map<String,Object> supplyNextChild(
|
||||
Iterator<Map<String,Object>> rowIterator) {
|
||||
preparePeeker(rowIterator);
|
||||
|
||||
while(peeker.hasNext()){
|
||||
Map<String,Object> current = peeker.peek();
|
||||
Comparable childId = (Comparable) current.get(relation.primaryKey);
|
||||
|
||||
if(lastChildId!=null && lastChildId.compareTo(childId)>0){
|
||||
throw new IllegalArgumentException("expect increasing foreign keys for "+relation+
|
||||
" got: "+lastChildId+","+childId);
|
||||
}
|
||||
lastChildId = childId;
|
||||
int cmp = childId.compareTo(parentId);
|
||||
if(cmp==0){
|
||||
Map<String,Object> child = peeker.next();
|
||||
assert child==current: "peeker should be right but "+current+" != " + child;
|
||||
log.trace("yeild child {} entry {}",relation, current);
|
||||
return child;// TODO it's for one->many for many->one it should be just peek()
|
||||
}else{
|
||||
if(cmp<0){ // child belongs to 10th and parent is 20th, skip for the next one
|
||||
Map<String,Object> child = peeker.next();
|
||||
assert child==current: "peeker should be right but "+current+" != " + child;
|
||||
log.trace("skip child {}, {} > {}",relation, parentId, childId);
|
||||
}else{ // child belongs to 20th and parent is 10th, no more children, go to next parent
|
||||
log.trace("childen is over {}, {} < {}", relation, parentId, current);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private void preparePeeker(Iterator<Map<String,Object>> rowIterator) {
|
||||
if(this.rowIterator==null){
|
||||
this.rowIterator = rowIterator;
|
||||
peeker = Iterators.peekingIterator(rowIterator);
|
||||
}else{
|
||||
assert this.rowIterator==rowIterator: "rowIterator should never change but "+this.rowIterator+
|
||||
" supplied before has been changed to "+rowIterator;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
public void onNewParent(Context context) {
|
||||
Comparable newParent = (Comparable) context.resolve(relation.foreignKey);
|
||||
if(parentId!=null && parentId.compareTo(newParent)>=0){
|
||||
throw new IllegalArgumentException("expect strictly increasing primary keys for "+relation+
|
||||
" got: "+parentId+","+newParent);
|
||||
}
|
||||
log.trace("{}: {}->{}",relation, newParent, parentId);
|
||||
parentId = newParent;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -45,8 +45,15 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
protected boolean useSimpleCaches;
|
||||
protected boolean countryEntity;
|
||||
protected boolean countryCached;
|
||||
protected boolean countryZipper;
|
||||
protected boolean sportsEntity;
|
||||
protected boolean sportsCached;
|
||||
protected boolean sportsZipper;
|
||||
|
||||
protected boolean wrongPeopleOrder ;
|
||||
protected boolean wrongSportsOrder ;
|
||||
protected boolean wrongCountryOrder;
|
||||
|
||||
protected String rootTransformerName;
|
||||
protected boolean countryTransformer;
|
||||
protected boolean sportsTransformer;
|
||||
|
|
@ -65,8 +72,15 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
useSimpleCaches = false;
|
||||
countryEntity = false;
|
||||
countryCached = false;
|
||||
countryZipper = false;
|
||||
sportsEntity = false;
|
||||
sportsCached = false;
|
||||
sportsZipper = false;
|
||||
|
||||
wrongPeopleOrder = false;
|
||||
wrongSportsOrder = false;
|
||||
wrongCountryOrder= false;
|
||||
|
||||
rootTransformerName = null;
|
||||
countryTransformer = false;
|
||||
sportsTransformer = false;
|
||||
|
|
@ -177,9 +191,14 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
} else {
|
||||
countryEntity = true;
|
||||
sportsEntity = true;
|
||||
if (numChildren == 1) {
|
||||
countryEntity = random().nextBoolean();
|
||||
sportsEntity = !countryEntity;
|
||||
if(countryZipper||sportsZipper){// zipper tests fully cover nums of children
|
||||
countryEntity = countryZipper;
|
||||
sportsEntity = sportsZipper;
|
||||
}else{// apply default randomization on cached cases
|
||||
if (numChildren == 1) {
|
||||
countryEntity = random().nextBoolean();
|
||||
sportsEntity = !countryEntity;
|
||||
}
|
||||
}
|
||||
if (countryEntity) {
|
||||
countryTransformer = random().nextBoolean();
|
||||
|
|
@ -212,18 +231,29 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
+ (totalPeople()) + "']");
|
||||
}
|
||||
if (countryEntity) {
|
||||
if (personNameExists("Jayden")) {
|
||||
String nrName = countryNameByCode("NP");
|
||||
if (nrName != null && nrName.length() > 0) {
|
||||
assertQ(req("NAME_mult_s:Jayden"), "//*[@numFound='1']",
|
||||
"//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
|
||||
}
|
||||
{
|
||||
String[] people = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
|
||||
String man = people[random().nextInt(people.length)];
|
||||
String[] countryNames = getStringsFromQuery("SELECT C.COUNTRY_NAME FROM PEOPLE P "
|
||||
+ "INNER JOIN COUNTRIES C ON P.COUNTRY_CODE=C.CODE "
|
||||
+ "WHERE P.DELETED!='Y' AND C.DELETED!='Y' AND P.NAME='" + man + "'");
|
||||
|
||||
assertQ(req("{!term f=NAME_mult_s}"+ man), "//*[@numFound='1']",
|
||||
countryNames.length>0?
|
||||
"//doc/str[@name='COUNTRY_NAME_s']='" + countryNames[random().nextInt(countryNames.length)] + "'"
|
||||
:"//doc[count(*[@name='COUNTRY_NAME_s'])=0]");
|
||||
}
|
||||
String nrName = countryNameByCode("NR");
|
||||
int num = numberPeopleByCountryCode("NR");
|
||||
if (nrName != null && num > 0) {
|
||||
assertQ(req("COUNTRY_CODES_mult_s:NR"), "//*[@numFound='" + num + "']",
|
||||
"//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
|
||||
{
|
||||
String[] countryCodes = getStringsFromQuery("SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y'");
|
||||
String theCode = countryCodes[random().nextInt(countryCodes.length)];
|
||||
int num = numberPeopleByCountryCode(theCode);
|
||||
if(num>0){
|
||||
String nrName = countryNameByCode(theCode);
|
||||
assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']",
|
||||
"//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
|
||||
}else{ // no one lives there anyway
|
||||
assertQ(req("COUNTRY_CODES_mult_s:"+theCode), "//*[@numFound='" + num + "']");
|
||||
}
|
||||
}
|
||||
if (countryTransformer && !underlyingDataModified) {
|
||||
assertQ(req("countryAdded_s:country_added"), "//*[@numFound='"
|
||||
|
|
@ -234,24 +264,27 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
if (!underlyingDataModified) {
|
||||
assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
|
||||
}
|
||||
String michaelsName = personNameById(3);
|
||||
String[] michaelsSports = sportNamesByPersonId(3);
|
||||
if (michaelsName != null && michaelsSports.length > 0) {
|
||||
String[] xpath = new String[michaelsSports.length + 1];
|
||||
xpath[0] = "//*[@numFound='1']";
|
||||
int i = 1;
|
||||
for (String ms : michaelsSports) {
|
||||
xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='"
|
||||
+ ms + "'";
|
||||
i++;
|
||||
}
|
||||
assertQ(req("NAME_mult_s:" + michaelsName.replaceAll("\\W", "\\\\$0")),
|
||||
xpath);
|
||||
String [] names = getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE DELETED != 'Y'");
|
||||
String name = names[random().nextInt(names.length)];
|
||||
int personId = getIntFromQuery("SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND NAME='"+name+"'");
|
||||
String[] michaelsSports = sportNamesByPersonId(personId);
|
||||
|
||||
String[] xpath = new String[michaelsSports.length + 1];
|
||||
xpath[0] = "//*[@numFound='1']";
|
||||
int i = 1;
|
||||
for (String ms : michaelsSports) {
|
||||
xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str='"//[" + i + "]='" don't care about particular order
|
||||
+ ms + "'";
|
||||
i++;
|
||||
}
|
||||
assertQ(req("NAME_mult_s:" + name.replaceAll("\\W", "\\\\$0")),
|
||||
xpath);
|
||||
if (!underlyingDataModified && sportsTransformer) {
|
||||
assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='"
|
||||
+ (totalPeople()) + "']");
|
||||
+ (totalSportsmen()) + "']");
|
||||
}
|
||||
assertQ("checking orphan sport is absent",
|
||||
req("{!term f=SPORT_NAME_mult_s}No Fishing"), "//*[@numFound='0']");
|
||||
}
|
||||
if (checkDatabaseRequests) {
|
||||
Assert.assertTrue("Expecting " + numDatabaseRequests
|
||||
|
|
@ -304,6 +337,7 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
private int getIntFromQuery(String query) throws Exception {
|
||||
Connection conn = null;
|
||||
Statement s = null;
|
||||
|
|
@ -367,6 +401,12 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
|
||||
}
|
||||
|
||||
public int totalSportsmen() throws Exception {
|
||||
return getIntFromQuery("SELECT COUNT(*) FROM PEOPLE WHERE "
|
||||
+ "EXISTS(SELECT ID FROM PEOPLE_SPORTS WHERE PERSON_ID=PEOPLE.ID AND PEOPLE_SPORTS.DELETED != 'Y')"
|
||||
+ " AND PEOPLE.DELETED != 'Y'");
|
||||
}
|
||||
|
||||
public boolean countryCodeExists(String cc) throws Exception {
|
||||
return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='"
|
||||
+ cc + "'") > 0;
|
||||
|
|
@ -574,7 +614,11 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
sb.append("dataSource=''" + ds + "'' ");
|
||||
sb.append(rootTransformerName != null ? "transformer=''"
|
||||
+ rootTransformerName + "'' " : "");
|
||||
sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' '' ");
|
||||
|
||||
sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' "
|
||||
+((sportsZipper||countryZipper?"ORDER BY ID":"")
|
||||
+(wrongPeopleOrder? " DESC":""))+"'' ");
|
||||
|
||||
sb.append(deltaQueriesPersonTable());
|
||||
sb.append("> \n");
|
||||
|
||||
|
|
@ -595,9 +639,20 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
if (useSimpleCaches) {
|
||||
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
|
||||
} else {
|
||||
sb.append(random().nextBoolean() ? "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' "
|
||||
: "where=''CODE=People.COUNTRY_CODE'' ");
|
||||
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
|
||||
|
||||
if(countryZipper){// really odd join btw. it sends duped countries
|
||||
sb.append(random().nextBoolean() ? "cacheKey=''ID'' cacheLookup=''People.ID'' "
|
||||
: "where=''ID=People.ID'' ");
|
||||
sb.append("join=''zipper'' query=''SELECT PEOPLE.ID, CODE, COUNTRY_NAME FROM COUNTRIES"
|
||||
+ " JOIN PEOPLE ON COUNTRIES.CODE=PEOPLE.COUNTRY_CODE "
|
||||
+ "WHERE PEOPLE.DELETED != 'Y' ORDER BY PEOPLE.ID "+
|
||||
(wrongCountryOrder ? " DESC":"")
|
||||
+ "'' ");
|
||||
}else{
|
||||
sb.append(random().nextBoolean() ? "cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' "
|
||||
: "where=''CODE=People.COUNTRY_CODE'' ");
|
||||
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
|
||||
}
|
||||
sb.append("> \n");
|
||||
}
|
||||
} else {
|
||||
|
|
@ -623,7 +678,14 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
} else {
|
||||
sb.append(random().nextBoolean() ? "cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' "
|
||||
: "where=''PERSON_ID=People.ID'' ");
|
||||
sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
|
||||
if(sportsZipper){
|
||||
sb.append("join=''zipper'' query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY PERSON_ID"
|
||||
+ (wrongSportsOrder?" DESC" : "")+
|
||||
"'' ");
|
||||
}
|
||||
else{
|
||||
sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID=${People.ID} ORDER BY ID'' ");
|
||||
|
|
@ -726,7 +788,9 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
{7,"Noah","NI"},
|
||||
{8,"Daniel","NG"},
|
||||
{9,"Aiden","NF"},
|
||||
{10,"Anthony","NE"},
|
||||
|
||||
{21,"Anthony","NE"}, // there is no ID=10 anymore
|
||||
|
||||
{11,"Emma","NL"},
|
||||
{12,"Grace","NI"},
|
||||
{13,"Hailey","NG"},
|
||||
|
|
@ -751,7 +815,10 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
{700, 7, "Boating"},
|
||||
{800, 8, "Bodyboarding"},
|
||||
{900, 9, "Canoeing"},
|
||||
{1000, 10, "Fishing"},
|
||||
|
||||
{1000, 10, "No Fishing"}, // orhpaned sport
|
||||
//
|
||||
|
||||
{1100, 11, "Jet Ski"},
|
||||
{1110, 11, "Rowing"},
|
||||
{1120, 11, "Sailing"},
|
||||
|
|
@ -760,10 +827,12 @@ public abstract class AbstractSqlEntityProcessorTestCase extends
|
|||
{1300, 13, "Kite surfing"},
|
||||
{1400, 14, "Parasailing"},
|
||||
{1500, 15, "Rafting"},
|
||||
{1600, 16, "Rowing"},
|
||||
//{1600, 16, "Rowing"}, Madison has no sport
|
||||
{1700, 17, "Sailing"},
|
||||
{1800, 18, "White Water Rafting"},
|
||||
{1900, 19, "Water skiing"},
|
||||
{2000, 20, "Windsurfing"}
|
||||
{2000, 20, "Windsurfing"},
|
||||
{2100, 21, "Concrete diving"},
|
||||
{2110, 21, "Bubble rugby"}
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,10 +31,6 @@ public class MockSolrEntityProcessor extends SolrEntityProcessor {
|
|||
|
||||
private int rows;
|
||||
|
||||
public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData) {
|
||||
this(docsData, ROWS_DEFAULT);
|
||||
}
|
||||
|
||||
public MockSolrEntityProcessor(List<SolrTestCaseJ4.Doc> docsData, int rows) {
|
||||
this.docsData = docsData;
|
||||
this.rows = rows;
|
||||
|
|
|
|||
|
|
@ -31,20 +31,33 @@ public class TestSolrEntityProcessorUnit extends AbstractDataImportHandlerTestCa
|
|||
public void testQuery() {
|
||||
List<Doc> docs = generateUniqueDocs(2);
|
||||
|
||||
MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs);
|
||||
MockSolrEntityProcessor processor = createAndInit(docs);
|
||||
|
||||
assertExpectedDocs(docs, processor);
|
||||
assertEquals(1, processor.getQueryCount());
|
||||
}
|
||||
|
||||
private MockSolrEntityProcessor createAndInit(List<Doc> docs) {
|
||||
return createAndInit(docs, SolrEntityProcessor.ROWS_DEFAULT);
|
||||
}
|
||||
|
||||
public void testNumDocsGreaterThanRows() {
|
||||
List<Doc> docs = generateUniqueDocs(44);
|
||||
|
||||
MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, 10);
|
||||
int rowsNum = 10;
|
||||
MockSolrEntityProcessor processor = createAndInit(docs, rowsNum);
|
||||
assertExpectedDocs(docs, processor);
|
||||
assertEquals(5, processor.getQueryCount());
|
||||
}
|
||||
|
||||
private MockSolrEntityProcessor createAndInit(List<Doc> docs, int rowsNum) {
|
||||
MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs, rowsNum);
|
||||
HashMap<String,String> entityAttrs = new HashMap<String,String>(){{put(SolrEntityProcessor.SOLR_SERVER,"http://route:66/no");}};
|
||||
processor.init(getContext(null, null, null, null, Collections.emptyList(),
|
||||
entityAttrs));
|
||||
return processor;
|
||||
}
|
||||
|
||||
public void testMultiValuedFields() {
|
||||
List<Doc> docs = new ArrayList<>();
|
||||
List<FldType> types = new ArrayList<>();
|
||||
|
|
@ -53,7 +66,7 @@ public class TestSolrEntityProcessorUnit extends AbstractDataImportHandlerTestCa
|
|||
Doc testDoc = createDoc(types);
|
||||
docs.add(testDoc);
|
||||
|
||||
MockSolrEntityProcessor processor = new MockSolrEntityProcessor(docs);
|
||||
MockSolrEntityProcessor processor = createAndInit(docs);
|
||||
Map<String, Object> next = processor.nextRow();
|
||||
assertNotNull(next);
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,60 @@ public class TestSqlEntityProcessor extends AbstractSqlEntityProcessorTestCase {
|
|||
public void testCachedChildEntities() throws Exception {
|
||||
withChildEntities(true, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSportZipperChildEntities() throws Exception {
|
||||
sportsZipper = true;
|
||||
withChildEntities(true, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCountryZipperChildEntities() throws Exception {
|
||||
countryZipper = true;
|
||||
withChildEntities(true, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testBothZipperChildEntities() throws Exception {
|
||||
countryZipper = true;
|
||||
sportsZipper = true;
|
||||
withChildEntities(true, true);
|
||||
}
|
||||
|
||||
@Test(expected=RuntimeException.class /* DIH exceptions are not propagated, here we capturing assertQ exceptions */)
|
||||
public void testSportZipperChildEntitiesWrongOrder() throws Exception {
|
||||
if(random().nextBoolean()){
|
||||
wrongPeopleOrder = true;
|
||||
}else{
|
||||
wrongSportsOrder = true;
|
||||
}
|
||||
testSportZipperChildEntities();
|
||||
}
|
||||
|
||||
@Test(expected=RuntimeException.class )
|
||||
public void testCountryZipperChildEntitiesWrongOrder() throws Exception {
|
||||
if(random().nextBoolean()){
|
||||
wrongPeopleOrder = true;
|
||||
}else{
|
||||
wrongCountryOrder = true;
|
||||
}
|
||||
testCountryZipperChildEntities();
|
||||
}
|
||||
|
||||
@Test(expected=RuntimeException.class)
|
||||
public void testBothZipperChildEntitiesWrongOrder() throws Exception {
|
||||
if(random().nextBoolean()){
|
||||
wrongPeopleOrder = true;
|
||||
}else{
|
||||
if(random().nextBoolean()){
|
||||
wrongSportsOrder = true;
|
||||
}else{
|
||||
wrongCountryOrder = true;
|
||||
}
|
||||
}
|
||||
testBothZipperChildEntities();
|
||||
}
|
||||
|
||||
@Test
|
||||
@Ignore("broken see SOLR-3857")
|
||||
public void testSimpleCacheChildEntities() throws Exception {
|
||||
|
|
|
|||
Loading…
Reference in New Issue