Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-3856: Improve tests for SqlEntityProcessor 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1406231 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
James Dyer 2012-11-06 17:24:40 +00:00
parent b6db7c76f1
commit 5461ef1ac1
13 changed files with 930 additions and 1703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig-end-to-end.xml
View File

@ -1,41 +0,0 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
<updateHandler class="solr.DirectUpdateHandler2">
<maxPendingDeletes>100000</maxPendingDeletes>
</updateHandler>
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<httpCaching never304="true" />
</requestDispatcher>
<requestHandler name="standard" class="solr.StandardRequestHandler" default="true">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
</requestHandler>
<requestHandler name="/dataimport-end-to-end" class="org.apache.solr.handler.dataimport.DataImportHandler" />
<requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
</requestHandler>
</config>

619
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AbstractDIHJdbcTestCase.java
View File

@ -19,72 +19,69 @@ package org.apache.solr.handler.dataimport;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import junit.framework.Assert;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
/**
* This sets up an in-memory Derby Sql database with a little sample data.
* The schema here is poorly-designed to illustrate DIH's ability to
* overcome these kinds of challenges.
*/
public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerTestCase {
//Start with "true" so the first test run will populate the data.
protected boolean underlyingDataModified = true;
protected boolean useSimpleCaches;
protected boolean countryEntity;
protected boolean countryCached;
protected boolean sportsEntity;
protected boolean sportsCached;
protected String rootTransformerName;
protected boolean countryTransformer;
protected boolean sportsTransformer;
private static final Pattern totalRequestsPattern = Pattern
.compile(".str name..Total Requests made to DataSource..(\\d+)..str.");
@BeforeClass
public static void beforeClassDihJdbcTest() throws Exception {
try {
Class.forName("org.hsqldb.jdbcDriver").newInstance();
} catch (Exception e) {
throw e;
}
}
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
@Before
public void beforeDihJdbcTest() throws Exception {
useSimpleCaches = false;
countryEntity = false;
countryCached = false;
sportsEntity = false;
sportsCached = false;
rootTransformerName = null;
countryTransformer = false;
sportsTransformer = false;
Connection conn = null;
Statement s = null;
PreparedStatement ps = null;
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
s = conn.createStatement();
s.executeUpdate("create table countries(code char(2) not null primary key, country_name varchar(50))");
s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_codes varchar(100))");
s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50))");
ps = conn.prepareStatement("insert into countries values (?,?)");
for(String[] country : countries) {
ps.setString(1, country[0]);
ps.setString(2, country[1]);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
ps = conn.prepareStatement("insert into people values (?,?,?)");
for(Object[] person : people) {
ps.setInt(1, (Integer) person[0]);
ps.setString(2, (String) person[1]);
ps.setString(3, (String) person[2]);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
ps = conn.prepareStatement("insert into people_sports values (?,?,?)");
for(Object[] sport : people_sports) {
ps.setInt(1, (Integer) sport[0]);
ps.setInt(2, (Integer) sport[1]);
ps.setString(3, (String) sport[2]);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
conn.close();
} catch(Exception e) {
throw e;
} finally {
if(s!=null) { s.close(); }
if(ps!=null) { ps.close(); }
if(conn!=null) { conn.close(); }
clearIndex();
assertU(commit());
if(underlyingDataModified) {
refreshDatabase();
}
}
}
@AfterClass
public static void afterClassDihJdbcTest() throws Exception {
Connection conn = null;
@ -96,10 +93,527 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
} catch (SQLException e) {
throw e;
} finally {
if(s!=null) { s.close(); }
if(conn!=null) { conn.close(); }
try { s.close(); } catch(Exception ex) { }
try { conn.close(); } catch(Exception ex) { }
}
}
protected void singleEntity(int numToExpect) throws Exception {
h.query("/dataimport", generateRequest());
assertQ(req("*:*"), "//*[@numFound='" + totalPeople() + "']");
Assert.assertTrue(totalDatabaseRequests() == numToExpect);
}
protected void simpleTransform(int numToExpect) throws Exception {
rootTransformerName = "AddAColumnTransformer";
h.query("/dataimport", generateRequest());
assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + totalPeople() + "']");
Assert.assertTrue(totalDatabaseRequests() == numToExpect);
}
/**
* A delta update will not clean up documents added by a transformer
* even if the parent document that the transformer used to base the new documents
* were deleted
*/
protected void complexTransform(int numToExpect, int numDeleted) throws Exception {
rootTransformerName = "TripleThreatTransformer";
h.query("/dataimport", generateRequest());
assertQ(req("q", "*:*", "rows", "" + (totalPeople() * 3), "sort", "id asc"), "//*[@numFound='" + ((totalPeople() * 3) + (numDeleted * 2)) + "']");
assertQ(req("id:TripleThreat-1-*"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
assertQ(req("id:TripleThreat-2-*"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
if(personNameExists("Michael") && countryCodeExists("NR") && countryCodeExists("RN"))
{
assertQ(
req("+id:TripleThreat-1-3 +NAME_mult_s:Michael +NAME_mult_s:leahciM +COUNTRY_CODES_mult_s:NR +COUNTRY_CODES_mult_s:RN"),
"//*[@numFound='1']");
}
assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
Assert.assertTrue(totalDatabaseRequests() == numToExpect);
}
protected void withChildEntities(boolean cached, boolean checkDatabaseRequests) throws Exception {
rootTransformerName = random().nextBoolean() ? null : "AddAColumnTransformer";
int numChildren = random().nextInt(1) + 1;
int numDatabaseRequests = 1;
if(underlyingDataModified) {
if (countryEntity) {
if (cached) {
numDatabaseRequests++;
} else {
numDatabaseRequests += totalPeople();
}
}
if (sportsEntity) {
if (cached) {
numDatabaseRequests++;
} else {
numDatabaseRequests += totalPeople();
}
}
} else {
countryEntity = true;
sportsEntity = true;
if(numChildren==1) {
countryEntity = random().nextBoolean();
sportsEntity = !countryEntity;
}
if(countryEntity) {
countryTransformer = random().nextBoolean();
if(cached) {
numDatabaseRequests++;
countryCached = true;
} else {
numDatabaseRequests += totalPeople();
}
}
if(sportsEntity) {
sportsTransformer = random().nextBoolean();
if(cached) {
numDatabaseRequests++;
sportsCached = true;
} else {
numDatabaseRequests += totalPeople();
}
}
}
h.query("/dataimport", generateRequest());
assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
if(!underlyingDataModified && "AddAColumnTransformer".equals(rootTransformerName)) {
assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + (totalPeople()) + "']");
}
if(countryEntity) {
if(personNameExists("Jayden"))
{
String nrName = countryNameByCode("NP");
if(nrName!=null && nrName.length()>0) {
assertQ(req("NAME_mult_s:Jayden"),
"//*[@numFound='1']", "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
}
}
String nrName = countryNameByCode("NR");
int num = numberPeopleByCountryCode("NR");
if(nrName!=null && num>0) {
assertQ(req("COUNTRY_CODES_mult_s:NR"),
"//*[@numFound='" + num + "']", "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
}
if(countryTransformer && !underlyingDataModified) {
assertQ(req("countryAdded_s:country_added"), "//*[@numFound='" + totalPeople() + "']");
}
}
if(sportsEntity) {
if(!underlyingDataModified) {
assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
}
String michaelsName = personNameById(3);
String[] michaelsSports = sportNamesByPersonId(3);
if(michaelsName != null && michaelsSports.length>0) {
String[] xpath = new String[michaelsSports.length + 1];
xpath[0] = "//*[@numFound='1']";
int i=1;
for(String ms : michaelsSports) {
xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms + "'";
i++;
}
assertQ(req("NAME_mult_s:" + michaelsName.replaceAll("\\W", "\\\\$0")), xpath);
}
if (!underlyingDataModified && sportsTransformer) {
assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='" + (totalPeople()) + "']");
}
}
if(checkDatabaseRequests) {
Assert.assertTrue("Expecting " + numDatabaseRequests
+ " database calls, but DIH reported " + totalDatabaseRequests(),
totalDatabaseRequests() == numDatabaseRequests);
}
}
protected void simpleCacheChildEntities(boolean checkDatabaseRequests) throws Exception {
useSimpleCaches = true;
countryEntity = true;
sportsEntity = true;
countryCached=true;
sportsCached=true;
int dbRequestsMoreThan = 3;
int dbRequestsLessThan = totalPeople() * 2 + 1;
h.query("/dataimport", generateRequest());
assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
if(!underlyingDataModified || (personNameExists("Samantha") && "Nauru".equals(countryNameByCode("NR"))))
{
assertQ(req("NAME_mult_s:Samantha"),
"//*[@numFound='1']", "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
}
if(!underlyingDataModified)
{
assertQ(req("COUNTRY_CODES_mult_s:NR"),
"//*[@numFound='2']", "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
}
String[] michaelsSports = sportNamesByPersonId(3);
if(!underlyingDataModified || michaelsSports.length>0) {
String[] xpath = new String[michaelsSports.length + 1];
xpath[0] = "//*[@numFound='1']";
int i=1;
for(String ms : michaelsSports) {
xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms + "'";
i++;
}
assertQ(req("NAME_mult_s:Michael"), xpath);
}
if(checkDatabaseRequests) {
Assert.assertTrue("Expecting more than " + dbRequestsMoreThan
+ " database calls, but DIH reported " + totalDatabaseRequests(),
totalDatabaseRequests() > dbRequestsMoreThan);
Assert.assertTrue("Expecting fewer than " + dbRequestsLessThan
+ " database calls, but DIH reported " + totalDatabaseRequests(),
totalDatabaseRequests() < dbRequestsLessThan);
}
}
public void refreshDatabase() throws Exception
{
underlyingDataModified = false;
Connection conn = null;
Statement s = null;
PreparedStatement ps = null;
Timestamp theTime = new Timestamp(System.currentTimeMillis() - 10000); //10 seconds ago
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
s = conn.createStatement();
try {
s.executeUpdate("drop table countries");
s.executeUpdate("drop table people");
s.executeUpdate("drop table people_sports");
} catch(Exception e) {
//ignore. dbs complain when tables do no exist.
}
s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
ps = conn.prepareStatement("insert into countries (code, country_name, last_modified) values (?,?,?)");
for(String[] country : countries) {
ps.setString(1, country[0]);
ps.setString(2, country[1]);
ps.setTimestamp(3, theTime);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
ps = conn.prepareStatement("insert into people (id, name, country_code, last_modified) values (?,?,?,?)");
for(Object[] person : people) {
ps.setInt(1, (Integer) person[0]);
ps.setString(2, (String) person[1]);
ps.setString(3, (String) person[2]);
ps.setTimestamp(4, theTime);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
ps = conn.prepareStatement("insert into people_sports (id, person_id, sport_name, last_modified) values (?,?,?,?)");
for(Object[] sport : people_sports) {
ps.setInt(1, (Integer) sport[0]);
ps.setInt(2, (Integer) sport[1]);
ps.setString(3, (String) sport[2]);
ps.setTimestamp(4, theTime);
Assert.assertEquals(1, ps.executeUpdate());
}
ps.close();
conn.commit();
conn.close();
} catch(Exception e) {
throw e;
} finally {
try { ps.close(); } catch(Exception ex) { }
try { s.close(); } catch(Exception ex) { }
try { conn.close(); } catch(Exception ex) { }
}
}
protected abstract String deltaQueriesCountryTable() ;
protected abstract String deltaQueriesPersonTable() ;
private int getIntFromQuery(String query) throws Exception {
Connection conn = null;
Statement s = null;
ResultSet rs = null;
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
s = conn.createStatement();
rs = s.executeQuery(query);
if(rs.next()) {
return rs.getInt(1);
}
return 0;
} catch (SQLException e) {
throw e;
} finally {
try { rs.close(); } catch(Exception ex) { }
try { s.close(); } catch(Exception ex) { }
try { conn.close(); } catch(Exception ex) { }
}
}
private String[] getStringsFromQuery(String query) throws Exception {
Connection conn = null;
Statement s = null;
ResultSet rs = null;
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
s = conn.createStatement();
rs = s.executeQuery(query);
List<String> results = new ArrayList<String>();
while(rs.next()) {
results.add(rs.getString(1));
}
return results.toArray(new String[results.size()]);
} catch (SQLException e) {
throw e;
} finally {
try { rs.close(); } catch(Exception ex) { }
try { s.close(); } catch(Exception ex) { }
try { conn.close(); } catch(Exception ex) { }
}
}
public int totalCountries() throws Exception {
return getIntFromQuery("SELECT COUNT(1) FROM COUNTRIES WHERE DELETED != 'Y' ");
}
public int totalPeople() throws Exception {
return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
}
public boolean countryCodeExists(String cc) throws Exception {
return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='" + cc + "'")>0;
}
public String countryNameByCode(String cc) throws Exception {
String[] s = getStringsFromQuery("SELECT country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='" + cc + "'");
return s.length==0 ? null : s[0];
}
public int numberPeopleByCountryCode(String cc) throws Exception {
return getIntFromQuery(
"Select count(1) " +
"from people p " +
"inner join countries c on p.country_code=c.code " +
"where p.deleted!='Y' and c.deleted!='Y' and c.code='" + cc + "'");
}
public String[] sportNamesByPersonId(int personId) throws Exception {
return getStringsFromQuery(
"SELECT ps.SPORT_NAME " +
"FROM people_sports ps " +
"INNER JOIN PEOPLE p ON p.id = ps.person_Id " +
"WHERE ps.DELETED != 'Y' AND p.DELETED != 'Y' " +
"AND ps.person_id=" + personId + " " +
"ORDER BY ps.id"
);
}
public boolean personNameExists(String pn) throws Exception {
return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' AND NAME='" + pn + "'")>0;
}
public String personNameById(int id) throws Exception {
String[] nameArr= getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE ID=" + id);
if(nameArr.length==0) {
return null;
}
return nameArr[0];
}
public int totalDatabaseRequests(String dihHandlerName) throws Exception {
LocalSolrQueryRequest request = lrf.makeRequest("indent", "true");
String response = h.query(dihHandlerName, request);
Matcher m = totalRequestsPattern.matcher(response);
Assert.assertTrue("The handler " + dihHandlerName
+ " is not reporting any database requests. ", m.find()
&& m.groupCount() == 1);
return Integer.parseInt(m.group(1));
}
public int totalDatabaseRequests() throws Exception {
return totalDatabaseRequests("/dataimport");
}
public IntChanges modifySomePeople() throws Exception {
underlyingDataModified = true;
int numberToChange = random().nextInt(people.length + 1);
Set<Integer> changeSet = new HashSet<Integer>();
Set<Integer> deleteSet = new HashSet<Integer>();
Set<Integer> addSet = new HashSet<Integer>();
Connection conn = null;
PreparedStatement change = null;
PreparedStatement delete = null;
PreparedStatement add = null;
//One second in the future ensures a change time after the last import (DIH uses second precision only)
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
change = conn.prepareStatement("update people set name=?, last_modified=? where id=?");
delete = conn.prepareStatement("update people set deleted='Y', last_modified=? where id=?");
add = conn.prepareStatement("insert into people (id,name,country_code,last_modified) values (?,?,'NEW',?)");
for(int i=0 ; i<numberToChange ; i++) {
int tryIndex = random().nextInt(people.length);
Integer id = (Integer) people[tryIndex][0];
if(!changeSet.contains(id) && !deleteSet.contains(id)) {
boolean changeDontDelete = random().nextBoolean();
if(changeDontDelete) {
changeSet.add(id);
change.setString(1, "MODIFIED " + people[tryIndex][1]);
change.setTimestamp(2, theTime);
change.setInt(3, id);
Assert.assertEquals(1, change.executeUpdate());
} else {
deleteSet.add(id);
delete.setTimestamp(1, theTime);
delete.setInt(2, id);
Assert.assertEquals(1, delete.executeUpdate());
}
}
}
int numberToAdd = random().nextInt(3);
for(int i=0 ; i<numberToAdd ; i++) {
int tryIndex = random().nextInt(people.length);
Integer id = (Integer) people[tryIndex][0];
Integer newId = id+1000;
String newDesc = "ADDED " + people[tryIndex][1];
if(!addSet.contains(newId)) {
addSet.add(newId);
add.setInt(1, newId);
add.setString(2, newDesc);
add.setTimestamp(3, theTime);
Assert.assertEquals(1, add.executeUpdate());
}
}
conn.commit();
} catch (SQLException e) {
throw e;
} finally {
try { change.close(); } catch(Exception ex) { }
try { conn.close(); } catch(Exception ex) { }
}
IntChanges c = new IntChanges();
c.changedKeys=changeSet.toArray(new Integer[changeSet.size()]);
c.deletedKeys=deleteSet.toArray(new Integer[deleteSet.size()]);
c.addedKeys=addSet.toArray(new Integer[addSet.size()]);
return c;
}
public String[] modifySomeCountries() throws Exception {
underlyingDataModified = true;
int numberToChange = random().nextInt(countries.length + 1);
Set<String> changeSet = new HashSet<String>();
Connection conn = null;
PreparedStatement change = null;
// One second in the future ensures a change time after the last import (DIH
// uses second precision only)
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
try {
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
change = conn
.prepareStatement("update countries set country_name=?, last_modified=? where code=?");
for (int i = 0; i < numberToChange; i++) {
int tryIndex = random().nextInt(countries.length);
String code = countries[tryIndex][0];
if (!changeSet.contains(code)) {
changeSet.add(code);
change.setString(1, "MODIFIED " + countries[tryIndex][1]);
change.setTimestamp(2, theTime);
change.setString(3, code);
Assert.assertEquals(1, change.executeUpdate());
}
}
} catch (SQLException e) {
throw e;
} finally {
try {
change.close();
} catch (Exception ex) {}
try {
conn.close();
} catch (Exception ex) {}
}
return changeSet.toArray(new String[changeSet.size()]);
}
class IntChanges {
public Integer[] changedKeys;
public Integer[] deletedKeys;
public Integer[] addedKeys;
}
protected LocalSolrQueryRequest generateRequest() {
return lrf.makeRequest("command", "full-import", "dataConfig", generateConfig(),
"clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
}
protected String generateConfig() {
StringBuilder sb = new StringBuilder();
sb.append("<dataConfig> \n");
sb.append("<dataSource name=''hsqldb'' driver=''org.hsqldb.jdbcDriver'' url=''jdbc:hsqldb:mem:.'' /> \n");
sb.append("<document name=''TestSqlEntityProcessor''> \n");
sb.append("<entity name=''People'' ");
sb.append("pk=''" + (random().nextBoolean() ? "ID" : "People.ID") + "'' ");
sb.append("processor=''SqlEntityProcessor'' dataSource=''hsqldb'' ");
sb.append(rootTransformerName !=null ? "transformer=''" + rootTransformerName + "'' " : "");
sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' '' ");
sb.append(deltaQueriesPersonTable());
sb.append("> \n");
sb.append("<field column=''NAME'' name=''NAME_mult_s'' /> \n");
sb.append("<field column=''COUNTRY_CODE'' name=''COUNTRY_CODES_mult_s'' /> \n");
if(countryEntity) {
sb.append("<entity name=''Countries'' ");
sb.append("pk=''" + (random().nextBoolean() ? "CODE" : "Countries.CODE") + "'' ");
sb.append("dataSource=''hsqldb'' ");
sb.append(countryTransformer ? "transformer=''AddAColumnTransformer'' " +
"newColumnName=''countryAdded_s'' newColumnValue=''country_added'' " : "");
if(countryCached) {
sb.append(random().nextBoolean() ?
"processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' " :
"processor=''CachedSqlEntityProcessor'' "
);
if(useSimpleCaches) {
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
} else {
sb.append(random().nextBoolean() ?
"cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' " :
"where=''CODE=People.COUNTRY_CODE'' "
);
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
sb.append("> \n");
}
} else {
sb.append("processor=''SqlEntityProcessor'' query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' '' ");
sb.append(deltaQueriesCountryTable());
sb.append("> \n");
}
sb.append("<field column=''CODE'' name=''COUNTRY_CODE_s'' /> \n");
sb.append("<field column=''COUNTRY_NAME'' name=''COUNTRY_NAME_s'' /> \n");
sb.append("</entity> \n");
}
if(sportsEntity) {
sb.append("<entity name=''Sports'' dataSource=''hsqldb'' "
+ (sportsTransformer ? "transformer=''AddAColumnTransformer'' " +
"newColumnName=''sportsAdded_s'' newColumnValue=''sport_added'' " : ""));
if(sportsCached) {
sb.append(random().nextBoolean() ?
"processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' " :
"processor=''CachedSqlEntityProcessor'' "
);
if(useSimpleCaches) {
sb.append("query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID='${People.ID}' ORDER BY ID'' ");
} else {
sb.append(random().nextBoolean() ?
"cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' " :
"where=''PERSON_ID=People.ID'' "
);
sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
}
} else {
sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID='${People.ID}' ORDER BY ID'' ");
}
sb.append("> \n");
sb.append("<field column=''SPORT_NAME'' name=''SPORT_NAME_mult_s'' /> \n");
sb.append("<field column=''id'' name=''SPORT_ID_mult_s'' /> \n");
sb.append("</entity> \n");
}
sb.append("</entity> \n");
sb.append("</document> \n");
sb.append("</dataConfig> \n");
String config = sb.toString().replaceAll("[']{2}", "\"");
log.debug(config);
return config;
}
public static final String[][] countries = {
{"NA", "Namibia"},
@ -118,7 +632,7 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
public static final Object[][] people = {
{1,"Jacob","NZ"},
{2,"Ethan","NU,NA,NE"},
{2,"Ethan","NU"},
{3,"Michael","NR"},
{4,"Jayden","NP"},
{5,"William","NO"},
@ -134,7 +648,7 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
{15,"Lily","NE"},
{16,"Madison","NC"},
{17,"Mia","NA"},
{18,"Natalie","NP,NR,NU,NZ"},
{18,"Natalie","NZ"},
{19,"Olivia","NU"},
{20,"Samantha","NR"}
};
@ -165,6 +679,5 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
{1800, 18, "White Water Rafting"},
{1900, 19, "Water skiing"},
{2000, 20, "Windsurfing"}
};
};
}

32
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/AddAColumnTransformer.java Normal file
View File

@ -0,0 +1,32 @@
package org.apache.solr.handler.dataimport;
import java.util.Map;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class AddAColumnTransformer extends Transformer {
@Override
public Object transformRow(Map<String,Object> aRow, Context context) {
String colName = context.getEntityAttribute("newColumnName");
colName = colName==null ? "AddAColumn_s" : colName;
String colValue = context.getEntityAttribute("newColumnValue");
colValue = colValue==null ? "Added" : colValue;
aRow.put(colName, colValue);
return aRow;
}
}

260
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestCachedSqlEntityProcessor.java
View File

@ -1,260 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
/**
* <p>
* Test for CachedSqlEntityProcessor
* </p>
*
*
* @since solr 1.3
*/
public class TestCachedSqlEntityProcessor extends AbstractDataImportHandlerTestCase {
@Test
public void withoutWhereClause() {
List fields = new ArrayList();
fields.add(createMap("column", "id"));
fields.add(createMap("column", "desc"));
String q = "select * from x where id=${x.id}";
Map<String, String> entityAttrs = createMap("query", q);
MockDataSource ds = new MockDataSource();
VariableResolverImpl vr = new VariableResolverImpl();
vr.addNamespace("x", createMap("id", 1));
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
rows.add(createMap("id", 1, "desc", "one"));
rows.add(createMap("id", 1, "desc", "another one"));
MockDataSource.setIterator(vr.replaceTokens(q), rows.iterator());
EntityProcessor csep = new EntityProcessorWrapper(new CachedSqlEntityProcessor(), null, null);
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(2, rows.size());
ds.close();
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(2, rows.size());
assertEquals(2, rows.get(0).size());
assertEquals(2, rows.get(1).size());
}
@Test
public void withoutWhereClauseWithTransformers() {
List fields = new ArrayList();
fields.add(createMap("column", "id"));
fields.add(createMap("column", "desc"));
String q = "select * from x where id=${x.id}";
Map<String, String> entityAttrs = createMap(
"query", q, "transformer", UppercaseTransformer.class.getName());
MockDataSource ds = new MockDataSource();
VariableResolverImpl vr = new VariableResolverImpl();
vr.addNamespace("x", createMap("id", 1));
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
rows.add(createMap("id", 1, "desc", "one"));
rows.add(createMap("id", 1, "desc", "another one"));
MockDataSource.setIterator(vr.replaceTokens(q), rows.iterator());
EntityProcessor csep = new EntityProcessorWrapper( new CachedSqlEntityProcessor(), null, null);
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(2, rows.size());
ds.close();
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
assertEquals(r.get("desc").toString().toUpperCase(Locale.ROOT), r.get("desc"));
}
assertEquals(2, rows.size());
assertEquals(2, rows.get(0).size());
assertEquals(2, rows.get(1).size());
}
@Test
public void withoutWhereClauseWithMultiRowTransformer() {
List fields = new ArrayList();
fields.add(createMap("column", "id"));
fields.add(createMap("column", "desc"));
String q = "select * from x where id=${x.id}";
Map<String, String> entityAttrs = createMap(
"query", q, "transformer", DoubleTransformer.class.getName());
MockDataSource ds = new MockDataSource();
VariableResolverImpl vr = new VariableResolverImpl();
vr.addNamespace("x", createMap("id", 1));
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
rows.add(createMap("id", 1, "desc", "one"));
rows.add(createMap("id", 1, "desc", "another one"));
MockDataSource.setIterator(vr.replaceTokens(q), rows.iterator());
EntityProcessor csep = new EntityProcessorWrapper( new CachedSqlEntityProcessor(), null, null);
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(4, rows.size());
ds.close();
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(4, rows.size());
assertEquals(2, rows.get(0).size());
assertEquals(2, rows.get(1).size());
}
public static class DoubleTransformer extends Transformer {
@Override
public Object transformRow(Map<String, Object> row, Context context) {
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
rows.add(row);
rows.add(row);
return rows;
}
}
public static class UppercaseTransformer extends Transformer {
@Override
public Object transformRow(Map<String, Object> row, Context context) {
for (Map.Entry<String, Object> entry : row.entrySet()) {
Object val = entry.getValue();
if (val instanceof String) {
String s = (String) val;
entry.setValue(s.toUpperCase(Locale.ROOT));
}
}
return row;
}
}
@Test
public void withKeyAndLookup() {
List fields = new ArrayList();
fields.add(createMap("column", "id"));
fields.add(createMap("column", "desc"));
String q = "select * from x";
Map<String, String> entityAttrs = createMap(
"query", q, DIHCacheSupport.CACHE_PRIMARY_KEY,"id", DIHCacheSupport.CACHE_FOREIGN_KEY ,"x.id");
MockDataSource ds = new MockDataSource();
VariableResolverImpl vr = new VariableResolverImpl();
Map xNamespace = createMap("id", 0);
vr.addNamespace("x", xNamespace);
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
doWhereTest(q, context, ds, xNamespace);
}
@Test
public void withWhereClause() {
List fields = new ArrayList();
fields.add(createMap("column", "id"));
fields.add(createMap("column", "desc"));
String q = "select * from x";
Map<String, String> entityAttrs = createMap("query", q, "where", "id=x.id");
MockDataSource ds = new MockDataSource();
VariableResolverImpl vr = new VariableResolverImpl();
Map xNamespace = createMap("id", 0);
vr.addNamespace("x", xNamespace);
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
doWhereTest(q, context, ds, xNamespace);
}
private void doWhereTest(String q, Context context, MockDataSource ds, Map xNamespace) {
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
rows.add(createMap("desc", "one", "id", 1));
rows.add(createMap("id", 2, "desc", "two"));
rows.add(createMap("id", 2, "desc", "another two"));
rows.add(createMap("id", 3, "desc", "three"));
rows.add(createMap("id", 3, "desc", "another three"));
rows.add(createMap("id", 3, "desc", "another another three"));
MockDataSource.setIterator(q, rows.iterator());
EntityProcessor csep = new EntityProcessorWrapper(new CachedSqlEntityProcessor(), null, null);
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(0, rows.size());
ds.close();
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
xNamespace.put("id", 2);
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(2, rows.size());
csep.init(context);
rows = new ArrayList<Map<String, Object>>();
xNamespace.put("id", 3);
while (true) {
Map<String, Object> r = csep.nextRow();
if (r == null)
break;
rows.add(r);
}
assertEquals(3, rows.size());
}
}

59
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDIHEndToEnd.java
View File

@ -1,59 +0,0 @@
package org.apache.solr.handler.dataimport;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to You under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestDIHEndToEnd extends AbstractDIHJdbcTestCase {
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig-end-to-end.xml", "dataimport-schema.xml");
}
@Test
public void testEndToEnd() throws Exception {
LocalSolrQueryRequest request = lrf.makeRequest(
"command", "full-import", "config", "data-config-end-to-end.xml",
"clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
h.query("/dataimport-end-to-end", request);
assertQ(req("*:*"), "//*[@numFound='20']");
assertQ(req("COUNTRY_NAME:zealand"), "//*[@numFound='2']");
assertQ(req("COUNTRY_NAME:niue"), "//*[@numFound='3']");
//It would be nice if there was a way to get it to run transformers before putting
//data in the cache, then id=2 (person=Ethan, country=NU,NA,NE) could join...)
//assertQ(req("COUNTRY_NAME:Netherlands"), "//*[@numFound='3']");
assertQ(req("NAME:michael"), "//*[@numFound='1']");
assertQ(req("SPORT_NAME:kayaking"), "//*[@numFound='2']");
assertQ(req("SPORT_NAME:fishing"), "//*[@numFound='1']");
request = lrf.makeRequest("indent", "true");
String response = h.query("/dataimport-end-to-end", request);
Matcher m = Pattern.compile(".str name..Total Requests made to DataSource..(\\d+)..str.").matcher(response);
Assert.assertTrue(m.find() && m.groupCount()==1);
int numRequests = Integer.parseInt(m.group(1));
Assert.assertTrue(
"The database should have been hit once each " +
"for 'Person' & 'Country' and ~20 times for 'Sport'", numRequests<30);
}
}

92
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestNonWritablePersistFile.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
public class TestNonWritablePersistFile extends AbstractDataImportHandlerTestCase {
private static final String FULLIMPORT_QUERY = "select * from x";
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
private static final String dataConfig_delta =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>\n" +
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
" query=\"" + FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
" <field column=\"id\" name=\"id\"/>\n" +
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
" <field column=\"desc\" />\n" +
" </entity>\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
@Test
@SuppressWarnings("unchecked")
public void testNonWritablePersistFile() throws Exception {
// See SOLR-2551
String configDir = h.getCore().getResourceLoader().getConfigDir();
String filePath = configDir;
if (configDir != null && !configDir.endsWith(File.separator))
filePath += File.separator;
filePath += "dataimport.properties";
File f = new File(filePath);
try {
// execute the test only if we are able to set file to read only mode
assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly());
assumeFalse("dataimport.proprties is still writable even though " +
"marked readonly - test running as superuser?", f.canWrite());
ignoreException("Properties is not writable");
@SuppressWarnings("rawtypes")
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
@SuppressWarnings("rawtypes")
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'",
childRow.iterator());
runFullImport(dataConfig_delta);
assertQ(req("id:1"), "//*[@numFound='0']");
} finally {
f.delete();
}
}
}

183
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor.java
View File

@ -1,3 +1,8 @@
package org.apache.solr.handler.dataimport;
import org.junit.Ignore;
import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -14,168 +19,44 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Test;
import org.junit.Ignore;
import java.util.*;
/**
* <p>
* Test for SqlEntityProcessor
* </p>
*
*
* @since solr 1.3
* Test with various combinations of parameters, child entites, caches, transformers.
*/
@Ignore("FIXME: I fail so often it makes me ill!")
public class TestSqlEntityProcessor extends AbstractDataImportHandlerTestCase {
private static ThreadLocal<Integer> local = new ThreadLocal<Integer>();
public class TestSqlEntityProcessor extends AbstractDIHJdbcTestCase {
@Test
public void testSingleBatch() {
SqlEntityProcessor sep = new SqlEntityProcessor();
List<Map<String, Object>> rows = getRows(3);
VariableResolverImpl vr = new VariableResolverImpl();
HashMap<String, String> ea = new HashMap<String, String>();
ea.put("query", "SELECT * FROM A");
Context c = getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea);
sep.init(c);
int count = 0;
while (true) {
Map<String, Object> r = sep.nextRow();
if (r == null)
break;
count++;
}
assertEquals(3, count);
}
public void testSingleEntity() throws Exception {
singleEntity(1);
}
@Test
public void testTranformer() {
EntityProcessor sep = new EntityProcessorWrapper( new SqlEntityProcessor(), null, null);
List<Map<String, Object>> rows = getRows(2);
VariableResolverImpl vr = new VariableResolverImpl();
HashMap<String, String> ea = new HashMap<String, String>();
ea.put("query", "SELECT * FROM A");
ea.put("transformer", T.class.getName());
sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
Map<String, Object> r = null;
while (true) {
r = sep.nextRow();
if (r == null)
break;
rs.add(r);
}
assertEquals(2, rs.size());
assertNotNull(rs.get(0).get("T"));
public void testWithSimpleTransformer() throws Exception {
simpleTransform(1);
}
@Test
public void testTranformerWithReflection() {
EntityProcessor sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
List<Map<String, Object>> rows = getRows(2);
VariableResolverImpl vr = new VariableResolverImpl();
HashMap<String, String> ea = new HashMap<String, String>();
ea.put("query", "SELECT * FROM A");
ea.put("transformer", T3.class.getName());
sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
Map<String, Object> r = null;
while (true) {
r = sep.nextRow();
if (r == null)
break;
rs.add(r);
}
assertEquals(2, rs.size());
assertNotNull(rs.get(0).get("T3"));
public void testWithComplexTransformer() throws Exception {
complexTransform(1, 0);
}
@Test
public void testTranformerList() {
EntityProcessor sep = new EntityProcessorWrapper(new SqlEntityProcessor(),null, null);
List<Map<String, Object>> rows = getRows(2);
VariableResolverImpl vr = new VariableResolverImpl();
HashMap<String, String> ea = new HashMap<String, String>();
ea.put("query", "SELECT * FROM A");
ea.put("transformer", T2.class.getName());
sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
local.set(0);
Map<String, Object> r = null;
int count = 0;
while (true) {
r = sep.nextRow();
if (r == null)
break;
count++;
}
assertEquals(2, (int) local.get());
assertEquals(4, count);
public void testChildEntities() throws Exception {
withChildEntities(false, true);
}
private List<Map<String, Object>> getRows(int count) {
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
for (int i = 0; i < count; i++) {
Map<String, Object> row = new HashMap<String, Object>();
row.put("id", i);
row.put("value", "The value is " + i);
rows.add(row);
}
return rows;
@Test
public void testCachedChildEntities() throws Exception {
withChildEntities(true, true);
}
private static DataSource<Iterator<Map<String, Object>>> getDs(
final List<Map<String, Object>> rows) {
return new DataSource<Iterator<Map<String, Object>>>() {
@Override
public Iterator<Map<String, Object>> getData(String query) {
return rows.iterator();
}
@Override
public void init(Context context, Properties initProps) {
}
@Override
public void close() {
}
};
@Test
@Ignore("broken see SOLR-3857")
public void testSimpleCacheChildEntities() throws Exception {
simpleCacheChildEntities(true);
}
public static class T extends Transformer {
@Override
public Object transformRow(Map<String, Object> aRow, Context context) {
aRow.put("T", "Class T");
return aRow;
}
}
public static class T3 {
public Object transformRow(Map<String, Object> aRow) {
aRow.put("T3", "T3 class");
return aRow;
}
}
public static class T2 extends Transformer {
@Override
public Object transformRow(Map<String, Object> aRow, Context context) {
Integer count = local.get();
local.set(count + 1);
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
l.add(aRow);
l.add(aRow);
return l;
}
@Override
protected String deltaQueriesCountryTable() {
return "";
}
@Override
protected String deltaQueriesPersonTable() {
return "";
}
}

162
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessor2.java
View File

@ -41,150 +41,24 @@ public class TestSqlEntityProcessor2 extends AbstractDataImportHandlerTestCase {
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
@Before @Override
@Before
@Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
assertU(commit());
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x", parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A=1", childRow
.iterator());
runFullImport(dataConfig);
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImportWithoutCommit() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "10"));
MockDataSource.setIterator("select * from x", parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A=10", childRow
.iterator());
runFullImport(dataConfig,createMap("commit","false"));
assertQ(req("id:10"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "5"));
MockDataSource.setIterator("select id from x where last_modified > NOW",
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "5"));
MockDataSource.setIterator("select * from x where id = '5'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A=5", childRow
.iterator());
runDeltaImport(dataConfig);
assertQ(req("id:5"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_DeletedPkQuery() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "11"));
MockDataSource.setIterator("select * from x", parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A=11", childRow
.iterator());
runFullImport(dataConfig);
assertQ(req("id:11"), "//*[@numFound='1']");
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "15"));
deltaRow.add(createMap("id", "17"));
MockDataSource.setIterator("select id from x where last_modified > NOW",
deltaRow.iterator());
List deltaDeleteRow = new ArrayList();
deltaDeleteRow.add(createMap("id", "11"));
deltaDeleteRow.add(createMap("id", "17"));
MockDataSource.setIterator("select id from x where last_modified > NOW AND deleted='true'",
deltaDeleteRow.iterator());
parentRow = new ArrayList();
parentRow.add(createMap("id", "15"));
MockDataSource.setIterator("select * from x where id = '15'", parentRow
.iterator());
parentRow = new ArrayList();
parentRow.add(createMap("id", "17"));
MockDataSource.setIterator("select * from x where id = '17'", parentRow
.iterator());
runDeltaImport(dataConfig);
assertQ(req("id:15"), "//*[@numFound='1']");
assertQ(req("id:11"), "//*[@numFound='0']");
assertQ(req("id:17"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_DeltaImportQuery() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "5"));
MockDataSource.setIterator("select id from x where last_modified > NOW",
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "5"));
MockDataSource.setIterator("select * from x where id=5", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A=5", childRow
.iterator());
runDeltaImport(dataConfig_deltaimportquery);
assertQ(req("id:5"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
@Ignore("Known Locale/TZ problems: see https://issues.apache.org/jira/browse/SOLR-1916")
/**
* This test is here for historical purposes only.
* When SOLR-1916 is fixed, it would be best to rewrite this test.
*
* @throws Exception
*/
public void testLastIndexTime() throws Exception {
List row = new ArrayList();
row.add(createMap("id", 5));
@ -220,22 +94,4 @@ public class TestSqlEntityProcessor2 extends AbstractDataImportHandlerTestCase {
"\t\t<entity name=\"x\" query=\"select * from x where last_modified > ${dih.functions.checkDateFormat(dih.last_index_time)}\" />\n" +
"\t</document>\n" +
"</dataConfig>";
private static String dataConfig = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
+ " <document>\n"
+ " <entity name=\"x\" pk=\"id\" query=\"select * from x\" deletedPkQuery=\"select id from x where last_modified > NOW AND deleted='true'\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
+ " <field column=\"id\" />\n"
+ " <entity name=\"y\" query=\"select * from y where y.A=${x.id}\">\n"
+ " <field column=\"desc\" />\n"
+ " </entity>\n" + " </entity>\n"
+ " </document>\n" + "</dataConfig>\n";
private static String dataConfig_deltaimportquery = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
+ " <document>\n"
+ " <entity name=\"x\" deltaImportQuery=\"select * from x where id=${dataimporter.delta.id}\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
+ " <field column=\"id\" />\n"
+ " <entity name=\"y\" query=\"select * from y where y.A=${x.id}\">\n"
+ " <field column=\"desc\" />\n"
+ " </entity>\n" + " </entity>\n"
+ " </document>\n" + "</dataConfig>\n";
}

439
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
View File

@ -1,3 +1,9 @@
package org.apache.solr.handler.dataimport;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.junit.Before;
import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@ -14,330 +20,133 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* <p>
* Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
* </p>
*
*
*
* @since solr 1.3
* Test with various combinations of parameters, child entites, transformers.
*/
@Ignore("FIXME: I fail so often it makes me ill!")
public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCase {
private static final String FULLIMPORT_QUERY = "select * from x";
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
private static final String dataConfig_delta =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>\n" +
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
" query=\"" + FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
" <field column=\"id\" name=\"id\"/>\n" +
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
" <field column=\"desc\" />\n" +
" </entity>\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
//TODO: fix this test to not require FSDirectory.
static String savedFactory;
@BeforeClass
public static void beforeClass() throws Exception {
savedFactory = System.getProperty("solr.DirectoryFactory");
System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
public class TestSqlEntityProcessorDelta extends AbstractDIHJdbcTestCase {
private boolean delta = false;
private boolean useParentDeltaQueryParam = false;
private IntChanges personChanges = null;
private String[] countryChanges = null;
@Before
public void setupDeltaTest() {
delta = false;
personChanges = null;
countryChanges = null;
}
@Test
public void testSingleEntity() throws Exception {
singleEntity(1);
changeStuff();
int c = calculateDatabaseCalls();
singleEntity(c);
validateChanges();
}
@Test
public void testWithSimpleTransformer() throws Exception {
simpleTransform(1);
changeStuff();
simpleTransform(calculateDatabaseCalls());
validateChanges();
}
@Test
public void testWithComplexTransformer() throws Exception {
complexTransform(1, 0);
changeStuff();
complexTransform(calculateDatabaseCalls(), personChanges.deletedKeys.length);
validateChanges();
}
@Test
public void testChildEntities() throws Exception {
useParentDeltaQueryParam = random().nextBoolean();
withChildEntities(false, true);
changeStuff();
withChildEntities(false, false);
validateChanges();
}
@AfterClass
public static void afterClass() {
if (savedFactory == null) {
System.clearProperty("solr.directoryFactory");
private int calculateDatabaseCalls() {
//The main query generates 1
//Deletes generate 1
//Each add/mod generate 1
int c = 1;
if (countryChanges != null) {
c += countryChanges.length + 1;
}
if (personChanges != null) {
c += personChanges.addedKeys.length + personChanges.changedKeys.length + 1;
}
return c;
}
private void validateChanges() throws Exception
{
if(personChanges!=null) {
for(int id : personChanges.addedKeys) {
assertQ(req("id:" + id), "//*[@numFound='1']");
}
for(int id : personChanges.deletedKeys) {
assertQ(req("id:" + id), "//*[@numFound='0']");
}
for(int id : personChanges.changedKeys) {
assertQ(req("id:" + id), "//*[@numFound='1']", "substring(//doc/arr[@name='NAME_mult_s']/str[1], 1, 8)='MODIFIED'");
}
}
if(countryChanges!=null) {
for(String code : countryChanges) {
assertQ(req("COUNTRY_CODE_s:" + code), "//*[@numFound='" + numberPeopleByCountryCode(code) + "']", "substring(//doc/str[@name='COUNTRY_NAME_s'], 1, 8)='MODIFIED'");
}
}
}
private void changeStuff() throws Exception {
if(countryEntity)
{
int n = random().nextInt(2);
switch(n) {
case 0:
personChanges = modifySomePeople();
break;
case 1:
countryChanges = modifySomeCountries();
break;
case 2:
personChanges = modifySomePeople();
countryChanges = modifySomeCountries();
break;
}
} else {
System.setProperty("solr.directoryFactory", savedFactory);
personChanges = modifySomePeople();
}
delta = true;
}
protected LocalSolrQueryRequest generateRequest() {
return lrf.makeRequest("command", (delta ? "delta-import" : "full-import"), "dataConfig", generateConfig(),
"clean", (delta ? "false" : "true"), "commit", "true", "synchronous", "true", "indent", "true");
}
@Before @Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
assertU(commit());
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runFullImport(dataConfig_delta);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
add1document();
}
@Test
@SuppressWarnings("unchecked")
public void testNonWritablePersistFile() throws Exception {
// See SOLR-2551
String configDir = h.getCore().getResourceLoader().getConfigDir();
String filePath = configDir;
if (configDir != null && !configDir.endsWith(File.separator))
filePath += File.separator;
filePath += "dataimport.properties";
File f = new File(filePath);
try {
// execute the test only if we are able to set file to read only mode
assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly());
assumeFalse("dataimport.proprties is still writable even though " +
"marked readonly - test running as superuser?", f.canWrite());
ignoreException("Properties is not writable");
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'",
childRow.iterator());
runFullImport(dataConfig_delta);
assertQ(req("id:1"), "//*[@numFound='0']");
} finally {
f.delete();
}
}
// WORKS
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_delete() throws Exception {
add1document();
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
MockDataSource.setIterator(DELTA_QUERY, Collections
.EMPTY_LIST.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_empty() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'",
childRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
// WORKS
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
protected String deltaQueriesPersonTable() {
return
"deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified &gt;='${dih.last_index_time}' '' " +
"deltaImportQuery=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE where ID=${dih.delta.ID} '' " +
"deltaQuery=''" +
"SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified &gt;='${dih.last_index_time}' " +
(useParentDeltaQueryParam ? "" :
"UNION DISTINCT " +
"SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified &gt;='${dih.last_index_time}') "
) + "'' "
;
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_nodelta() throws Exception {
add1document();
MockDataSource.clearCache();
MockDataSource.setIterator(DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("id:1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
@Override
protected String deltaQueriesCountryTable() {
if(useParentDeltaQueryParam) {
return
"deltaQuery=''SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y' AND last_modified &gt;='${dih.last_index_time}' '' " +
"parentDeltaQuery=''SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND COUNTRY_CODE='${Countries.CODE}' '' "
;
}
return "";
}
}

288
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
View File

@ -1,288 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* <p>
* Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
* </p>
*
*
*
* @since solr 1.3
*/
@Ignore("FIXME: I fail so often it makes me ill!")
public class TestSqlEntityProcessorDelta2 extends AbstractDataImportHandlerTestCase {
private static final String FULLIMPORT_QUERY = "select * from x";
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
private static final String dataConfig_delta2 =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>\n" +
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
" query=\"" + FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
" <field column=\"tmpid\" template=\"prefix-${x.id}\" name=\"solr_id\"/>\n" +
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
" <field column=\"desc\" />\n" +
" </entity>\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-solr_id-schema.xml");
}
@Before @Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
assertU(commit());
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runFullImport(dataConfig_delta2);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
add1document();
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_delete() throws Exception {
add1document();
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
MockDataSource.setIterator(DELTA_QUERY, Collections
.EMPTY_LIST.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_empty() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_nodelta() throws Exception {
add1document();
MockDataSource.clearCache();
MockDataSource.setIterator(DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
}

236
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java
View File

@ -1,236 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@Ignore("FIXME: I fail so often it makes me ill!")
public class TestSqlEntityProcessorDelta3 extends AbstractDataImportHandlerTestCase {
private static final String P_FULLIMPORT_QUERY = "select * from parent";
private static final String P_DELTA_QUERY = "select parent_id from parent where last_modified > NOW";
private static final String P_DELTAIMPORT_QUERY = "select * from parent where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
private static final String C_FULLIMPORT_QUERY = "select * from child";
private static final String C_DELETED_PK_QUERY = "select id from child where last_modified > NOW AND deleted='true'";
private static final String C_DELTA_QUERY = "select id from child where last_modified > NOW";
private static final String C_PARENTDELTA_QUERY = "select parent_id from child where id=${child.id}";
private static final String C_DELTAIMPORT_QUERY = "select * from child where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
private static final String dataConfig_delta =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>" +
" <entity name=\"parent\" pk=\"parent_id\" rootEntity=\"false\"" +
" query=\"" + P_FULLIMPORT_QUERY + "\"" +
" deltaQuery=\"" + P_DELTA_QUERY + "\"" +
" deltaImportQuery=\"" + P_DELTAIMPORT_QUERY + "\">" +
" <field column=\"desc\" name=\"desc\"/>" +
" <entity name=\"child\" pk=\"id\" rootEntity=\"true\"" +
" query=\"" + C_FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + C_DELETED_PK_QUERY + "\"" +
" deltaQuery=\"" + C_DELTA_QUERY + "\"" +
" parentDeltaQuery=\"" + C_PARENTDELTA_QUERY + "\"" +
" deltaImportQuery=\"" + C_DELTAIMPORT_QUERY + "\">" +
" <field column=\"id\" name=\"id\" />" +
" </entity>" +
" </entity>" +
" </document>" +
"</dataConfig>\n";
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
@Before @Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
assertU(commit());
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("parent_id", "1", "desc", "d1"));
MockDataSource.setIterator(P_FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("id", "2"));
MockDataSource.setIterator(C_FULLIMPORT_QUERY, childRow.iterator());
runFullImport(dataConfig_delta);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='0']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:d1"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
add1document();
}
// WORKS
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_delete() throws Exception {
add1document();
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "2"));
MockDataSource.setIterator(C_DELETED_PK_QUERY, deletedRow.iterator());
MockDataSource.setIterator(C_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
List deletedParentRow = new ArrayList();
deletedParentRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator("select parent_id from child where id=2", deletedParentRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_empty() throws Exception {
List childDeltaRow = new ArrayList();
childDeltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(C_DELTA_QUERY, childDeltaRow.iterator());
MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections.EMPTY_LIST.iterator());
List childParentDeltaRow = new ArrayList();
childParentDeltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator("select parent_id from child where id=2", childParentDeltaRow.iterator());
MockDataSource.setIterator(P_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
List parentDeltaImportRow = new ArrayList();
parentDeltaImportRow.add(createMap("parent_id", "1", "desc", "d1"));
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
parentDeltaImportRow.iterator());
List childDeltaImportRow = new ArrayList();
childDeltaImportRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
childDeltaImportRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:d1"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator(P_DELTA_QUERY,
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("parent_id", "1", "desc", "d2"));
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
childRow.iterator());
MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:s1 OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
assertQ(req("desc:d2"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add() throws Exception {
add1document();
MockDataSource.clearCache();
List parentDeltaRow = new ArrayList();
parentDeltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator(P_DELTA_QUERY,
parentDeltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("parent_id", "1", "desc", "d1"));
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
parentRow.iterator());
List childDeltaRow = new ArrayList();
childDeltaRow.add(createMap("id", "3"));
MockDataSource.setIterator(C_DELTA_QUERY,
childDeltaRow.iterator());
List childParentDeltaRow = new ArrayList();
childParentDeltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator("select parent_id from child where id='3'",
childParentDeltaRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("id", "3"));
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
childRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("id:3"), "//*[@numFound='1']");
assertQ(req("desc:d1"), "//*[@numFound='2']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_nodelta() throws Exception {
add1document();
MockDataSource.clearCache();
MockDataSource.setIterator(P_DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
MockDataSource.setIterator(C_DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("id:2 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("desc:d1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
}
}

147
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDeltaPrefixedPk.java
View File

@ -1,147 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.logging.*;
/**
* <p>
* Test for SqlEntityProcessorDelta verifying fix for SOLR-1191
* </p>
*
*
*
* @since solr 3.1
*/
@Ignore("FIXME: I fail so often it makes me ill!")
public class TestSqlEntityProcessorDeltaPrefixedPk extends AbstractDataImportHandlerTestCase {
private static final String FULLIMPORT_QUERY = "select * from x";
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
private static final String dataConfig_delta =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>\n" +
" <entity name=\"x\" transformer=\"TemplateTransformer\" pk=\"x.id\"" +
" query=\"" + FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
" <field column=\"id\" name=\"id\"/>\n" +
" <field column=\"desc\" name=\"desc\"/>\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
private static final List EMPTY_LIST = Collections.EMPTY_LIST;
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
@Before @Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
assertU(commit());
//Logger.getLogger("").setLevel(Level.ALL);
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List row = new ArrayList();
row.add(createMap("id", "1", "desc", "bar"));
MockDataSource.setIterator(FULLIMPORT_QUERY, row.iterator());
runFullImport(dataConfig_delta);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:bar"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testDeltaImport_deleteResolvesUnprefixedPk() throws Exception {
add1document();
MockDataSource.clearCache();
List deletedRows = new ArrayList();
deletedRows.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRows.iterator());
MockDataSource.setIterator(DELTA_QUERY, EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testDeltaImport_deleteResolvesUnprefixedPk"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testDeltaImport_replace_resolvesUnprefixedPk() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRows = new ArrayList();
deltaRows.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY, deltaRows.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, EMPTY_LIST.iterator());
List rows = new ArrayList();
rows.add(createMap("id", "1", "desc", "baz"));
MockDataSource.setIterator("select * from x where id='1'", rows.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testDeltaImport_replace_resolvesUnprefixedPk"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:bar"), "//*[@numFound='0']");
assertQ(req("desc:baz"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testDeltaImport_addResolvesUnprefixedPk() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRows = new ArrayList();
deltaRows.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY, deltaRows.iterator());
List rows = new ArrayList();
rows.add(createMap("id", "2", "desc", "xyzzy"));
MockDataSource.setIterator("select * from x where id='2'", rows.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testDeltaImport_addResolvesUnprefixedPk"), "//*[@numFound='2']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:bar"), "//*[@numFound='1']");
assertQ(req("desc:xyzzy"), "//*[@numFound='1']");
}
}

75
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TripleThreatTransformer.java Normal file
View File

@ -0,0 +1,75 @@
package org.apache.solr.handler.dataimport;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* This transformer does 3 things
* <ul>
* <li>It turns every row into 3 rows,
* modifying any "id" column to ensure duplicate entries in the index
* <li>The 2nd Row has 2x values for every column,
* with the added one being backwards of the original
* <li>The 3rd Row has an added static value
* </ul>
*
* Also, this does not extend Transformer.
*/
public class TripleThreatTransformer {
public Object transformRow(Map<String, Object> row) {
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>(3);
rows.add(row);
rows.add(addDuplicateBackwardsValues(row));
rows.add(new LinkedHashMap<String,Object>(row));
rows.get(2).put("AddAColumn_s", "Added");
modifyIdColumn(rows.get(1), 1);
modifyIdColumn(rows.get(2), 2);
return rows;
}
private LinkedHashMap<String,Object> addDuplicateBackwardsValues(Map<String, Object> row) {
LinkedHashMap<String,Object> n = new LinkedHashMap<String,Object>();
for(Map.Entry<String,Object> entry : row.entrySet()) {
String key = entry.getKey();
if(!"id".equalsIgnoreCase(key)) {
String[] vals = new String[2];
vals[0] = entry.getValue()==null ? "null" : entry.getValue().toString();
vals[1] = new StringBuilder(vals[0]).reverse().toString();
n.put(key, Arrays.asList(vals));
} else {
n.put(key, entry.getValue());
}
}
return n;
}
private void modifyIdColumn(Map<String, Object> row, int num) {
Object o = row.remove("ID");
if(o==null) {
o = row.remove("id");
}
if(o!=null) {
String id = o.toString();
id = "TripleThreat-" + num + "-" + id;
row.put("id", id);
}
}
}