mirror of https://github.com/apache/lucene.git
SOLR-3856: Improve tests for SqlEntityProcessor
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1406231 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
b6db7c76f1
commit
5461ef1ac1
|
@ -1,41 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<!--
|
||||
Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
(the "License"); you may not use this file except in compliance with
|
||||
the License. You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
-->
|
||||
|
||||
<config>
|
||||
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
|
||||
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
|
||||
<updateHandler class="solr.DirectUpdateHandler2">
|
||||
<maxPendingDeletes>100000</maxPendingDeletes>
|
||||
</updateHandler>
|
||||
<requestDispatcher handleSelect="true" >
|
||||
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
|
||||
<httpCaching never304="true" />
|
||||
</requestDispatcher>
|
||||
<requestHandler name="standard" class="solr.StandardRequestHandler" default="true">
|
||||
<lst name="defaults">
|
||||
<str name="echoParams">explicit</str>
|
||||
</lst>
|
||||
</requestHandler>
|
||||
<requestHandler name="/dataimport-end-to-end" class="org.apache.solr.handler.dataimport.DataImportHandler" />
|
||||
<requestHandler name="/search" class="org.apache.solr.handler.component.SearchHandler">
|
||||
<lst name="defaults">
|
||||
<str name="echoParams">explicit</str>
|
||||
</lst>
|
||||
</requestHandler>
|
||||
</config>
|
||||
|
|
@ -19,19 +19,42 @@ package org.apache.solr.handler.dataimport;
|
|||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import junit.framework.Assert;
|
||||
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
/**
|
||||
* This sets up an in-memory Derby Sql database with a little sample data.
|
||||
* The schema here is poorly-designed to illustrate DIH's ability to
|
||||
* overcome these kinds of challenges.
|
||||
*/
|
||||
public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerTestCase {
|
||||
//Start with "true" so the first test run will populate the data.
|
||||
protected boolean underlyingDataModified = true;
|
||||
|
||||
protected boolean useSimpleCaches;
|
||||
protected boolean countryEntity;
|
||||
protected boolean countryCached;
|
||||
protected boolean sportsEntity;
|
||||
protected boolean sportsCached;
|
||||
protected String rootTransformerName;
|
||||
protected boolean countryTransformer;
|
||||
protected boolean sportsTransformer;
|
||||
|
||||
private static final Pattern totalRequestsPattern = Pattern
|
||||
.compile(".str name..Total Requests made to DataSource..(\\d+)..str.");
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClassDihJdbcTest() throws Exception {
|
||||
try {
|
||||
|
@ -39,52 +62,26 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
|
|||
} catch (Exception e) {
|
||||
throw e;
|
||||
}
|
||||
|
||||
Connection conn = null;
|
||||
Statement s = null;
|
||||
PreparedStatement ps = null;
|
||||
try {
|
||||
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
|
||||
s = conn.createStatement();
|
||||
s.executeUpdate("create table countries(code char(2) not null primary key, country_name varchar(50))");
|
||||
s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_codes varchar(100))");
|
||||
s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50))");
|
||||
|
||||
ps = conn.prepareStatement("insert into countries values (?,?)");
|
||||
for(String[] country : countries) {
|
||||
ps.setString(1, country[0]);
|
||||
ps.setString(2, country[1]);
|
||||
Assert.assertEquals(1, ps.executeUpdate());
|
||||
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
|
||||
}
|
||||
ps.close();
|
||||
@Before
|
||||
public void beforeDihJdbcTest() throws Exception {
|
||||
useSimpleCaches = false;
|
||||
countryEntity = false;
|
||||
countryCached = false;
|
||||
sportsEntity = false;
|
||||
sportsCached = false;
|
||||
rootTransformerName = null;
|
||||
countryTransformer = false;
|
||||
sportsTransformer = false;
|
||||
|
||||
ps = conn.prepareStatement("insert into people values (?,?,?)");
|
||||
for(Object[] person : people) {
|
||||
ps.setInt(1, (Integer) person[0]);
|
||||
ps.setString(2, (String) person[1]);
|
||||
ps.setString(3, (String) person[2]);
|
||||
Assert.assertEquals(1, ps.executeUpdate());
|
||||
}
|
||||
ps.close();
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
|
||||
ps = conn.prepareStatement("insert into people_sports values (?,?,?)");
|
||||
for(Object[] sport : people_sports) {
|
||||
ps.setInt(1, (Integer) sport[0]);
|
||||
ps.setInt(2, (Integer) sport[1]);
|
||||
ps.setString(3, (String) sport[2]);
|
||||
Assert.assertEquals(1, ps.executeUpdate());
|
||||
}
|
||||
ps.close();
|
||||
conn.close();
|
||||
} catch(Exception e) {
|
||||
throw e;
|
||||
} finally {
|
||||
if(s!=null) { s.close(); }
|
||||
if(ps!=null) { ps.close(); }
|
||||
if(conn!=null) { conn.close(); }
|
||||
if(underlyingDataModified) {
|
||||
refreshDatabase();
|
||||
}
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClassDihJdbcTest() throws Exception {
|
||||
Connection conn = null;
|
||||
|
@ -96,10 +93,527 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
|
|||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
if(s!=null) { s.close(); }
|
||||
if(conn!=null) { conn.close(); }
|
||||
try { s.close(); } catch(Exception ex) { }
|
||||
try { conn.close(); } catch(Exception ex) { }
|
||||
}
|
||||
}
|
||||
protected void singleEntity(int numToExpect) throws Exception {
|
||||
h.query("/dataimport", generateRequest());
|
||||
assertQ(req("*:*"), "//*[@numFound='" + totalPeople() + "']");
|
||||
Assert.assertTrue(totalDatabaseRequests() == numToExpect);
|
||||
}
|
||||
protected void simpleTransform(int numToExpect) throws Exception {
|
||||
rootTransformerName = "AddAColumnTransformer";
|
||||
h.query("/dataimport", generateRequest());
|
||||
assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + totalPeople() + "']");
|
||||
Assert.assertTrue(totalDatabaseRequests() == numToExpect);
|
||||
}
|
||||
/**
|
||||
* A delta update will not clean up documents added by a transformer
|
||||
* even if the parent document that the transformer used to base the new documents
|
||||
* were deleted
|
||||
*/
|
||||
protected void complexTransform(int numToExpect, int numDeleted) throws Exception {
|
||||
rootTransformerName = "TripleThreatTransformer";
|
||||
h.query("/dataimport", generateRequest());
|
||||
assertQ(req("q", "*:*", "rows", "" + (totalPeople() * 3), "sort", "id asc"), "//*[@numFound='" + ((totalPeople() * 3) + (numDeleted * 2)) + "']");
|
||||
assertQ(req("id:TripleThreat-1-*"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
|
||||
assertQ(req("id:TripleThreat-2-*"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
|
||||
if(personNameExists("Michael") && countryCodeExists("NR") && countryCodeExists("RN"))
|
||||
{
|
||||
assertQ(
|
||||
req("+id:TripleThreat-1-3 +NAME_mult_s:Michael +NAME_mult_s:leahciM +COUNTRY_CODES_mult_s:NR +COUNTRY_CODES_mult_s:RN"),
|
||||
"//*[@numFound='1']");
|
||||
}
|
||||
assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + (totalPeople() + numDeleted) + "']");
|
||||
Assert.assertTrue(totalDatabaseRequests() == numToExpect);
|
||||
}
|
||||
protected void withChildEntities(boolean cached, boolean checkDatabaseRequests) throws Exception {
|
||||
rootTransformerName = random().nextBoolean() ? null : "AddAColumnTransformer";
|
||||
int numChildren = random().nextInt(1) + 1;
|
||||
int numDatabaseRequests = 1;
|
||||
if(underlyingDataModified) {
|
||||
if (countryEntity) {
|
||||
if (cached) {
|
||||
numDatabaseRequests++;
|
||||
} else {
|
||||
numDatabaseRequests += totalPeople();
|
||||
}
|
||||
}
|
||||
if (sportsEntity) {
|
||||
if (cached) {
|
||||
numDatabaseRequests++;
|
||||
} else {
|
||||
numDatabaseRequests += totalPeople();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
countryEntity = true;
|
||||
sportsEntity = true;
|
||||
if(numChildren==1) {
|
||||
countryEntity = random().nextBoolean();
|
||||
sportsEntity = !countryEntity;
|
||||
}
|
||||
if(countryEntity) {
|
||||
countryTransformer = random().nextBoolean();
|
||||
if(cached) {
|
||||
numDatabaseRequests++;
|
||||
countryCached = true;
|
||||
} else {
|
||||
numDatabaseRequests += totalPeople();
|
||||
}
|
||||
}
|
||||
if(sportsEntity) {
|
||||
sportsTransformer = random().nextBoolean();
|
||||
if(cached) {
|
||||
numDatabaseRequests++;
|
||||
sportsCached = true;
|
||||
} else {
|
||||
numDatabaseRequests += totalPeople();
|
||||
}
|
||||
}
|
||||
}
|
||||
h.query("/dataimport", generateRequest());
|
||||
|
||||
assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
|
||||
if(!underlyingDataModified && "AddAColumnTransformer".equals(rootTransformerName)) {
|
||||
assertQ(req("AddAColumn_s:Added"), "//*[@numFound='" + (totalPeople()) + "']");
|
||||
}
|
||||
if(countryEntity) {
|
||||
if(personNameExists("Jayden"))
|
||||
{
|
||||
String nrName = countryNameByCode("NP");
|
||||
if(nrName!=null && nrName.length()>0) {
|
||||
assertQ(req("NAME_mult_s:Jayden"),
|
||||
"//*[@numFound='1']", "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
|
||||
}
|
||||
}
|
||||
String nrName = countryNameByCode("NR");
|
||||
int num = numberPeopleByCountryCode("NR");
|
||||
if(nrName!=null && num>0) {
|
||||
assertQ(req("COUNTRY_CODES_mult_s:NR"),
|
||||
"//*[@numFound='" + num + "']", "//doc/str[@name='COUNTRY_NAME_s']='" + nrName + "'");
|
||||
}
|
||||
if(countryTransformer && !underlyingDataModified) {
|
||||
assertQ(req("countryAdded_s:country_added"), "//*[@numFound='" + totalPeople() + "']");
|
||||
}
|
||||
}
|
||||
if(sportsEntity) {
|
||||
if(!underlyingDataModified) {
|
||||
assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
|
||||
}
|
||||
String michaelsName = personNameById(3);
|
||||
String[] michaelsSports = sportNamesByPersonId(3);
|
||||
if(michaelsName != null && michaelsSports.length>0) {
|
||||
String[] xpath = new String[michaelsSports.length + 1];
|
||||
xpath[0] = "//*[@numFound='1']";
|
||||
int i=1;
|
||||
for(String ms : michaelsSports) {
|
||||
xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms + "'";
|
||||
i++;
|
||||
}
|
||||
assertQ(req("NAME_mult_s:" + michaelsName.replaceAll("\\W", "\\\\$0")), xpath);
|
||||
}
|
||||
if (!underlyingDataModified && sportsTransformer) {
|
||||
assertQ(req("sportsAdded_s:sport_added"), "//*[@numFound='" + (totalPeople()) + "']");
|
||||
}
|
||||
}
|
||||
if(checkDatabaseRequests) {
|
||||
Assert.assertTrue("Expecting " + numDatabaseRequests
|
||||
+ " database calls, but DIH reported " + totalDatabaseRequests(),
|
||||
totalDatabaseRequests() == numDatabaseRequests);
|
||||
}
|
||||
}
|
||||
protected void simpleCacheChildEntities(boolean checkDatabaseRequests) throws Exception {
|
||||
useSimpleCaches = true;
|
||||
countryEntity = true;
|
||||
sportsEntity = true;
|
||||
countryCached=true;
|
||||
sportsCached=true;
|
||||
int dbRequestsMoreThan = 3;
|
||||
int dbRequestsLessThan = totalPeople() * 2 + 1;
|
||||
h.query("/dataimport", generateRequest());
|
||||
assertQ(req("*:*"), "//*[@numFound='" + (totalPeople()) + "']");
|
||||
if(!underlyingDataModified || (personNameExists("Samantha") && "Nauru".equals(countryNameByCode("NR"))))
|
||||
{
|
||||
assertQ(req("NAME_mult_s:Samantha"),
|
||||
"//*[@numFound='1']", "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
|
||||
}
|
||||
if(!underlyingDataModified)
|
||||
{
|
||||
assertQ(req("COUNTRY_CODES_mult_s:NR"),
|
||||
"//*[@numFound='2']", "//doc/str[@name='COUNTRY_NAME_s']='Nauru'");
|
||||
assertQ(req("SPORT_NAME_mult_s:Sailing"), "//*[@numFound='2']");
|
||||
}
|
||||
String[] michaelsSports = sportNamesByPersonId(3);
|
||||
if(!underlyingDataModified || michaelsSports.length>0) {
|
||||
String[] xpath = new String[michaelsSports.length + 1];
|
||||
xpath[0] = "//*[@numFound='1']";
|
||||
int i=1;
|
||||
for(String ms : michaelsSports) {
|
||||
xpath[i] = "//doc/arr[@name='SPORT_NAME_mult_s']/str[" + i + "]='" + ms + "'";
|
||||
i++;
|
||||
}
|
||||
assertQ(req("NAME_mult_s:Michael"), xpath);
|
||||
}
|
||||
if(checkDatabaseRequests) {
|
||||
Assert.assertTrue("Expecting more than " + dbRequestsMoreThan
|
||||
+ " database calls, but DIH reported " + totalDatabaseRequests(),
|
||||
totalDatabaseRequests() > dbRequestsMoreThan);
|
||||
Assert.assertTrue("Expecting fewer than " + dbRequestsLessThan
|
||||
+ " database calls, but DIH reported " + totalDatabaseRequests(),
|
||||
totalDatabaseRequests() < dbRequestsLessThan);
|
||||
}
|
||||
}
|
||||
public void refreshDatabase() throws Exception
|
||||
{
|
||||
underlyingDataModified = false;
|
||||
Connection conn = null;
|
||||
Statement s = null;
|
||||
PreparedStatement ps = null;
|
||||
Timestamp theTime = new Timestamp(System.currentTimeMillis() - 10000); //10 seconds ago
|
||||
try {
|
||||
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
|
||||
s = conn.createStatement();
|
||||
try {
|
||||
s.executeUpdate("drop table countries");
|
||||
s.executeUpdate("drop table people");
|
||||
s.executeUpdate("drop table people_sports");
|
||||
} catch(Exception e) {
|
||||
//ignore. dbs complain when tables do no exist.
|
||||
}
|
||||
s.executeUpdate("create table countries(code varchar(3) not null primary key, country_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
|
||||
s.executeUpdate("create table people(id int not null primary key, name varchar(50), country_code char(2), deleted char(1) default 'N', last_modified timestamp not null)");
|
||||
s.executeUpdate("create table people_sports(id int not null primary key, person_id int, sport_name varchar(50), deleted char(1) default 'N', last_modified timestamp not null)");
|
||||
|
||||
ps = conn.prepareStatement("insert into countries (code, country_name, last_modified) values (?,?,?)");
|
||||
for(String[] country : countries) {
|
||||
ps.setString(1, country[0]);
|
||||
ps.setString(2, country[1]);
|
||||
ps.setTimestamp(3, theTime);
|
||||
Assert.assertEquals(1, ps.executeUpdate());
|
||||
}
|
||||
ps.close();
|
||||
|
||||
ps = conn.prepareStatement("insert into people (id, name, country_code, last_modified) values (?,?,?,?)");
|
||||
for(Object[] person : people) {
|
||||
ps.setInt(1, (Integer) person[0]);
|
||||
ps.setString(2, (String) person[1]);
|
||||
ps.setString(3, (String) person[2]);
|
||||
ps.setTimestamp(4, theTime);
|
||||
Assert.assertEquals(1, ps.executeUpdate());
|
||||
}
|
||||
ps.close();
|
||||
|
||||
ps = conn.prepareStatement("insert into people_sports (id, person_id, sport_name, last_modified) values (?,?,?,?)");
|
||||
for(Object[] sport : people_sports) {
|
||||
ps.setInt(1, (Integer) sport[0]);
|
||||
ps.setInt(2, (Integer) sport[1]);
|
||||
ps.setString(3, (String) sport[2]);
|
||||
ps.setTimestamp(4, theTime);
|
||||
Assert.assertEquals(1, ps.executeUpdate());
|
||||
}
|
||||
ps.close();
|
||||
conn.commit();
|
||||
conn.close();
|
||||
} catch(Exception e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try { ps.close(); } catch(Exception ex) { }
|
||||
try { s.close(); } catch(Exception ex) { }
|
||||
try { conn.close(); } catch(Exception ex) { }
|
||||
}
|
||||
}
|
||||
protected abstract String deltaQueriesCountryTable() ;
|
||||
protected abstract String deltaQueriesPersonTable() ;
|
||||
|
||||
|
||||
private int getIntFromQuery(String query) throws Exception {
|
||||
Connection conn = null;
|
||||
Statement s = null;
|
||||
ResultSet rs = null;
|
||||
try {
|
||||
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
|
||||
s = conn.createStatement();
|
||||
rs = s.executeQuery(query);
|
||||
if(rs.next()) {
|
||||
return rs.getInt(1);
|
||||
}
|
||||
return 0;
|
||||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try { rs.close(); } catch(Exception ex) { }
|
||||
try { s.close(); } catch(Exception ex) { }
|
||||
try { conn.close(); } catch(Exception ex) { }
|
||||
}
|
||||
}
|
||||
private String[] getStringsFromQuery(String query) throws Exception {
|
||||
Connection conn = null;
|
||||
Statement s = null;
|
||||
ResultSet rs = null;
|
||||
try {
|
||||
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
|
||||
s = conn.createStatement();
|
||||
rs = s.executeQuery(query);
|
||||
List<String> results = new ArrayList<String>();
|
||||
while(rs.next()) {
|
||||
results.add(rs.getString(1));
|
||||
}
|
||||
return results.toArray(new String[results.size()]);
|
||||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try { rs.close(); } catch(Exception ex) { }
|
||||
try { s.close(); } catch(Exception ex) { }
|
||||
try { conn.close(); } catch(Exception ex) { }
|
||||
}
|
||||
}
|
||||
|
||||
public int totalCountries() throws Exception {
|
||||
return getIntFromQuery("SELECT COUNT(1) FROM COUNTRIES WHERE DELETED != 'Y' ");
|
||||
}
|
||||
public int totalPeople() throws Exception {
|
||||
return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' ");
|
||||
}
|
||||
public boolean countryCodeExists(String cc) throws Exception {
|
||||
return getIntFromQuery("SELECT COUNT(1) country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='" + cc + "'")>0;
|
||||
}
|
||||
public String countryNameByCode(String cc) throws Exception {
|
||||
String[] s = getStringsFromQuery("SELECT country_name FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='" + cc + "'");
|
||||
return s.length==0 ? null : s[0];
|
||||
}
|
||||
public int numberPeopleByCountryCode(String cc) throws Exception {
|
||||
return getIntFromQuery(
|
||||
"Select count(1) " +
|
||||
"from people p " +
|
||||
"inner join countries c on p.country_code=c.code " +
|
||||
"where p.deleted!='Y' and c.deleted!='Y' and c.code='" + cc + "'");
|
||||
}
|
||||
public String[] sportNamesByPersonId(int personId) throws Exception {
|
||||
return getStringsFromQuery(
|
||||
"SELECT ps.SPORT_NAME " +
|
||||
"FROM people_sports ps " +
|
||||
"INNER JOIN PEOPLE p ON p.id = ps.person_Id " +
|
||||
"WHERE ps.DELETED != 'Y' AND p.DELETED != 'Y' " +
|
||||
"AND ps.person_id=" + personId + " " +
|
||||
"ORDER BY ps.id"
|
||||
);
|
||||
}
|
||||
public boolean personNameExists(String pn) throws Exception {
|
||||
return getIntFromQuery("SELECT COUNT(1) FROM PEOPLE WHERE DELETED != 'Y' AND NAME='" + pn + "'")>0;
|
||||
}
|
||||
public String personNameById(int id) throws Exception {
|
||||
String[] nameArr= getStringsFromQuery("SELECT NAME FROM PEOPLE WHERE ID=" + id);
|
||||
if(nameArr.length==0) {
|
||||
return null;
|
||||
}
|
||||
return nameArr[0];
|
||||
}
|
||||
|
||||
public int totalDatabaseRequests(String dihHandlerName) throws Exception {
|
||||
LocalSolrQueryRequest request = lrf.makeRequest("indent", "true");
|
||||
String response = h.query(dihHandlerName, request);
|
||||
Matcher m = totalRequestsPattern.matcher(response);
|
||||
Assert.assertTrue("The handler " + dihHandlerName
|
||||
+ " is not reporting any database requests. ", m.find()
|
||||
&& m.groupCount() == 1);
|
||||
return Integer.parseInt(m.group(1));
|
||||
}
|
||||
|
||||
public int totalDatabaseRequests() throws Exception {
|
||||
return totalDatabaseRequests("/dataimport");
|
||||
}
|
||||
public IntChanges modifySomePeople() throws Exception {
|
||||
underlyingDataModified = true;
|
||||
int numberToChange = random().nextInt(people.length + 1);
|
||||
Set<Integer> changeSet = new HashSet<Integer>();
|
||||
Set<Integer> deleteSet = new HashSet<Integer>();
|
||||
Set<Integer> addSet = new HashSet<Integer>();
|
||||
Connection conn = null;
|
||||
PreparedStatement change = null;
|
||||
PreparedStatement delete = null;
|
||||
PreparedStatement add = null;
|
||||
//One second in the future ensures a change time after the last import (DIH uses second precision only)
|
||||
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
|
||||
try {
|
||||
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
|
||||
change = conn.prepareStatement("update people set name=?, last_modified=? where id=?");
|
||||
delete = conn.prepareStatement("update people set deleted='Y', last_modified=? where id=?");
|
||||
add = conn.prepareStatement("insert into people (id,name,country_code,last_modified) values (?,?,'NEW',?)");
|
||||
for(int i=0 ; i<numberToChange ; i++) {
|
||||
int tryIndex = random().nextInt(people.length);
|
||||
Integer id = (Integer) people[tryIndex][0];
|
||||
if(!changeSet.contains(id) && !deleteSet.contains(id)) {
|
||||
boolean changeDontDelete = random().nextBoolean();
|
||||
if(changeDontDelete) {
|
||||
changeSet.add(id);
|
||||
change.setString(1, "MODIFIED " + people[tryIndex][1]);
|
||||
change.setTimestamp(2, theTime);
|
||||
change.setInt(3, id);
|
||||
Assert.assertEquals(1, change.executeUpdate());
|
||||
} else {
|
||||
deleteSet.add(id);
|
||||
delete.setTimestamp(1, theTime);
|
||||
delete.setInt(2, id);
|
||||
Assert.assertEquals(1, delete.executeUpdate());
|
||||
}
|
||||
}
|
||||
}
|
||||
int numberToAdd = random().nextInt(3);
|
||||
for(int i=0 ; i<numberToAdd ; i++) {
|
||||
int tryIndex = random().nextInt(people.length);
|
||||
Integer id = (Integer) people[tryIndex][0];
|
||||
Integer newId = id+1000;
|
||||
String newDesc = "ADDED " + people[tryIndex][1];
|
||||
if(!addSet.contains(newId)) {
|
||||
addSet.add(newId);
|
||||
add.setInt(1, newId);
|
||||
add.setString(2, newDesc);
|
||||
add.setTimestamp(3, theTime);
|
||||
Assert.assertEquals(1, add.executeUpdate());
|
||||
}
|
||||
}
|
||||
conn.commit();
|
||||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try { change.close(); } catch(Exception ex) { }
|
||||
try { conn.close(); } catch(Exception ex) { }
|
||||
}
|
||||
IntChanges c = new IntChanges();
|
||||
c.changedKeys=changeSet.toArray(new Integer[changeSet.size()]);
|
||||
c.deletedKeys=deleteSet.toArray(new Integer[deleteSet.size()]);
|
||||
c.addedKeys=addSet.toArray(new Integer[addSet.size()]);
|
||||
return c;
|
||||
}
|
||||
|
||||
public String[] modifySomeCountries() throws Exception {
|
||||
underlyingDataModified = true;
|
||||
int numberToChange = random().nextInt(countries.length + 1);
|
||||
Set<String> changeSet = new HashSet<String>();
|
||||
Connection conn = null;
|
||||
PreparedStatement change = null;
|
||||
// One second in the future ensures a change time after the last import (DIH
|
||||
// uses second precision only)
|
||||
Timestamp theTime = new Timestamp(System.currentTimeMillis() + 1000);
|
||||
try {
|
||||
conn = DriverManager.getConnection("jdbc:hsqldb:mem:.");
|
||||
change = conn
|
||||
.prepareStatement("update countries set country_name=?, last_modified=? where code=?");
|
||||
for (int i = 0; i < numberToChange; i++) {
|
||||
int tryIndex = random().nextInt(countries.length);
|
||||
String code = countries[tryIndex][0];
|
||||
if (!changeSet.contains(code)) {
|
||||
changeSet.add(code);
|
||||
change.setString(1, "MODIFIED " + countries[tryIndex][1]);
|
||||
change.setTimestamp(2, theTime);
|
||||
change.setString(3, code);
|
||||
Assert.assertEquals(1, change.executeUpdate());
|
||||
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
throw e;
|
||||
} finally {
|
||||
try {
|
||||
change.close();
|
||||
} catch (Exception ex) {}
|
||||
try {
|
||||
conn.close();
|
||||
} catch (Exception ex) {}
|
||||
}
|
||||
return changeSet.toArray(new String[changeSet.size()]);
|
||||
}
|
||||
class IntChanges {
|
||||
public Integer[] changedKeys;
|
||||
public Integer[] deletedKeys;
|
||||
public Integer[] addedKeys;
|
||||
}
|
||||
protected LocalSolrQueryRequest generateRequest() {
|
||||
return lrf.makeRequest("command", "full-import", "dataConfig", generateConfig(),
|
||||
"clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
|
||||
}
|
||||
protected String generateConfig() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append("<dataConfig> \n");
|
||||
sb.append("<dataSource name=''hsqldb'' driver=''org.hsqldb.jdbcDriver'' url=''jdbc:hsqldb:mem:.'' /> \n");
|
||||
sb.append("<document name=''TestSqlEntityProcessor''> \n");
|
||||
sb.append("<entity name=''People'' ");
|
||||
sb.append("pk=''" + (random().nextBoolean() ? "ID" : "People.ID") + "'' ");
|
||||
sb.append("processor=''SqlEntityProcessor'' dataSource=''hsqldb'' ");
|
||||
sb.append(rootTransformerName !=null ? "transformer=''" + rootTransformerName + "'' " : "");
|
||||
sb.append("query=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE WHERE DELETED != 'Y' '' ");
|
||||
sb.append(deltaQueriesPersonTable());
|
||||
sb.append("> \n");
|
||||
|
||||
sb.append("<field column=''NAME'' name=''NAME_mult_s'' /> \n");
|
||||
sb.append("<field column=''COUNTRY_CODE'' name=''COUNTRY_CODES_mult_s'' /> \n");
|
||||
|
||||
if(countryEntity) {
|
||||
sb.append("<entity name=''Countries'' ");
|
||||
sb.append("pk=''" + (random().nextBoolean() ? "CODE" : "Countries.CODE") + "'' ");
|
||||
sb.append("dataSource=''hsqldb'' ");
|
||||
sb.append(countryTransformer ? "transformer=''AddAColumnTransformer'' " +
|
||||
"newColumnName=''countryAdded_s'' newColumnValue=''country_added'' " : "");
|
||||
if(countryCached) {
|
||||
sb.append(random().nextBoolean() ?
|
||||
"processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' " :
|
||||
"processor=''CachedSqlEntityProcessor'' "
|
||||
);
|
||||
if(useSimpleCaches) {
|
||||
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' ''>\n");
|
||||
} else {
|
||||
sb.append(random().nextBoolean() ?
|
||||
"cacheKey=''CODE'' cacheLookup=''People.COUNTRY_CODE'' " :
|
||||
"where=''CODE=People.COUNTRY_CODE'' "
|
||||
);
|
||||
sb.append("query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES'' ");
|
||||
sb.append("> \n");
|
||||
}
|
||||
} else {
|
||||
sb.append("processor=''SqlEntityProcessor'' query=''SELECT CODE, COUNTRY_NAME FROM COUNTRIES WHERE DELETED != 'Y' AND CODE='${People.COUNTRY_CODE}' '' ");
|
||||
sb.append(deltaQueriesCountryTable());
|
||||
sb.append("> \n");
|
||||
}
|
||||
sb.append("<field column=''CODE'' name=''COUNTRY_CODE_s'' /> \n");
|
||||
sb.append("<field column=''COUNTRY_NAME'' name=''COUNTRY_NAME_s'' /> \n");
|
||||
sb.append("</entity> \n");
|
||||
}
|
||||
if(sportsEntity) {
|
||||
sb.append("<entity name=''Sports'' dataSource=''hsqldb'' "
|
||||
+ (sportsTransformer ? "transformer=''AddAColumnTransformer'' " +
|
||||
"newColumnName=''sportsAdded_s'' newColumnValue=''sport_added'' " : ""));
|
||||
if(sportsCached) {
|
||||
sb.append(random().nextBoolean() ?
|
||||
"processor=''SqlEntityProcessor'' cacheImpl=''SortedMapBackedCache'' " :
|
||||
"processor=''CachedSqlEntityProcessor'' "
|
||||
);
|
||||
if(useSimpleCaches) {
|
||||
sb.append("query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID='${People.ID}' ORDER BY ID'' ");
|
||||
} else {
|
||||
sb.append(random().nextBoolean() ?
|
||||
"cacheKey=''PERSON_ID'' cacheLookup=''People.ID'' " :
|
||||
"where=''PERSON_ID=People.ID'' "
|
||||
);
|
||||
sb.append("query=''SELECT ID, PERSON_ID, SPORT_NAME FROM PEOPLE_SPORTS ORDER BY ID'' ");
|
||||
}
|
||||
} else {
|
||||
sb.append("processor=''SqlEntityProcessor'' query=''SELECT ID, SPORT_NAME FROM PEOPLE_SPORTS WHERE DELETED != 'Y' AND PERSON_ID='${People.ID}' ORDER BY ID'' ");
|
||||
}
|
||||
sb.append("> \n");
|
||||
sb.append("<field column=''SPORT_NAME'' name=''SPORT_NAME_mult_s'' /> \n");
|
||||
sb.append("<field column=''id'' name=''SPORT_ID_mult_s'' /> \n");
|
||||
sb.append("</entity> \n");
|
||||
}
|
||||
|
||||
sb.append("</entity> \n");
|
||||
sb.append("</document> \n");
|
||||
sb.append("</dataConfig> \n");
|
||||
String config = sb.toString().replaceAll("[']{2}", "\"");
|
||||
log.debug(config);
|
||||
return config;
|
||||
}
|
||||
|
||||
public static final String[][] countries = {
|
||||
{"NA", "Namibia"},
|
||||
|
@ -118,7 +632,7 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
|
|||
|
||||
public static final Object[][] people = {
|
||||
{1,"Jacob","NZ"},
|
||||
{2,"Ethan","NU,NA,NE"},
|
||||
{2,"Ethan","NU"},
|
||||
{3,"Michael","NR"},
|
||||
{4,"Jayden","NP"},
|
||||
{5,"William","NO"},
|
||||
|
@ -134,7 +648,7 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
|
|||
{15,"Lily","NE"},
|
||||
{16,"Madison","NC"},
|
||||
{17,"Mia","NA"},
|
||||
{18,"Natalie","NP,NR,NU,NZ"},
|
||||
{18,"Natalie","NZ"},
|
||||
{19,"Olivia","NU"},
|
||||
{20,"Samantha","NR"}
|
||||
};
|
||||
|
@ -166,5 +680,4 @@ public abstract class AbstractDIHJdbcTestCase extends AbstractDataImportHandlerT
|
|||
{1900, 19, "Water skiing"},
|
||||
{2000, 20, "Windsurfing"}
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
public class AddAColumnTransformer extends Transformer {
|
||||
@Override
|
||||
public Object transformRow(Map<String,Object> aRow, Context context) {
|
||||
String colName = context.getEntityAttribute("newColumnName");
|
||||
colName = colName==null ? "AddAColumn_s" : colName;
|
||||
String colValue = context.getEntityAttribute("newColumnValue");
|
||||
colValue = colValue==null ? "Added" : colValue;
|
||||
aRow.put(colName, colValue);
|
||||
return aRow;
|
||||
}
|
||||
}
|
|
@ -1,260 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Test for CachedSqlEntityProcessor
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
* @since solr 1.3
|
||||
*/
|
||||
public class TestCachedSqlEntityProcessor extends AbstractDataImportHandlerTestCase {
|
||||
|
||||
@Test
|
||||
public void withoutWhereClause() {
|
||||
List fields = new ArrayList();
|
||||
fields.add(createMap("column", "id"));
|
||||
fields.add(createMap("column", "desc"));
|
||||
String q = "select * from x where id=${x.id}";
|
||||
Map<String, String> entityAttrs = createMap("query", q);
|
||||
MockDataSource ds = new MockDataSource();
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
|
||||
vr.addNamespace("x", createMap("id", 1));
|
||||
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
|
||||
rows.add(createMap("id", 1, "desc", "one"));
|
||||
rows.add(createMap("id", 1, "desc", "another one"));
|
||||
MockDataSource.setIterator(vr.replaceTokens(q), rows.iterator());
|
||||
EntityProcessor csep = new EntityProcessorWrapper(new CachedSqlEntityProcessor(), null, null);
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(2, rows.size());
|
||||
ds.close();
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(2, rows.size());
|
||||
assertEquals(2, rows.get(0).size());
|
||||
assertEquals(2, rows.get(1).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void withoutWhereClauseWithTransformers() {
|
||||
List fields = new ArrayList();
|
||||
fields.add(createMap("column", "id"));
|
||||
fields.add(createMap("column", "desc"));
|
||||
String q = "select * from x where id=${x.id}";
|
||||
Map<String, String> entityAttrs = createMap(
|
||||
"query", q, "transformer", UppercaseTransformer.class.getName());
|
||||
MockDataSource ds = new MockDataSource();
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
|
||||
vr.addNamespace("x", createMap("id", 1));
|
||||
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
|
||||
rows.add(createMap("id", 1, "desc", "one"));
|
||||
rows.add(createMap("id", 1, "desc", "another one"));
|
||||
MockDataSource.setIterator(vr.replaceTokens(q), rows.iterator());
|
||||
EntityProcessor csep = new EntityProcessorWrapper( new CachedSqlEntityProcessor(), null, null);
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(2, rows.size());
|
||||
ds.close();
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
assertEquals(r.get("desc").toString().toUpperCase(Locale.ROOT), r.get("desc"));
|
||||
}
|
||||
assertEquals(2, rows.size());
|
||||
assertEquals(2, rows.get(0).size());
|
||||
assertEquals(2, rows.get(1).size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void withoutWhereClauseWithMultiRowTransformer() {
|
||||
List fields = new ArrayList();
|
||||
fields.add(createMap("column", "id"));
|
||||
fields.add(createMap("column", "desc"));
|
||||
String q = "select * from x where id=${x.id}";
|
||||
Map<String, String> entityAttrs = createMap(
|
||||
"query", q, "transformer", DoubleTransformer.class.getName());
|
||||
MockDataSource ds = new MockDataSource();
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
|
||||
vr.addNamespace("x", createMap("id", 1));
|
||||
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
|
||||
rows.add(createMap("id", 1, "desc", "one"));
|
||||
rows.add(createMap("id", 1, "desc", "another one"));
|
||||
MockDataSource.setIterator(vr.replaceTokens(q), rows.iterator());
|
||||
EntityProcessor csep = new EntityProcessorWrapper( new CachedSqlEntityProcessor(), null, null);
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(4, rows.size());
|
||||
ds.close();
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(4, rows.size());
|
||||
assertEquals(2, rows.get(0).size());
|
||||
assertEquals(2, rows.get(1).size());
|
||||
}
|
||||
|
||||
public static class DoubleTransformer extends Transformer {
|
||||
|
||||
@Override
|
||||
public Object transformRow(Map<String, Object> row, Context context) {
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
|
||||
rows.add(row);
|
||||
rows.add(row);
|
||||
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
|
||||
public static class UppercaseTransformer extends Transformer {
|
||||
|
||||
@Override
|
||||
public Object transformRow(Map<String, Object> row, Context context) {
|
||||
for (Map.Entry<String, Object> entry : row.entrySet()) {
|
||||
Object val = entry.getValue();
|
||||
if (val instanceof String) {
|
||||
String s = (String) val;
|
||||
entry.setValue(s.toUpperCase(Locale.ROOT));
|
||||
}
|
||||
}
|
||||
return row;
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void withKeyAndLookup() {
|
||||
List fields = new ArrayList();
|
||||
fields.add(createMap("column", "id"));
|
||||
fields.add(createMap("column", "desc"));
|
||||
String q = "select * from x";
|
||||
Map<String, String> entityAttrs = createMap(
|
||||
"query", q, DIHCacheSupport.CACHE_PRIMARY_KEY,"id", DIHCacheSupport.CACHE_FOREIGN_KEY ,"x.id");
|
||||
MockDataSource ds = new MockDataSource();
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
Map xNamespace = createMap("id", 0);
|
||||
vr.addNamespace("x", xNamespace);
|
||||
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
|
||||
doWhereTest(q, context, ds, xNamespace);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void withWhereClause() {
|
||||
List fields = new ArrayList();
|
||||
fields.add(createMap("column", "id"));
|
||||
fields.add(createMap("column", "desc"));
|
||||
String q = "select * from x";
|
||||
Map<String, String> entityAttrs = createMap("query", q, "where", "id=x.id");
|
||||
MockDataSource ds = new MockDataSource();
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
Map xNamespace = createMap("id", 0);
|
||||
vr.addNamespace("x", xNamespace);
|
||||
Context context = getContext(null, vr, ds, Context.FULL_DUMP, fields, entityAttrs);
|
||||
doWhereTest(q, context, ds, xNamespace);
|
||||
}
|
||||
|
||||
private void doWhereTest(String q, Context context, MockDataSource ds, Map xNamespace) {
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
|
||||
rows.add(createMap("desc", "one", "id", 1));
|
||||
rows.add(createMap("id", 2, "desc", "two"));
|
||||
rows.add(createMap("id", 2, "desc", "another two"));
|
||||
rows.add(createMap("id", 3, "desc", "three"));
|
||||
rows.add(createMap("id", 3, "desc", "another three"));
|
||||
rows.add(createMap("id", 3, "desc", "another another three"));
|
||||
MockDataSource.setIterator(q, rows.iterator());
|
||||
EntityProcessor csep = new EntityProcessorWrapper(new CachedSqlEntityProcessor(), null, null);
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(0, rows.size());
|
||||
ds.close();
|
||||
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
xNamespace.put("id", 2);
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(2, rows.size());
|
||||
|
||||
csep.init(context);
|
||||
rows = new ArrayList<Map<String, Object>>();
|
||||
xNamespace.put("id", 3);
|
||||
while (true) {
|
||||
Map<String, Object> r = csep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rows.add(r);
|
||||
}
|
||||
assertEquals(3, rows.size());
|
||||
}
|
||||
}
|
|
@ -1,59 +0,0 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with this
|
||||
* work for additional information regarding copyright ownership. The ASF
|
||||
* licenses this file to You under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations under
|
||||
* the License.
|
||||
*/
|
||||
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.junit.Assert;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestDIHEndToEnd extends AbstractDIHJdbcTestCase {
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("dataimport-solrconfig-end-to-end.xml", "dataimport-schema.xml");
|
||||
}
|
||||
@Test
|
||||
public void testEndToEnd() throws Exception {
|
||||
LocalSolrQueryRequest request = lrf.makeRequest(
|
||||
"command", "full-import", "config", "data-config-end-to-end.xml",
|
||||
"clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
|
||||
h.query("/dataimport-end-to-end", request);
|
||||
assertQ(req("*:*"), "//*[@numFound='20']");
|
||||
assertQ(req("COUNTRY_NAME:zealand"), "//*[@numFound='2']");
|
||||
assertQ(req("COUNTRY_NAME:niue"), "//*[@numFound='3']");
|
||||
|
||||
//It would be nice if there was a way to get it to run transformers before putting
|
||||
//data in the cache, then id=2 (person=Ethan, country=NU,NA,NE) could join...)
|
||||
//assertQ(req("COUNTRY_NAME:Netherlands"), "//*[@numFound='3']");
|
||||
|
||||
assertQ(req("NAME:michael"), "//*[@numFound='1']");
|
||||
assertQ(req("SPORT_NAME:kayaking"), "//*[@numFound='2']");
|
||||
assertQ(req("SPORT_NAME:fishing"), "//*[@numFound='1']");
|
||||
|
||||
request = lrf.makeRequest("indent", "true");
|
||||
String response = h.query("/dataimport-end-to-end", request);
|
||||
Matcher m = Pattern.compile(".str name..Total Requests made to DataSource..(\\d+)..str.").matcher(response);
|
||||
Assert.assertTrue(m.find() && m.groupCount()==1);
|
||||
int numRequests = Integer.parseInt(m.group(1));
|
||||
Assert.assertTrue(
|
||||
"The database should have been hit once each " +
|
||||
"for 'Person' & 'Country' and ~20 times for 'Sport'", numRequests<30);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,92 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class TestNonWritablePersistFile extends AbstractDataImportHandlerTestCase {
|
||||
private static final String FULLIMPORT_QUERY = "select * from x";
|
||||
|
||||
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
|
||||
|
||||
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
|
||||
|
||||
private static final String dataConfig_delta =
|
||||
"<dataConfig>" +
|
||||
" <dataSource type=\"MockDataSource\"/>\n" +
|
||||
" <document>\n" +
|
||||
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
|
||||
" query=\"" + FULLIMPORT_QUERY + "\"" +
|
||||
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
|
||||
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
|
||||
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
|
||||
" <field column=\"id\" name=\"id\"/>\n" +
|
||||
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
|
||||
" <field column=\"desc\" />\n" +
|
||||
" </entity>\n" +
|
||||
" </entity>\n" +
|
||||
" </document>\n" +
|
||||
"</dataConfig>\n";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testNonWritablePersistFile() throws Exception {
|
||||
// See SOLR-2551
|
||||
String configDir = h.getCore().getResourceLoader().getConfigDir();
|
||||
String filePath = configDir;
|
||||
if (configDir != null && !configDir.endsWith(File.separator))
|
||||
filePath += File.separator;
|
||||
filePath += "dataimport.properties";
|
||||
File f = new File(filePath);
|
||||
|
||||
try {
|
||||
// execute the test only if we are able to set file to read only mode
|
||||
assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
|
||||
assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly());
|
||||
assumeFalse("dataimport.proprties is still writable even though " +
|
||||
"marked readonly - test running as superuser?", f.canWrite());
|
||||
|
||||
ignoreException("Properties is not writable");
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
|
||||
|
||||
@SuppressWarnings("rawtypes")
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'",
|
||||
childRow.iterator());
|
||||
|
||||
runFullImport(dataConfig_delta);
|
||||
assertQ(req("id:1"), "//*[@numFound='0']");
|
||||
} finally {
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,3 +1,8 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
|
@ -14,168 +19,44 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.Ignore;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Test for SqlEntityProcessor
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
* @since solr 1.3
|
||||
* Test with various combinations of parameters, child entites, caches, transformers.
|
||||
*/
|
||||
@Ignore("FIXME: I fail so often it makes me ill!")
|
||||
public class TestSqlEntityProcessor extends AbstractDataImportHandlerTestCase {
|
||||
private static ThreadLocal<Integer> local = new ThreadLocal<Integer>();
|
||||
public class TestSqlEntityProcessor extends AbstractDIHJdbcTestCase {
|
||||
|
||||
@Test
|
||||
public void testSingleBatch() {
|
||||
SqlEntityProcessor sep = new SqlEntityProcessor();
|
||||
List<Map<String, Object>> rows = getRows(3);
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
HashMap<String, String> ea = new HashMap<String, String>();
|
||||
ea.put("query", "SELECT * FROM A");
|
||||
Context c = getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea);
|
||||
sep.init(c);
|
||||
int count = 0;
|
||||
while (true) {
|
||||
Map<String, Object> r = sep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
count++;
|
||||
public void testSingleEntity() throws Exception {
|
||||
singleEntity(1);
|
||||
}
|
||||
|
||||
assertEquals(3, count);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTranformer() {
|
||||
EntityProcessor sep = new EntityProcessorWrapper( new SqlEntityProcessor(), null, null);
|
||||
List<Map<String, Object>> rows = getRows(2);
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
HashMap<String, String> ea = new HashMap<String, String>();
|
||||
ea.put("query", "SELECT * FROM A");
|
||||
ea.put("transformer", T.class.getName());
|
||||
|
||||
sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
|
||||
List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
|
||||
Map<String, Object> r = null;
|
||||
while (true) {
|
||||
r = sep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rs.add(r);
|
||||
|
||||
public void testWithSimpleTransformer() throws Exception {
|
||||
simpleTransform(1);
|
||||
}
|
||||
assertEquals(2, rs.size());
|
||||
assertNotNull(rs.get(0).get("T"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTranformerWithReflection() {
|
||||
EntityProcessor sep = new EntityProcessorWrapper(new SqlEntityProcessor(), null, null);
|
||||
List<Map<String, Object>> rows = getRows(2);
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
HashMap<String, String> ea = new HashMap<String, String>();
|
||||
ea.put("query", "SELECT * FROM A");
|
||||
ea.put("transformer", T3.class.getName());
|
||||
|
||||
sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
|
||||
List<Map<String, Object>> rs = new ArrayList<Map<String, Object>>();
|
||||
Map<String, Object> r = null;
|
||||
while (true) {
|
||||
r = sep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
rs.add(r);
|
||||
|
||||
public void testWithComplexTransformer() throws Exception {
|
||||
complexTransform(1, 0);
|
||||
}
|
||||
assertEquals(2, rs.size());
|
||||
assertNotNull(rs.get(0).get("T3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTranformerList() {
|
||||
EntityProcessor sep = new EntityProcessorWrapper(new SqlEntityProcessor(),null, null);
|
||||
List<Map<String, Object>> rows = getRows(2);
|
||||
VariableResolverImpl vr = new VariableResolverImpl();
|
||||
|
||||
HashMap<String, String> ea = new HashMap<String, String>();
|
||||
ea.put("query", "SELECT * FROM A");
|
||||
ea.put("transformer", T2.class.getName());
|
||||
sep.init(getContext(null, vr, getDs(rows), Context.FULL_DUMP, null, ea));
|
||||
|
||||
local.set(0);
|
||||
Map<String, Object> r = null;
|
||||
int count = 0;
|
||||
while (true) {
|
||||
r = sep.nextRow();
|
||||
if (r == null)
|
||||
break;
|
||||
count++;
|
||||
public void testChildEntities() throws Exception {
|
||||
withChildEntities(false, true);
|
||||
}
|
||||
assertEquals(2, (int) local.get());
|
||||
assertEquals(4, count);
|
||||
@Test
|
||||
public void testCachedChildEntities() throws Exception {
|
||||
withChildEntities(true, true);
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> getRows(int count) {
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>();
|
||||
for (int i = 0; i < count; i++) {
|
||||
Map<String, Object> row = new HashMap<String, Object>();
|
||||
row.put("id", i);
|
||||
row.put("value", "The value is " + i);
|
||||
rows.add(row);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
private static DataSource<Iterator<Map<String, Object>>> getDs(
|
||||
final List<Map<String, Object>> rows) {
|
||||
return new DataSource<Iterator<Map<String, Object>>>() {
|
||||
@Override
|
||||
public Iterator<Map<String, Object>> getData(String query) {
|
||||
return rows.iterator();
|
||||
@Test
|
||||
@Ignore("broken see SOLR-3857")
|
||||
public void testSimpleCacheChildEntities() throws Exception {
|
||||
simpleCacheChildEntities(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Context context, Properties initProps) {
|
||||
protected String deltaQueriesCountryTable() {
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
public static class T extends Transformer {
|
||||
@Override
|
||||
public Object transformRow(Map<String, Object> aRow, Context context) {
|
||||
aRow.put("T", "Class T");
|
||||
return aRow;
|
||||
}
|
||||
}
|
||||
|
||||
public static class T3 {
|
||||
public Object transformRow(Map<String, Object> aRow) {
|
||||
aRow.put("T3", "T3 class");
|
||||
return aRow;
|
||||
}
|
||||
}
|
||||
|
||||
public static class T2 extends Transformer {
|
||||
@Override
|
||||
public Object transformRow(Map<String, Object> aRow, Context context) {
|
||||
Integer count = local.get();
|
||||
local.set(count + 1);
|
||||
List<Map<String, Object>> l = new ArrayList<Map<String, Object>>();
|
||||
l.add(aRow);
|
||||
l.add(aRow);
|
||||
return l;
|
||||
}
|
||||
protected String deltaQueriesPersonTable() {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -42,149 +42,23 @@ public class TestSqlEntityProcessor2 extends AbstractDataImportHandlerTestCase {
|
|||
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
|
||||
}
|
||||
|
||||
@Before @Override
|
||||
@Before
|
||||
@Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_FullImport() throws Exception {
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x", parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
|
||||
MockDataSource.setIterator("select * from y where y.A=1", childRow
|
||||
.iterator());
|
||||
|
||||
runFullImport(dataConfig);
|
||||
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_FullImportWithoutCommit() throws Exception {
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "10"));
|
||||
MockDataSource.setIterator("select * from x", parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
|
||||
MockDataSource.setIterator("select * from y where y.A=10", childRow
|
||||
.iterator());
|
||||
|
||||
|
||||
runFullImport(dataConfig,createMap("commit","false"));
|
||||
assertQ(req("id:10"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport() throws Exception {
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "5"));
|
||||
MockDataSource.setIterator("select id from x where last_modified > NOW",
|
||||
deltaRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "5"));
|
||||
MockDataSource.setIterator("select * from x where id = '5'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A=5", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig);
|
||||
|
||||
assertQ(req("id:5"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_DeletedPkQuery() throws Exception {
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "11"));
|
||||
MockDataSource.setIterator("select * from x", parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
|
||||
MockDataSource.setIterator("select * from y where y.A=11", childRow
|
||||
.iterator());
|
||||
|
||||
runFullImport(dataConfig);
|
||||
|
||||
assertQ(req("id:11"), "//*[@numFound='1']");
|
||||
|
||||
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "15"));
|
||||
deltaRow.add(createMap("id", "17"));
|
||||
MockDataSource.setIterator("select id from x where last_modified > NOW",
|
||||
deltaRow.iterator());
|
||||
|
||||
List deltaDeleteRow = new ArrayList();
|
||||
deltaDeleteRow.add(createMap("id", "11"));
|
||||
deltaDeleteRow.add(createMap("id", "17"));
|
||||
MockDataSource.setIterator("select id from x where last_modified > NOW AND deleted='true'",
|
||||
deltaDeleteRow.iterator());
|
||||
|
||||
parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "15"));
|
||||
MockDataSource.setIterator("select * from x where id = '15'", parentRow
|
||||
.iterator());
|
||||
|
||||
parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "17"));
|
||||
MockDataSource.setIterator("select * from x where id = '17'", parentRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig);
|
||||
|
||||
assertQ(req("id:15"), "//*[@numFound='1']");
|
||||
assertQ(req("id:11"), "//*[@numFound='0']");
|
||||
assertQ(req("id:17"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_DeltaImportQuery() throws Exception {
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "5"));
|
||||
MockDataSource.setIterator("select id from x where last_modified > NOW",
|
||||
deltaRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "5"));
|
||||
MockDataSource.setIterator("select * from x where id=5", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A=5", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_deltaimportquery);
|
||||
|
||||
assertQ(req("id:5"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
@Ignore("Known Locale/TZ problems: see https://issues.apache.org/jira/browse/SOLR-1916")
|
||||
/**
|
||||
* This test is here for historical purposes only.
|
||||
* When SOLR-1916 is fixed, it would be best to rewrite this test.
|
||||
*
|
||||
* @throws Exception
|
||||
*/
|
||||
public void testLastIndexTime() throws Exception {
|
||||
List row = new ArrayList();
|
||||
row.add(createMap("id", 5));
|
||||
|
@ -220,22 +94,4 @@ public class TestSqlEntityProcessor2 extends AbstractDataImportHandlerTestCase {
|
|||
"\t\t<entity name=\"x\" query=\"select * from x where last_modified > ${dih.functions.checkDateFormat(dih.last_index_time)}\" />\n" +
|
||||
"\t</document>\n" +
|
||||
"</dataConfig>";
|
||||
|
||||
private static String dataConfig = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
|
||||
+ " <document>\n"
|
||||
+ " <entity name=\"x\" pk=\"id\" query=\"select * from x\" deletedPkQuery=\"select id from x where last_modified > NOW AND deleted='true'\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
|
||||
+ " <field column=\"id\" />\n"
|
||||
+ " <entity name=\"y\" query=\"select * from y where y.A=${x.id}\">\n"
|
||||
+ " <field column=\"desc\" />\n"
|
||||
+ " </entity>\n" + " </entity>\n"
|
||||
+ " </document>\n" + "</dataConfig>\n";
|
||||
|
||||
private static String dataConfig_deltaimportquery = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
|
||||
+ " <document>\n"
|
||||
+ " <entity name=\"x\" deltaImportQuery=\"select * from x where id=${dataimporter.delta.id}\" deltaQuery=\"select id from x where last_modified > NOW\">\n"
|
||||
+ " <field column=\"id\" />\n"
|
||||
+ " <entity name=\"y\" query=\"select * from y where y.A=${x.id}\">\n"
|
||||
+ " <field column=\"desc\" />\n"
|
||||
+ " </entity>\n" + " </entity>\n"
|
||||
+ " </document>\n" + "</dataConfig>\n";
|
||||
}
|
||||
|
|
|
@ -1,3 +1,9 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
|
@ -14,330 +20,133 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.File;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
*
|
||||
* @since solr 1.3
|
||||
* Test with various combinations of parameters, child entites, transformers.
|
||||
*/
|
||||
@Ignore("FIXME: I fail so often it makes me ill!")
|
||||
public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCase {
|
||||
private static final String FULLIMPORT_QUERY = "select * from x";
|
||||
public class TestSqlEntityProcessorDelta extends AbstractDIHJdbcTestCase {
|
||||
private boolean delta = false;
|
||||
private boolean useParentDeltaQueryParam = false;
|
||||
private IntChanges personChanges = null;
|
||||
private String[] countryChanges = null;
|
||||
|
||||
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
|
||||
|
||||
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
|
||||
|
||||
private static final String dataConfig_delta =
|
||||
"<dataConfig>" +
|
||||
" <dataSource type=\"MockDataSource\"/>\n" +
|
||||
" <document>\n" +
|
||||
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
|
||||
" query=\"" + FULLIMPORT_QUERY + "\"" +
|
||||
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
|
||||
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
|
||||
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
|
||||
" <field column=\"id\" name=\"id\"/>\n" +
|
||||
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
|
||||
" <field column=\"desc\" />\n" +
|
||||
" </entity>\n" +
|
||||
" </entity>\n" +
|
||||
" </document>\n" +
|
||||
"</dataConfig>\n";
|
||||
|
||||
//TODO: fix this test to not require FSDirectory.
|
||||
static String savedFactory;
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
savedFactory = System.getProperty("solr.DirectoryFactory");
|
||||
System.setProperty("solr.directoryFactory", "solr.MockFSDirectoryFactory");
|
||||
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
|
||||
@Before
|
||||
public void setupDeltaTest() {
|
||||
delta = false;
|
||||
personChanges = null;
|
||||
countryChanges = null;
|
||||
}
|
||||
@Test
|
||||
public void testSingleEntity() throws Exception {
|
||||
singleEntity(1);
|
||||
changeStuff();
|
||||
int c = calculateDatabaseCalls();
|
||||
singleEntity(c);
|
||||
validateChanges();
|
||||
}
|
||||
@Test
|
||||
public void testWithSimpleTransformer() throws Exception {
|
||||
simpleTransform(1);
|
||||
changeStuff();
|
||||
simpleTransform(calculateDatabaseCalls());
|
||||
validateChanges();
|
||||
}
|
||||
@Test
|
||||
public void testWithComplexTransformer() throws Exception {
|
||||
complexTransform(1, 0);
|
||||
changeStuff();
|
||||
complexTransform(calculateDatabaseCalls(), personChanges.deletedKeys.length);
|
||||
validateChanges();
|
||||
}
|
||||
@Test
|
||||
public void testChildEntities() throws Exception {
|
||||
useParentDeltaQueryParam = random().nextBoolean();
|
||||
withChildEntities(false, true);
|
||||
changeStuff();
|
||||
withChildEntities(false, false);
|
||||
validateChanges();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() {
|
||||
if (savedFactory == null) {
|
||||
System.clearProperty("solr.directoryFactory");
|
||||
|
||||
private int calculateDatabaseCalls() {
|
||||
//The main query generates 1
|
||||
//Deletes generate 1
|
||||
//Each add/mod generate 1
|
||||
int c = 1;
|
||||
if (countryChanges != null) {
|
||||
c += countryChanges.length + 1;
|
||||
}
|
||||
if (personChanges != null) {
|
||||
c += personChanges.addedKeys.length + personChanges.changedKeys.length + 1;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
private void validateChanges() throws Exception
|
||||
{
|
||||
if(personChanges!=null) {
|
||||
for(int id : personChanges.addedKeys) {
|
||||
assertQ(req("id:" + id), "//*[@numFound='1']");
|
||||
}
|
||||
for(int id : personChanges.deletedKeys) {
|
||||
assertQ(req("id:" + id), "//*[@numFound='0']");
|
||||
}
|
||||
for(int id : personChanges.changedKeys) {
|
||||
assertQ(req("id:" + id), "//*[@numFound='1']", "substring(//doc/arr[@name='NAME_mult_s']/str[1], 1, 8)='MODIFIED'");
|
||||
}
|
||||
}
|
||||
if(countryChanges!=null) {
|
||||
for(String code : countryChanges) {
|
||||
assertQ(req("COUNTRY_CODE_s:" + code), "//*[@numFound='" + numberPeopleByCountryCode(code) + "']", "substring(//doc/str[@name='COUNTRY_NAME_s'], 1, 8)='MODIFIED'");
|
||||
}
|
||||
}
|
||||
}
|
||||
private void changeStuff() throws Exception {
|
||||
if(countryEntity)
|
||||
{
|
||||
int n = random().nextInt(2);
|
||||
switch(n) {
|
||||
case 0:
|
||||
personChanges = modifySomePeople();
|
||||
break;
|
||||
case 1:
|
||||
countryChanges = modifySomeCountries();
|
||||
break;
|
||||
case 2:
|
||||
personChanges = modifySomePeople();
|
||||
countryChanges = modifySomeCountries();
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
System.setProperty("solr.directoryFactory", savedFactory);
|
||||
personChanges = modifySomePeople();
|
||||
}
|
||||
delta = true;
|
||||
}
|
||||
protected LocalSolrQueryRequest generateRequest() {
|
||||
return lrf.makeRequest("command", (delta ? "delta-import" : "full-import"), "dataConfig", generateConfig(),
|
||||
"clean", (delta ? "false" : "true"), "commit", "true", "synchronous", "true", "indent", "true");
|
||||
}
|
||||
|
||||
@Before @Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
protected String deltaQueriesPersonTable() {
|
||||
return
|
||||
"deletedPkQuery=''SELECT ID FROM PEOPLE WHERE DELETED='Y' AND last_modified >='${dih.last_index_time}' '' " +
|
||||
"deltaImportQuery=''SELECT ID, NAME, COUNTRY_CODE FROM PEOPLE where ID=${dih.delta.ID} '' " +
|
||||
"deltaQuery=''" +
|
||||
"SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND last_modified >='${dih.last_index_time}' " +
|
||||
(useParentDeltaQueryParam ? "" :
|
||||
"UNION DISTINCT " +
|
||||
"SELECT ID FROM PEOPLE WHERE DELETED!='Y' AND COUNTRY_CODE IN (SELECT CODE FROM COUNTRIES WHERE last_modified >='${dih.last_index_time}') "
|
||||
) + "'' "
|
||||
;
|
||||
}
|
||||
@Override
|
||||
protected String deltaQueriesCountryTable() {
|
||||
if(useParentDeltaQueryParam) {
|
||||
return
|
||||
"deltaQuery=''SELECT CODE FROM COUNTRIES WHERE DELETED != 'Y' AND last_modified >='${dih.last_index_time}' '' " +
|
||||
"parentDeltaQuery=''SELECT ID FROM PEOPLE WHERE DELETED != 'Y' AND COUNTRY_CODE='${Countries.CODE}' '' "
|
||||
;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void add1document() throws Exception {
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runFullImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_FullImport() throws Exception {
|
||||
add1document();
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testNonWritablePersistFile() throws Exception {
|
||||
// See SOLR-2551
|
||||
String configDir = h.getCore().getResourceLoader().getConfigDir();
|
||||
String filePath = configDir;
|
||||
if (configDir != null && !configDir.endsWith(File.separator))
|
||||
filePath += File.separator;
|
||||
filePath += "dataimport.properties";
|
||||
File f = new File(filePath);
|
||||
|
||||
try {
|
||||
// execute the test only if we are able to set file to read only mode
|
||||
assumeTrue("No dataimport.properties file", f.exists() || f.createNewFile());
|
||||
assumeTrue("dataimport.proprties can't be set read only", f.setReadOnly());
|
||||
assumeFalse("dataimport.proprties is still writable even though " +
|
||||
"marked readonly - test running as superuser?", f.canWrite());
|
||||
|
||||
ignoreException("Properties is not writable");
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'",
|
||||
childRow.iterator());
|
||||
|
||||
runFullImport(dataConfig_delta);
|
||||
assertQ(req("id:1"), "//*[@numFound='0']");
|
||||
} finally {
|
||||
f.delete();
|
||||
}
|
||||
}
|
||||
|
||||
// WORKS
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_delete() throws Exception {
|
||||
add1document();
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(DELTA_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_empty() throws Exception {
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'",
|
||||
childRow.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
// WORKS
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_replace_delete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY,
|
||||
deletedRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
|
||||
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_add() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator("select * from x where id='2'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='2'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_nodelta() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
Collections.EMPTY_LIST.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_add_delete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY,
|
||||
deletedRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator("select * from x where id='2'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='2'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='0']");
|
||||
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,288 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
*
|
||||
* @since solr 1.3
|
||||
*/
|
||||
@Ignore("FIXME: I fail so often it makes me ill!")
|
||||
public class TestSqlEntityProcessorDelta2 extends AbstractDataImportHandlerTestCase {
|
||||
private static final String FULLIMPORT_QUERY = "select * from x";
|
||||
|
||||
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
|
||||
|
||||
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
|
||||
|
||||
private static final String dataConfig_delta2 =
|
||||
"<dataConfig>" +
|
||||
" <dataSource type=\"MockDataSource\"/>\n" +
|
||||
" <document>\n" +
|
||||
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
|
||||
" query=\"" + FULLIMPORT_QUERY + "\"" +
|
||||
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
|
||||
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
|
||||
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
|
||||
" <field column=\"tmpid\" template=\"prefix-${x.id}\" name=\"solr_id\"/>\n" +
|
||||
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
|
||||
" <field column=\"desc\" />\n" +
|
||||
" </entity>\n" +
|
||||
" </entity>\n" +
|
||||
" </document>\n" +
|
||||
"</dataConfig>\n";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("dataimport-solrconfig.xml", "dataimport-solr_id-schema.xml");
|
||||
}
|
||||
|
||||
@Before @Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void add1document() throws Exception {
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runFullImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
|
||||
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_FullImport() throws Exception {
|
||||
add1document();
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_delete() throws Exception {
|
||||
add1document();
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(DELTA_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_empty() throws Exception {
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "hello"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
|
||||
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_replace_delete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY,
|
||||
deletedRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='1'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
|
||||
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
|
||||
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_add() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator("select * from x where id='2'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='2'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
|
||||
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
|
||||
assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_nodelta() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
Collections.EMPTY_LIST.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
assertQ(req("solr_id:prefix-1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_add_delete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY,
|
||||
deletedRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator("select * from x where id='2'", parentRow
|
||||
.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("desc", "goodbye"));
|
||||
MockDataSource.setIterator("select * from y where y.A='2'", childRow
|
||||
.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta2);
|
||||
|
||||
assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
|
||||
assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:hello"), "//*[@numFound='0']");
|
||||
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
|
||||
}
|
||||
}
|
|
@ -1,236 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
@Ignore("FIXME: I fail so often it makes me ill!")
|
||||
public class TestSqlEntityProcessorDelta3 extends AbstractDataImportHandlerTestCase {
|
||||
private static final String P_FULLIMPORT_QUERY = "select * from parent";
|
||||
private static final String P_DELTA_QUERY = "select parent_id from parent where last_modified > NOW";
|
||||
private static final String P_DELTAIMPORT_QUERY = "select * from parent where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
|
||||
|
||||
private static final String C_FULLIMPORT_QUERY = "select * from child";
|
||||
private static final String C_DELETED_PK_QUERY = "select id from child where last_modified > NOW AND deleted='true'";
|
||||
private static final String C_DELTA_QUERY = "select id from child where last_modified > NOW";
|
||||
private static final String C_PARENTDELTA_QUERY = "select parent_id from child where id=${child.id}";
|
||||
private static final String C_DELTAIMPORT_QUERY = "select * from child where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
|
||||
|
||||
private static final String dataConfig_delta =
|
||||
"<dataConfig>" +
|
||||
" <dataSource type=\"MockDataSource\"/>\n" +
|
||||
" <document>" +
|
||||
" <entity name=\"parent\" pk=\"parent_id\" rootEntity=\"false\"" +
|
||||
" query=\"" + P_FULLIMPORT_QUERY + "\"" +
|
||||
" deltaQuery=\"" + P_DELTA_QUERY + "\"" +
|
||||
" deltaImportQuery=\"" + P_DELTAIMPORT_QUERY + "\">" +
|
||||
" <field column=\"desc\" name=\"desc\"/>" +
|
||||
" <entity name=\"child\" pk=\"id\" rootEntity=\"true\"" +
|
||||
" query=\"" + C_FULLIMPORT_QUERY + "\"" +
|
||||
" deletedPkQuery=\"" + C_DELETED_PK_QUERY + "\"" +
|
||||
" deltaQuery=\"" + C_DELTA_QUERY + "\"" +
|
||||
" parentDeltaQuery=\"" + C_PARENTDELTA_QUERY + "\"" +
|
||||
" deltaImportQuery=\"" + C_DELTAIMPORT_QUERY + "\">" +
|
||||
" <field column=\"id\" name=\"id\" />" +
|
||||
" </entity>" +
|
||||
" </entity>" +
|
||||
" </document>" +
|
||||
"</dataConfig>\n";
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
|
||||
}
|
||||
|
||||
@Before @Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void add1document() throws Exception {
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("parent_id", "1", "desc", "d1"));
|
||||
MockDataSource.setIterator(P_FULLIMPORT_QUERY, parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(C_FULLIMPORT_QUERY, childRow.iterator());
|
||||
|
||||
runFullImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1"), "//*[@numFound='0']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:d1"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_FullImport() throws Exception {
|
||||
add1document();
|
||||
}
|
||||
|
||||
// WORKS
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_delete() throws Exception {
|
||||
add1document();
|
||||
List deletedRow = new ArrayList();
|
||||
deletedRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(C_DELETED_PK_QUERY, deletedRow.iterator());
|
||||
MockDataSource.setIterator(C_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
|
||||
|
||||
List deletedParentRow = new ArrayList();
|
||||
deletedParentRow.add(createMap("parent_id", "1"));
|
||||
MockDataSource.setIterator("select parent_id from child where id=2", deletedParentRow.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_empty() throws Exception {
|
||||
List childDeltaRow = new ArrayList();
|
||||
childDeltaRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(C_DELTA_QUERY, childDeltaRow.iterator());
|
||||
MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections.EMPTY_LIST.iterator());
|
||||
|
||||
List childParentDeltaRow = new ArrayList();
|
||||
childParentDeltaRow.add(createMap("parent_id", "1"));
|
||||
MockDataSource.setIterator("select parent_id from child where id=2", childParentDeltaRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(P_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
|
||||
|
||||
List parentDeltaImportRow = new ArrayList();
|
||||
parentDeltaImportRow.add(createMap("parent_id", "1", "desc", "d1"));
|
||||
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
|
||||
parentDeltaImportRow.iterator());
|
||||
|
||||
List childDeltaImportRow = new ArrayList();
|
||||
childDeltaImportRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
|
||||
childDeltaImportRow.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:d1"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRow = new ArrayList();
|
||||
deltaRow.add(createMap("parent_id", "1"));
|
||||
MockDataSource.setIterator(P_DELTA_QUERY,
|
||||
deltaRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("parent_id", "1", "desc", "d2"));
|
||||
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
|
||||
parentRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
|
||||
childRow.iterator());
|
||||
|
||||
MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections
|
||||
.EMPTY_LIST.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:s1 OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
|
||||
assertQ(req("desc:d2"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_add() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List parentDeltaRow = new ArrayList();
|
||||
parentDeltaRow.add(createMap("parent_id", "1"));
|
||||
MockDataSource.setIterator(P_DELTA_QUERY,
|
||||
parentDeltaRow.iterator());
|
||||
|
||||
List parentRow = new ArrayList();
|
||||
parentRow.add(createMap("parent_id", "1", "desc", "d1"));
|
||||
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
|
||||
parentRow.iterator());
|
||||
|
||||
List childDeltaRow = new ArrayList();
|
||||
childDeltaRow.add(createMap("id", "3"));
|
||||
MockDataSource.setIterator(C_DELTA_QUERY,
|
||||
childDeltaRow.iterator());
|
||||
|
||||
List childParentDeltaRow = new ArrayList();
|
||||
childParentDeltaRow.add(createMap("parent_id", "1"));
|
||||
MockDataSource.setIterator("select parent_id from child where id='3'",
|
||||
childParentDeltaRow.iterator());
|
||||
|
||||
List childRow = new ArrayList();
|
||||
childRow.add(createMap("id", "3"));
|
||||
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
|
||||
childRow.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("id:3"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:d1"), "//*[@numFound='2']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testCompositePk_DeltaImport_nodelta() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
MockDataSource.setIterator(P_DELTA_QUERY,
|
||||
Collections.EMPTY_LIST.iterator());
|
||||
|
||||
MockDataSource.setIterator(C_DELTA_QUERY,
|
||||
Collections.EMPTY_LIST.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
assertQ(req("id:2 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:d1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
|
||||
}
|
||||
}
|
|
@ -1,147 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.logging.*;
|
||||
|
||||
/**
|
||||
* <p>
|
||||
* Test for SqlEntityProcessorDelta verifying fix for SOLR-1191
|
||||
* </p>
|
||||
*
|
||||
*
|
||||
*
|
||||
* @since solr 3.1
|
||||
*/
|
||||
@Ignore("FIXME: I fail so often it makes me ill!")
|
||||
public class TestSqlEntityProcessorDeltaPrefixedPk extends AbstractDataImportHandlerTestCase {
|
||||
private static final String FULLIMPORT_QUERY = "select * from x";
|
||||
|
||||
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
|
||||
|
||||
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
|
||||
|
||||
private static final String dataConfig_delta =
|
||||
"<dataConfig>" +
|
||||
" <dataSource type=\"MockDataSource\"/>\n" +
|
||||
" <document>\n" +
|
||||
" <entity name=\"x\" transformer=\"TemplateTransformer\" pk=\"x.id\"" +
|
||||
" query=\"" + FULLIMPORT_QUERY + "\"" +
|
||||
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
|
||||
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
|
||||
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
|
||||
" <field column=\"id\" name=\"id\"/>\n" +
|
||||
" <field column=\"desc\" name=\"desc\"/>\n" +
|
||||
" </entity>\n" +
|
||||
" </document>\n" +
|
||||
"</dataConfig>\n";
|
||||
|
||||
private static final List EMPTY_LIST = Collections.EMPTY_LIST;
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
|
||||
}
|
||||
|
||||
@Before @Override
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
clearIndex();
|
||||
assertU(commit());
|
||||
//Logger.getLogger("").setLevel(Level.ALL);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private void add1document() throws Exception {
|
||||
List row = new ArrayList();
|
||||
row.add(createMap("id", "1", "desc", "bar"));
|
||||
MockDataSource.setIterator(FULLIMPORT_QUERY, row.iterator());
|
||||
|
||||
runFullImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:bar"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testDeltaImport_deleteResolvesUnprefixedPk() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
List deletedRows = new ArrayList();
|
||||
deletedRows.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRows.iterator());
|
||||
MockDataSource.setIterator(DELTA_QUERY, EMPTY_LIST.iterator());
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testDeltaImport_deleteResolvesUnprefixedPk"), "//*[@numFound='0']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testDeltaImport_replace_resolvesUnprefixedPk() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
List deltaRows = new ArrayList();
|
||||
deltaRows.add(createMap("id", "1"));
|
||||
MockDataSource.setIterator(DELTA_QUERY, deltaRows.iterator());
|
||||
MockDataSource.setIterator(DELETED_PK_QUERY, EMPTY_LIST.iterator());
|
||||
List rows = new ArrayList();
|
||||
rows.add(createMap("id", "1", "desc", "baz"));
|
||||
MockDataSource.setIterator("select * from x where id='1'", rows.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testDeltaImport_replace_resolvesUnprefixedPk"), "//*[@numFound='1']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:bar"), "//*[@numFound='0']");
|
||||
assertQ(req("desc:baz"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("unchecked")
|
||||
public void testDeltaImport_addResolvesUnprefixedPk() throws Exception {
|
||||
add1document();
|
||||
MockDataSource.clearCache();
|
||||
|
||||
List deltaRows = new ArrayList();
|
||||
deltaRows.add(createMap("id", "2"));
|
||||
MockDataSource.setIterator(DELTA_QUERY, deltaRows.iterator());
|
||||
|
||||
List rows = new ArrayList();
|
||||
rows.add(createMap("id", "2", "desc", "xyzzy"));
|
||||
MockDataSource.setIterator("select * from x where id='2'", rows.iterator());
|
||||
|
||||
runDeltaImport(dataConfig_delta);
|
||||
|
||||
assertQ(req("*:* OR testDeltaImport_addResolvesUnprefixedPk"), "//*[@numFound='2']");
|
||||
assertQ(req("id:1"), "//*[@numFound='1']");
|
||||
assertQ(req("id:2"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:bar"), "//*[@numFound='1']");
|
||||
assertQ(req("desc:xyzzy"), "//*[@numFound='1']");
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,75 @@
|
|||
package org.apache.solr.handler.dataimport;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
/**
|
||||
* This transformer does 3 things
|
||||
* <ul>
|
||||
* <li>It turns every row into 3 rows,
|
||||
* modifying any "id" column to ensure duplicate entries in the index
|
||||
* <li>The 2nd Row has 2x values for every column,
|
||||
* with the added one being backwards of the original
|
||||
* <li>The 3rd Row has an added static value
|
||||
* </ul>
|
||||
*
|
||||
* Also, this does not extend Transformer.
|
||||
*/
|
||||
public class TripleThreatTransformer {
|
||||
public Object transformRow(Map<String, Object> row) {
|
||||
List<Map<String, Object>> rows = new ArrayList<Map<String, Object>>(3);
|
||||
rows.add(row);
|
||||
rows.add(addDuplicateBackwardsValues(row));
|
||||
rows.add(new LinkedHashMap<String,Object>(row));
|
||||
rows.get(2).put("AddAColumn_s", "Added");
|
||||
modifyIdColumn(rows.get(1), 1);
|
||||
modifyIdColumn(rows.get(2), 2);
|
||||
return rows;
|
||||
}
|
||||
private LinkedHashMap<String,Object> addDuplicateBackwardsValues(Map<String, Object> row) {
|
||||
LinkedHashMap<String,Object> n = new LinkedHashMap<String,Object>();
|
||||
for(Map.Entry<String,Object> entry : row.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
if(!"id".equalsIgnoreCase(key)) {
|
||||
String[] vals = new String[2];
|
||||
vals[0] = entry.getValue()==null ? "null" : entry.getValue().toString();
|
||||
vals[1] = new StringBuilder(vals[0]).reverse().toString();
|
||||
n.put(key, Arrays.asList(vals));
|
||||
} else {
|
||||
n.put(key, entry.getValue());
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
private void modifyIdColumn(Map<String, Object> row, int num) {
|
||||
Object o = row.remove("ID");
|
||||
if(o==null) {
|
||||
o = row.remove("id");
|
||||
}
|
||||
if(o!=null) {
|
||||
String id = o.toString();
|
||||
id = "TripleThreat-" + num + "-" + id;
|
||||
row.put("id", id);
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue