Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2252: when child entity is the root in nested entities, delta-import doesn't work 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1040608 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Koji Sekiguchi 2010-11-30 16:10:29 +00:00
parent 88a97b22f8
commit 0511306b53
6 changed files with 286 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
solr/CHANGES.txt
View File

@ -259,7 +259,7 @@ New Features
allows you to customize how WordDelimiterFilter tokenizes text with
a configuration file. (Peter Karich, rmuir)
* SOLR-1665: Add debug component options for timings, results and query info only (gsingers, hossman, yonik)
* SOLR-1665: Add debug component options for timings, results and query info only (gsingers, hossman, yonik)
* SOLR-2099: Add ability to throttle rsync based replication using rsync option --bwlimit.
(Brandon Evans via koji)
@ -555,6 +555,9 @@ Bug Fixes
true/on/yes (for TRUE) and false/off/no (for FALSE) can be used for sub-options
(debug, verbose, synchronous, commit, clean, optimize) for full/delta-import commands. (koji)
* SOLR-2252: When a child entity in nested entities is rootEntity="true", delta-import doesn't work.
(koji)
Other Changes
----------------------

4
solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java
View File

@ -175,6 +175,10 @@ public class DataConfig {
public String getPk(){
return pk == null ? pkMappingFromSchema : pk;
}
public String getSchemaPk(){
return pkMappingFromSchema != null ? pkMappingFromSchema : pk;
}
}
public static class Script {

13
solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -314,9 +314,10 @@ public class DocBuilder {
Iterator<Map<String, Object>> iter = deletedKeys.iterator();
while (iter.hasNext()) {
Map<String, Object> map = iter.next();
Object key = map.get(root.getPk());
String keyName = root.isDocRoot ? root.getPk() : root.getSchemaPk();
Object key = map.get(keyName);
if(key == null) {
LOG.warn("no key was available for deleteted pk query");
LOG.warn("no key was available for deleteted pk query. keyName = " + keyName);
continue;
}
writer.deleteDoc(key);
@ -607,7 +608,8 @@ public class DocBuilder {
if (entity.entities != null) {
vr.addNamespace(entity.name, arow);
for (DataConfig.Entity child : entity.entities) {
buildDocument(vr, doc, null, child, false, ctx);
buildDocument(vr, doc,
child.isDocRoot ? pk : null, child, false, ctx);
}
vr.removeNamespace(entity.name);
}
@ -910,8 +912,9 @@ public class DocBuilder {
if (entity.isDocRoot)
deletedRows.addAll(deletedSet);
return entity.isDocRoot ? myModifiedPks : new HashSet<Map<String, Object>>(
parentKeyList);
// Do not use entity.isDocRoot here because one of descendant entities may set rootEntity="true"
return entity.parentEntity == null ?
myModifiedPks : new HashSet<Map<String, Object>>(parentKeyList);
}
private void getModifiedParentRows(VariableResolverImpl resolver,

35
solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta.java
View File

@ -40,6 +40,23 @@ public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCa
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
private static final String dataConfig_delta =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>\n" +
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
" query=\"" + FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
" <field column=\"id\" name=\"id\"/>\n" +
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
" <field column=\"desc\" />\n" +
" </entity>\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
@ -115,8 +132,8 @@ public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCa
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
MockDataSource.setIterator("select * from y where y.A='1'",
childRow.iterator());
runDeltaImport(dataConfig_delta);
@ -270,18 +287,4 @@ public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTestCa
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
private static String dataConfig_delta = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
+ " <document>\n"
+ " <entity name=\"x\" transformer=\"TemplateTransformer\""
+ " query=\"" + FULLIMPORT_QUERY + "\""
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
+ " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n"
+ " <field column=\"id\" name=\"id\"/>\n"
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
+ " <field column=\"desc\" />\n"
+ " </entity>\n" + " </entity>\n"
+ " </document>\n" + "</dataConfig>\n";
}

31
solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta2.java
View File

@ -40,6 +40,23 @@ public class TestSqlEntityProcessorDelta2 extends AbstractDataImportHandlerTestC
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
private static final String dataConfig_delta2 =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>\n" +
" <entity name=\"x\" transformer=\"TemplateTransformer\"" +
" query=\"" + FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + DELETED_PK_QUERY + "\"" +
" deltaImportQuery=\"select * from x where id='${dih.delta.id}'\"" +
" deltaQuery=\"" + DELTA_QUERY + "\">\n" +
" <field column=\"tmpid\" template=\"prefix-${x.id}\" name=\"solr_id\"/>\n" +
" <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n" +
" <field column=\"desc\" />\n" +
" </entity>\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>\n";
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-solr_id-schema.xml");
@ -266,18 +283,4 @@ public class TestSqlEntityProcessorDelta2 extends AbstractDataImportHandlerTestC
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
private static String dataConfig_delta2 = "<dataConfig><dataSource type=\"MockDataSource\"/>\n"
+ " <document>\n"
+ " <entity name=\"x\" transformer=\"TemplateTransformer\""
+ " query=\"" + FULLIMPORT_QUERY + "\""
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
+ " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n"
+ " <field column=\"tmpid\" template=\"prefix-${x.id}\" name=\"solr_id\"/>\n"
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
+ " <field column=\"desc\" />\n"
+ " </entity>\n" + " </entity>\n"
+ " </document>\n" + "</dataConfig>\n";
}

234
solr/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestSqlEntityProcessorDelta3.java Normal file
View File

@ -0,0 +1,234 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class TestSqlEntityProcessorDelta3 extends AbstractDataImportHandlerTestCase {
private static final String P_FULLIMPORT_QUERY = "select * from parent";
private static final String P_DELTA_QUERY = "select parent_id from parent where last_modified > NOW";
private static final String P_DELTAIMPORT_QUERY = "select * from parent where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
private static final String C_FULLIMPORT_QUERY = "select * from child";
private static final String C_DELETED_PK_QUERY = "select id from child where last_modified > NOW AND deleted='true'";
private static final String C_DELTA_QUERY = "select id from child where last_modified > NOW";
private static final String C_PARENTDELTA_QUERY = "select parent_id from child where id=${child.id}";
private static final String C_DELTAIMPORT_QUERY = "select * from child where last_modified > NOW AND parent_id=${dih.delta.parent_id}";
private static final String dataConfig_delta =
"<dataConfig>" +
" <dataSource type=\"MockDataSource\"/>\n" +
" <document>" +
" <entity name=\"parent\" pk=\"parent_id\" rootEntity=\"false\"" +
" query=\"" + P_FULLIMPORT_QUERY + "\"" +
" deltaQuery=\"" + P_DELTA_QUERY + "\"" +
" deltaImportQuery=\"" + P_DELTAIMPORT_QUERY + "\">" +
" <field column=\"desc\" name=\"desc\"/>" +
" <entity name=\"child\" pk=\"id\" rootEntity=\"true\"" +
" query=\"" + C_FULLIMPORT_QUERY + "\"" +
" deletedPkQuery=\"" + C_DELETED_PK_QUERY + "\"" +
" deltaQuery=\"" + C_DELTA_QUERY + "\"" +
" parentDeltaQuery=\"" + C_PARENTDELTA_QUERY + "\"" +
" deltaImportQuery=\"" + C_DELTAIMPORT_QUERY + "\">" +
" <field column=\"id\" name=\"id\" />" +
" </entity>" +
" </entity>" +
" </document>" +
"</dataConfig>\n";
@BeforeClass
public static void beforeClass() throws Exception {
initCore("dataimport-solrconfig.xml", "dataimport-schema.xml");
}
@Before @Override
public void setUp() throws Exception {
super.setUp();
clearIndex();
assertU(commit());
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("parent_id", "1", "desc", "d1"));
MockDataSource.setIterator(P_FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("id", "2"));
MockDataSource.setIterator(C_FULLIMPORT_QUERY, childRow.iterator());
runFullImport(dataConfig_delta);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='0']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:d1"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
add1document();
}
// WORKS
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_delete() throws Exception {
add1document();
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "2"));
MockDataSource.setIterator(C_DELETED_PK_QUERY, deletedRow.iterator());
MockDataSource.setIterator(C_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
List deletedParentRow = new ArrayList();
deletedParentRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator("select parent_id from child where id=2", deletedParentRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_empty() throws Exception {
List childDeltaRow = new ArrayList();
childDeltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(C_DELTA_QUERY, childDeltaRow.iterator());
MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections.EMPTY_LIST.iterator());
List childParentDeltaRow = new ArrayList();
childParentDeltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator("select parent_id from child where id=2", childParentDeltaRow.iterator());
MockDataSource.setIterator(P_DELTA_QUERY, Collections.EMPTY_LIST.iterator());
List parentDeltaImportRow = new ArrayList();
parentDeltaImportRow.add(createMap("parent_id", "1", "desc", "d1"));
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
parentDeltaImportRow.iterator());
List childDeltaImportRow = new ArrayList();
childDeltaImportRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
childDeltaImportRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:d1"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator(P_DELTA_QUERY,
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("parent_id", "1", "desc", "d2"));
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
childRow.iterator());
MockDataSource.setIterator(C_DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:s1 OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
assertQ(req("desc:d2"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add() throws Exception {
add1document();
MockDataSource.clearCache();
List parentDeltaRow = new ArrayList();
parentDeltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator(P_DELTA_QUERY,
parentDeltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("parent_id", "1", "desc", "d1"));
MockDataSource.setIterator("select * from parent where last_modified > NOW AND parent_id=1",
parentRow.iterator());
List childDeltaRow = new ArrayList();
childDeltaRow.add(createMap("id", "3"));
MockDataSource.setIterator(C_DELTA_QUERY,
childDeltaRow.iterator());
List childParentDeltaRow = new ArrayList();
childParentDeltaRow.add(createMap("parent_id", "1"));
MockDataSource.setIterator("select parent_id from child where id='3'",
childParentDeltaRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("id", "3"));
MockDataSource.setIterator("select * from child where last_modified > NOW AND parent_id=1",
childRow.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("id:3"), "//*[@numFound='1']");
assertQ(req("desc:d1"), "//*[@numFound='2']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_nodelta() throws Exception {
add1document();
MockDataSource.clearCache();
MockDataSource.setIterator(P_DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
MockDataSource.setIterator(C_DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("id:2 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("desc:d1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
}
}