Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-838 -- The VariableResolver obtained from a DataSource's context does not have current data 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@712067 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Shalin Shekhar Mangar 2008-11-07 05:51:41 +00:00
parent 809545b993
commit 49d7932a92
6 changed files with 45 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
contrib/dataimporthandler/CHANGES.txt
View File

@ -46,6 +46,9 @@ Bug Fixes
5. SOLR-832: Rows parameter is not honored in non-debug mode and can abort a running import in debug mode.
(Akshay Ukey, shalin)
6. SOLR-838: The VariableResolver obtained from a DataSource's context does not have current data.
(Noble Paul via shalin)
Documentation
----------------------

3
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/AbstractDataImportHandlerTest.java
View File

@ -21,7 +21,6 @@ import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.util.AbstractSolrTestCase;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -83,7 +82,7 @@ public abstract class AbstractDataImportHandlerTest extends
int currProcess, final List<Map<String, String>> entityFields,
final Map<String, String> entityAttrs) {
final Context delegate = new ContextImpl(parentEntity, resolver,
parentDataSource, currProcess, Collections.EMPTY_MAP,
parentDataSource, currProcess,
new HashMap<String, Object>(), null, null);
return new Context() {
public String getEntityAttribute(String name) {

26
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/ContextImpl.java
View File

@ -50,17 +50,22 @@ public class ContextImpl extends Context {
private Map<String, Object> entitySession, globalSession, docSession;
private DocBuilder docBuilder;
public ContextImpl(DataConfig.Entity entity, VariableResolverImpl resolver,
DataSource ds, int currProcess, Map<String, Object> requestParams,
Map<String, Object> global, ContextImpl p, DataImporter di) {
DataSource ds, int currProcess,
Map<String, Object> global, ContextImpl parentContext, DocBuilder docBuilder) {
this.entity = entity;
this.resolver = resolver;
this.ds = ds;
this.currProcess = currProcess;
this.requestParams = requestParams;
this.docBuilder = docBuilder;
if (docBuilder != null) {
this.requestParams = docBuilder.requestParameters.requestParams;
dataImporter = docBuilder.dataImporter;
}
globalSession = global;
parent = p;
dataImporter = di;
parent = parentContext;
}
public String getEntityAttribute(String name) {
@ -76,7 +81,16 @@ public class ContextImpl extends Context {
}
public DataSource getDataSource() {
return ds;
if (ds != null) return ds;
if (entity.dataSrc == null) {
entity.dataSrc = dataImporter.getDataSourceInstance(entity, entity.dataSource, this);
}
if (entity.dataSrc != null && docBuilder != null && docBuilder.verboseDebug &&
currProcess == Context.FULL_DUMP) {
//debug is not yet implemented properly for deltas
return DebugLogger.wrapDs(entity.dataSrc);
}
return entity.dataSrc;
}
public DataSource getDataSource(String name) {

28
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImporter.java
View File

@ -22,6 +22,8 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.InputSource;
@ -34,9 +36,6 @@ import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* <p>
* Stores all configuration information for pulling and indexing data.
@ -86,7 +85,7 @@ public class DataImporter {
}
DataImporter(String dataConfig, SolrCore core,
Map<String, Properties> ds) {
Map<String, Properties> ds) {
if (dataConfig == null)
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"Configuration not found");
@ -227,8 +226,6 @@ public class DataImporter {
e.allFieldsList = Collections.unmodifiableList(e.allFieldsList);
e.allAttributes = Collections.unmodifiableMap(e.allAttributes);
addDataSource(e);
if (e.entities == null)
return;
for (DataConfig.Entity e1 : e.entities) {
@ -266,16 +263,8 @@ public class DataImporter {
return store.get(key);
}
@SuppressWarnings("unchecked")
public void addDataSource(DataConfig.Entity key) {
if ("null".equals(key.dataSource)) {
key.dataSrc = new MockDataSource();
return;
}
key.dataSrc = getDataSourceInstance(key, key.dataSource, null);
}
DataSource getDataSourceInstance(DataConfig.Entity key, String name, Context ctx ) {
DataSource getDataSourceInstance(DataConfig.Entity key, String name, Context ctx) {
if ("null".equals(name)) return new MockDataSource();
Properties p = dataSourceProps.get(name);
if (p == null)
p = config.dataSources.get(name);
@ -302,9 +291,6 @@ public class DataImporter {
try {
Properties copyProps = new Properties();
copyProps.putAll(p);
if(ctx == null)
ctx = new ContextImpl(key, null, dataSrc, 0,
Collections.EMPTY_MAP, new HashMap(), null, this);
dataSrc.init(ctx, copyProps);
} catch (Exception e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
@ -341,7 +327,7 @@ public class DataImporter {
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (RuntimeException e) {
LOG.error( "Full Import failed", e);
LOG.error("Full Import failed", e);
} finally {
setStatus(Status.IDLE);
config.clearCaches();
@ -364,7 +350,7 @@ public class DataImporter {
if (!requestParams.debug)
cumulativeStatistics.add(docBuilder.importStatistics);
} catch (RuntimeException e) {
LOG.error( "Delta Import Failed", e);
LOG.error("Delta Import Failed", e);
} finally {
setStatus(Status.IDLE);
config.clearCaches();

26
contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
View File

@ -45,7 +45,7 @@ public class DocBuilder {
private static final Logger LOG = LoggerFactory.getLogger(DocBuilder.class);
private DataImporter dataImporter;
DataImporter dataImporter;
private DataConfig.Document document;
@ -236,13 +236,10 @@ public class DocBuilder {
ContextImpl parentCtx) {
EntityProcessor entityProcessor = getEntityProcessor(entity, dataImporter.getCore());
DataSource ds = entity.dataSrc;
if (verboseDebug) {
ds = DebugLogger.wrapDs(ds);
}
ContextImpl ctx = new ContextImpl(entity, vr, ds,
ContextImpl ctx = new ContextImpl(entity, vr, null,
pk == null ? Context.FULL_DUMP : Context.DELTA_DUMP,
requestParameters.requestParams, session, parentCtx, dataImporter);
session, parentCtx, this);
entityProcessor.init(ctx);
if (requestParameters.start > 0) {
@ -451,6 +448,7 @@ public class DocBuilder {
public Set<Map<String, Object>> collectDelta(DataConfig.Entity entity,
DataConfig.Entity parentEntity, VariableResolverImpl resolver,
DataImporter context, Set<Map<String, Object>> deletedRows) {
//someone called abort
if (stop.get())
return new HashSet();
@ -459,6 +457,7 @@ public class DocBuilder {
if (entity.entities != null) {
for (DataConfig.Entity entity1 : entity.entities) {
//this ensures that we start from the leaf nodes
myModifiedPks.addAll(collectDelta(entity1, entity, resolver, context,
deletedRows));
}
@ -469,11 +468,11 @@ public class DocBuilder {
Set<Map<String, Object>> deltaSet = new HashSet<Map<String, Object>>();
resolver.addNamespace(null, (Map) entity.allAttributes);
EntityProcessor entityProcessor = getEntityProcessor(entity, context.getCore());
entityProcessor.init(new ContextImpl(entity, resolver, entity.dataSrc,
Context.FIND_DELTA, requestParameters.requestParams, session, null,
dataImporter));
entityProcessor.init(new ContextImpl(entity, resolver, null,
Context.FIND_DELTA, session, null, this));
LOG.info("Running ModifiedRowKey() for Entity: " + entity.name);
int count = 0;
//get the modified rows in this entity
while (true) {
Map<String, Object> row = entityProcessor.nextModifiedRowKey();
@ -489,6 +488,7 @@ public class DocBuilder {
count = 0;
// identifying the deleted rows from this entities
LOG.info("Running DeletedRowKey() for Entity: " + entity.name);
//get the deleted rows for this entity
Set<Map<String, Object>> deletedSet = new HashSet<Map<String, Object>>();
while (true) {
Map<String, Object> row = entityProcessor.nextDeletedRowKey();
@ -504,11 +504,11 @@ public class DocBuilder {
myModifiedPks.addAll(deltaSet);
Set<Map<String, Object>> parentKeyList = new HashSet<Map<String, Object>>();
//all that we have captured is useless (in a sub-entity) if no rows in the parent is modified because of these
//so propogate up the changes in the chain
if (parentEntity != null && parentEntity.isDocRoot) {
EntityProcessor parentEntityProcessor = getEntityProcessor(parentEntity, context.getCore());
parentEntityProcessor.init(new ContextImpl(parentEntity, resolver,
parentEntity.dataSrc, Context.FIND_DELTA,
requestParameters.requestParams, session, null, dataImporter));
parentEntityProcessor.init(new ContextImpl(parentEntity, resolver, null, Context.FIND_DELTA, session, null, this));
// identifying deleted rows with deltas
for (Map<String, Object> row : myModifiedPks)

3
contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestEntityProcessorBase.java
View File

@ -43,8 +43,7 @@ public class TestEntityProcessorBase {
fields.add(TestRegexTransformer.getField("A", null, null, null, null));
fields.add(TestRegexTransformer.getField("B", null, null, null, null));
Context context = AbstractDataImportHandlerTest.getContext(null, null,
null, 0, fields, entity);
Context context = AbstractDataImportHandlerTest.getContext(null, null, new MockDataSource(), 0, fields, entity);
Map<String, Object> src = new HashMap<String, Object>();
src.put("A", "NA");
src.put("B", "NA");