From 40a1e6da342cf26d186366783cc8ed5af2e33f42 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Mon, 13 Apr 2009 09:00:59 +0000 Subject: [PATCH] SOLR-1059 -- Fixing bug where skipping a row containing nested entities did not skip the nested entities. Handling special flag variables is in one method now. git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@764379 13f79535-47bb-0310-9956-ffa450edef68 --- .../solr/handler/dataimport/DataConfig.java | 6 +- .../DataImportHandlerException.java | 2 +- .../solr/handler/dataimport/DocBuilder.java | 72 ++++++++++++------- 3 files changed, 49 insertions(+), 31 deletions(-) diff --git a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java index 81be2cee9a2..d1af2b8b81c 100644 --- a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java +++ b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataConfig.java @@ -87,7 +87,7 @@ public class DataConfig { public boolean isDocRoot = false; - public List fields; + public List fields = new ArrayList(); public List> allFieldsList = new ArrayList>(); @@ -104,7 +104,7 @@ public class DataConfig { public Script script; - public Map> colNameVsField; + public Map> colNameVsField = new HashMap>(); public Entity() { } @@ -121,8 +121,6 @@ public class DataConfig { dataSource = getStringAttribute(element, DataImporter.DATA_SRC, null); allAttributes = getAllAttributes(element); List n = getChildNodes(element, "field"); - fields = new ArrayList(); - colNameVsField = new HashMap>(); for (Element elem : n) { Field field = new Field(elem); fields.add(field); diff --git a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java index 56a70b1478c..0ffeb2c25c6 100644 --- a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java +++ b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DataImportHandlerException.java @@ -31,7 +31,7 @@ public class DataImportHandlerException extends RuntimeException { public boolean debugged = false; - public static final int SEVERE = 500, WARN = 400, SKIP = 300; + public static final int SEVERE = 500, WARN = 400, SKIP = 300, SKIP_ROW =301; public DataImportHandlerException(int err) { super(); diff --git a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java index 3cde20c5072..c49c721c402 100644 --- a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java +++ b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java @@ -354,7 +354,8 @@ public class DocBuilder { writer.log(SolrWriter.ENTITY_OUT, entity.name, arow); } importStatistics.rowsCount.incrementAndGet(); - if (entity.fields != null && doc != null) { + if (doc != null) { + handleSpecialCommands(arow); addFields(entity, doc, arow); } if (isRoot) @@ -382,6 +383,9 @@ public class DocBuilder { if (verboseDebug) { writer.log(SolrWriter.ENTITY_EXCEPTION, entity.name, e); } + if(e.getErrCode() == DataImportHandlerException.SKIP_ROW){ + continue; + } if (isRoot) { if (e.getErrCode() == DataImportHandlerException.SKIP) { importStatistics.skipDocCount.getAndIncrement(); @@ -424,37 +428,53 @@ public class DocBuilder { doc.setDocumentBoost(value); } + private void handleSpecialCommands(Map arow) { + Object value = arow.get("$deleteDocById"); + if (value != null) { + if (value instanceof Collection) { + Collection collection = (Collection) value; + for (Object o : collection) { + writer.deleteDoc(o.toString()); + } + } else { + writer.deleteDoc(value); + } + } + value = arow.get("$deleteDocByQuery"); + if (value != null) { + if (value instanceof Collection) { + Collection collection = (Collection) value; + for (Object o : collection) { + writer.deleteByQuery(o.toString()); + } + + } else { + writer.deleteByQuery(value.toString()); + } + } + value = arow.get("$skipDoc"); + if (value != null) { + if (Boolean.parseBoolean(value.toString())) { + throw new DataImportHandlerException(DataImportHandlerException.SKIP, + "Document skipped :" + arow); + } + } + + value = arow.get("$skipRow"); + if (value != null) { + if (Boolean.parseBoolean(value.toString())) { + throw new DataImportHandlerException(DataImportHandlerException.SKIP_ROW); + } + } + } + @SuppressWarnings("unchecked") private void addFields(DataConfig.Entity entity, SolrInputDocument doc, Map arow) { - Object s = arow.get("$skipRow"); - if (s != null && Boolean.parseBoolean(s.toString())) { - return; - } for (Map.Entry entry : arow.entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); if (value == null) continue; - if (key.startsWith("$")) { - if ("$deleteDocById".equals(key)) { - if (value instanceof Collection) { - Collection collection = (Collection) value; - for (Object o : collection) { - writer.deleteDoc(o.toString()); - } - } else { - writer.deleteDoc(value); - } - } - if ("$deleteDocByQuery".equals(key)) { - writer.deleteByQuery(entry.getValue().toString()); - } - if ("$skipDoc".equals(key) && Boolean.parseBoolean(value.toString())) { - throw new DataImportHandlerException(DataImportHandlerException.SKIP, - "Document skipped :" + arow); - } - // All fields starting with $ are special values and don't need to be added - continue; - } + if (key.startsWith("$")) continue; List field = entity.colNameVsField.get(key); if (field == null && dataImporter.getSchema() != null) { // This can be a dynamic field or a field which does not have an entry in data-config ( an implicit field)