From 582df287cd55611ebf3e218048af25fc5f73c262 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Thu, 25 Aug 2011 11:16:44 +0000 Subject: [PATCH] SOLR-2668 -- DIH multithreaded mode does not rollback on errors from EntityProcessor git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1161505 13f79535-47bb-0310-9956-ffa450edef68 --- solr/contrib/dataimporthandler/CHANGES.txt | 1 + .../solr/handler/dataimport/DocBuilder.java | 2 +- .../dataimport/TestDocBuilderThreaded.java | 133 +++++++++++++++++- 3 files changed, 128 insertions(+), 8 deletions(-) diff --git a/solr/contrib/dataimporthandler/CHANGES.txt b/solr/contrib/dataimporthandler/CHANGES.txt index d2fb2161761..ddb8fbc4a91 100644 --- a/solr/contrib/dataimporthandler/CHANGES.txt +++ b/solr/contrib/dataimporthandler/CHANGES.txt @@ -21,6 +21,7 @@ Bug Fixes * SOLR-2186: DataImportHandler's multi-threaded option throws NPE (Lance Norskog, Frank Wesemann, shalin) * SOLR-2655: DIH multi threaded mode does not resolve attributes correctly (Frank Wesemann, shalin) * SOLR-2695: Documents are collected in unsynchronized list in multi-threaded debug mode (Michael McCandless, shalin) +* SOLR-2668: DIH multithreaded mode does not rollback on errors from EntityProcessor (Frank Wesemann, shalin) ================== 3.3.0 ================== diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java index dd112b87b95..77f41c850e6 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java @@ -305,7 +305,7 @@ public class DocBuilder { LOG.info("running multithreaded full-import"); new EntityRunner(root,null).run(null,Context.FULL_DUMP,null); } catch (Exception e) { - LOG.error("error in import", e); + throw new RuntimeException("Error in multi-threaded import", e); } } else { buildDocument(getVariableResolver(), null, null, root, true, null); diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java index 0e340c16deb..ca3a7ba7879 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilderThreaded.java @@ -16,15 +16,15 @@ */ package org.apache.solr.handler.dataimport; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - /** * Test DocBuilder with "threads" @@ -60,6 +60,8 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { DemoProcessor.entitiesInitied = 0; DemoEvaluator.evaluated = 0; MockDataSource.clearCache(); + assertU(delQ("*:*")); + assertU(commit()); super.tearDown(); } @@ -86,6 +88,23 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { assertEquals("Evaluator was invoked less times than the number of rows", 4, DemoEvaluator.evaluated); } + @Test + public void testContinue() throws Exception { + runFullImport(twoEntitiesWithFailingProcessor); + assertQ(req("*:*"), "//*[@numFound='0']"); // should rollback + } + + @Test + public void testContinueThreaded() throws Exception { + runFullImport(twoThreadedEntitiesWithFailingProcessor); + assertQ(req("*:*"), "//*[@numFound='0']"); // should rollback + } + + @Test + public void testFailingTransformerContinueThreaded() throws Exception { + runFullImport(twoThreadedEntitiesWithFailingTransformer); + assertQ(req("*:*"), "//*[@numFound='4']"); + } @SuppressWarnings("unchecked") private List> getDetails4Worker(String aWorker) { @@ -116,8 +135,7 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { "" + "" + ""; - - private final String twoEntitiesWithProcessor = + private final String twoEntitiesWithProcessor = " \n" + "" + @@ -138,7 +156,7 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { "" + "" + ""; - + private final String twoEntitiesWithEvaluatorProcessor = " \n" + @@ -164,6 +182,89 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { ""; + private final String twoThreadedEntitiesWithFailingProcessor = + + " \n" + + "" + + "" + + "\n" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "" + + ""; + + private final String twoEntitiesWithFailingProcessor = + + " \n" + + "" + + "" + + "\n" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "" + + ""; + + private final String twoThreadedEntitiesWithFailingTransformer = + + " \n" + + "" + + "" + + "\n" + + "" + + "" + + "" + + " " + + " " + + "" + + "" + + "" + + ""; + + public static class DemoProcessor extends SqlEntityProcessor { public static int entitiesInitied = 0; @@ -177,6 +278,23 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { } else entitiesInitied++; } } + public static class FailingProcessor extends SqlEntityProcessor { + @Override + public void init(Context context) { + super.init(context); + String fail = context.getResolvedEntityAttribute("fail"); + if (fail != null && fail.equalsIgnoreCase("yes")) { + throw new NullPointerException("I was told to"); + } + } + } + + public static class FailingTransformer extends Transformer { + @Override + public Object transformRow(Map row, Context context) { + throw new RuntimeException("Always fail"); + } + } public static class DemoEvaluator extends Evaluator { public static int evaluated = 0; @@ -196,4 +314,5 @@ public class TestDocBuilderThreaded extends AbstractDataImportHandlerTestCase { return result.toString(); } } + }