From 0291a24619180568df0c468a06f74560c41a4a30 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Fri, 15 Aug 2008 08:49:35 +0000 Subject: [PATCH] SOLR-700 -- Allow configurable locales through a locale attribute in fields for NumberFormatTransformer. git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@686176 13f79535-47bb-0310-9956-ffa450edef68 --- contrib/dataimporthandler/CHANGES.txt | 34 +++++++++++++++++++ .../dataimport/NumberFormatTransformer.java | 33 +++++++++++++----- .../TestNumberFormatTransformer.java | 14 ++++++-- 3 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 contrib/dataimporthandler/CHANGES.txt diff --git a/contrib/dataimporthandler/CHANGES.txt b/contrib/dataimporthandler/CHANGES.txt new file mode 100644 index 00000000000..8dea342ef84 --- /dev/null +++ b/contrib/dataimporthandler/CHANGES.txt @@ -0,0 +1,34 @@ + Apache Solr - DataImportHandler Version 1.3-dev + Release Notes + +Introduction +------------ +DataImportHandler is a data import tool for Solr which makes importing data from Databases, XML files and +HTTP data sources quick and easy. + + +$Id$ + +================== Release 1.3-dev ================== + +Status +------ +This is the first release since DataImportHandler was added to the contrib solr distribution. +The following changes list changes since the code was introduced, not since +the first official release. + + +Detailed Change List +-------------------- + +New Features +1. SOLR-700: Allow configurable locales through a locale attribute in fields for NumberFormatTransformer. + (Stefan Oestreicher, shalin) + +Changes in runtime behavior + +Bug Fixes + +Other Changes + + diff --git a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java index 9c00f9ce390..5b8606f1595 100644 --- a/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java +++ b/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/NumberFormatTransformer.java @@ -4,14 +4,17 @@ import java.text.NumberFormat; import java.text.ParseException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** *

* A Transformer instance which can extract numbers out of strings. It uses * java.text.NumberFormat class to parse strings and supports * Number, Integer, Currency and Percent styles as supported by - * java.text.NumberFormat + * java.text.NumberFormat with configurable locales. *

*

*

@@ -27,6 +30,8 @@ import java.util.Map; */ public class NumberFormatTransformer extends Transformer { + private static final Pattern localeRegex = Pattern.compile("^([a-z]{2})-([A-Z]{2})$"); + @SuppressWarnings("unchecked") public Object transformRow(Map row, Context context) { for (Map fld : context.getAllEntityFields()) { @@ -34,8 +39,20 @@ public class NumberFormatTransformer extends Transformer { if (style != null) { String column = fld.get(DataImporter.COLUMN); String srcCol = fld.get(RegexTransformer.SRC_COL_NAME); + Locale locale = null; + String localeStr = fld.get(LOCALE); if (srcCol == null) srcCol = column; + if (localeStr != null) { + Matcher matcher = localeRegex.matcher(localeStr); + if (matcher.find() && matcher.groupCount() == 2) { + locale = new Locale(matcher.group(1), matcher.group(2)); + } else { + throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Invalid Locale specified for field: " + fld); + } + } else { + locale = Locale.getDefault(); + } Object val = row.get(srcCol); String styleSmall = style.toLowerCase(); @@ -45,7 +62,7 @@ public class NumberFormatTransformer extends Transformer { List results = new ArrayList(); for (String input : inputs) { try { - results.add(process(input, styleSmall)); + results.add(process(input, styleSmall, locale)); } catch (ParseException e) { throw new DataImportHandlerException( DataImportHandlerException.SEVERE, @@ -57,7 +74,7 @@ public class NumberFormatTransformer extends Transformer { if (val == null || val.toString().trim().equals("")) continue; try { - row.put(column, process(val.toString(), styleSmall)); + row.put(column, process(val.toString(), styleSmall, locale)); } catch (ParseException e) { throw new DataImportHandlerException( DataImportHandlerException.SEVERE, @@ -69,15 +86,15 @@ public class NumberFormatTransformer extends Transformer { return row; } - private Number process(String val, String style) throws ParseException { + private Number process(String val, String style, Locale locale) throws ParseException { if (INTEGER.equals(style)) { - return NumberFormat.getIntegerInstance().parse(val); + return NumberFormat.getIntegerInstance(locale).parse(val); } else if (NUMBER.equals(style)) { - return NumberFormat.getNumberInstance().parse(val); + return NumberFormat.getNumberInstance(locale).parse(val); } else if (CURRENCY.equals(style)) { - return NumberFormat.getCurrencyInstance().parse(val); + return NumberFormat.getCurrencyInstance(locale).parse(val); } else if (PERCENT.equals(style)) { - return NumberFormat.getPercentInstance().parse(val); + return NumberFormat.getPercentInstance(locale).parse(val); } return null; diff --git a/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java b/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java index 41178be1481..55562859720 100644 --- a/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java +++ b/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestNumberFormatTransformer.java @@ -19,8 +19,10 @@ package org.apache.solr.handler.dataimport; import org.junit.Assert; import org.junit.Test; +import java.text.DecimalFormatSymbols; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; /** @@ -32,17 +34,23 @@ import java.util.Map; * @since solr 1.3 */ public class TestNumberFormatTransformer { + private char GROUPING_SEP = new DecimalFormatSymbols().getGroupingSeparator(); + @Test @SuppressWarnings("unchecked") public void testTransformRow_SingleNumber() { + char GERMAN_GROUPING_SEP = new DecimalFormatSymbols(Locale.GERMANY).getGroupingSeparator(); List l = new ArrayList(); l.add(AbstractDataImportHandlerTest.createMap("column", "num", NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER)); + l.add(AbstractDataImportHandlerTest.createMap("column", "localizedNum", + NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER, NumberFormatTransformer.LOCALE, "de-DE")); Context c = AbstractDataImportHandlerTest.getContext(null, null, null, 0, l, null); - Map m = AbstractDataImportHandlerTest.createMap("num", "123,567"); + Map m = AbstractDataImportHandlerTest.createMap("num", "123" + GROUPING_SEP + "567", "localizedNum", "123" + GERMAN_GROUPING_SEP + "567"); new NumberFormatTransformer().transformRow(m, c); Assert.assertEquals(new Long(123567), m.get("num")); + Assert.assertEquals(new Long(123567), m.get("localizedNum")); } @Test @@ -56,8 +64,8 @@ public class TestNumberFormatTransformer { NumberFormatTransformer.FORMAT_STYLE, NumberFormatTransformer.NUMBER)); List inputs = new ArrayList(); - inputs.add("123,567"); - inputs.add("245,678"); + inputs.add("123" + GROUPING_SEP + "567"); + inputs.add("245" + GROUPING_SEP + "678"); Map row = AbstractDataImportHandlerTest.createMap("inputs", inputs); VariableResolverImpl resolver = new VariableResolverImpl();