diff --git a/dev-tools/idea/.idea/libraries/HSQLDB.xml b/dev-tools/idea/.idea/libraries/HSQLDB.xml index 4eb35d50715..39efcbfc489 100644 --- a/dev-tools/idea/.idea/libraries/HSQLDB.xml +++ b/dev-tools/idea/.idea/libraries/HSQLDB.xml @@ -1,7 +1,7 @@ - + diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties index 3a1efbb665b..7e0e7c72f58 100644 --- a/lucene/ivy-versions.properties +++ b/lucene/ivy-versions.properties @@ -53,7 +53,6 @@ com.sun.jersey.version = 1.9 /commons-logging/commons-logging = 1.1.3 /de.l3s.boilerpipe/boilerpipe = 1.1.0 /dom4j/dom4j = 1.6.1 -/hsqldb/hsqldb = 1.8.0.10 /info.ganglia.gmetric4j/gmetric4j = 1.0.7 io.dropwizard.metrics.version = 3.2.2 @@ -242,6 +241,8 @@ org.gagravarr.vorbis.java.version = 0.8 /org.gagravarr/vorbis-java-core = ${org.gagravarr.vorbis.java.version} /org.gagravarr/vorbis-java-tika = ${org.gagravarr.vorbis.java.version} +/org.hsqldb/hsqldb = 2.4.0 + /org.jsoup/jsoup = 1.8.2 /org.locationtech.spatial4j/spatial4j = 0.6 diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 5f6f4829271..10edc525973 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -158,9 +158,14 @@ Apache UIMA 2.3.1 Apache ZooKeeper 3.4.10 Jetty 9.3.14.v20161028 +Detailed Change List +---------------------- -(No Changes) +Other Changes +---------------------- +* SOLR-10617: JDBCStream accepts columns of type TIME, DATE & TIMESTAMP as well as CLOBs and decimal + numeric types (James Dyer) ================== 6.6.0 ================== diff --git a/solr/contrib/dataimporthandler/ivy.xml b/solr/contrib/dataimporthandler/ivy.xml index ea138dd69a0..67af77b03a4 100644 --- a/solr/contrib/dataimporthandler/ivy.xml +++ b/solr/contrib/dataimporthandler/ivy.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java index 0f53b939338..8ee6878c0ee 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java @@ -73,14 +73,14 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase { "select " + " 1 as id, " + " 'SELECT' as SELECT_KEYWORD, " + - " CURRENT_TIMESTAMP as FIRST_TS " + + " {ts '2017-02-18 12:34:56'} as FIRST_TS " + "from DUAL \" >\n"); sb.append(" \n"); sb.append(" \n"); sb.append(" \n"); String config = sb.toString(); - log.debug(config); + log.info(config); return config; } @Override diff --git a/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java b/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java new file mode 100644 index 00000000000..3738f6223f5 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler; + +import java.io.IOException; +import java.sql.Array; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Properties; + +import org.apache.solr.client.solrj.io.comp.StreamComparator; +import org.apache.solr.client.solrj.io.stream.JDBCStream; + +/** + * Used with o.a.s.Handler.SQLHandler. + * + * @lucene.internal + */ +public class CalciteJDBCStream extends JDBCStream { + private static final long serialVersionUID = 1L; + + public CalciteJDBCStream(String connectionUrl, String sqlQuery, StreamComparator definedSort, + Properties connectionProperties, String driverClassName) throws IOException { + super(connectionUrl, sqlQuery, definedSort, connectionProperties, driverClassName); + } + + @Override + protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData metadata) + throws SQLException { + ResultSetValueSelector valueSelector = super.determineValueSelector(columnIdx, metadata); + if (valueSelector == null) { + final int columnNumber = columnIdx + 1; + final String columnName = metadata.getColumnLabel(columnNumber); + final String className = metadata.getColumnClassName(columnNumber); + if (Array.class.getName().equals(className)) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Object o = resultSet.getObject(columnNumber); + if (resultSet.wasNull()) { + return null; + } + if (o instanceof Array) { + Array array = (Array) o; + return array.getArray(); + } else { + return o; + } + } + + @Override + public String getColumnName() { + return columnName; + } + }; + } + } + return valueSelector; + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java index c80d0d7743b..67ea1f69c7f 100644 --- a/solr/core/src/java/org/apache/solr/handler/SQLHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SQLHandler.java @@ -31,7 +31,6 @@ import org.apache.calcite.config.Lex; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; import org.apache.solr.client.solrj.io.stream.ExceptionStream; -import org.apache.solr.client.solrj.io.stream.JDBCStream; import org.apache.solr.client.solrj.io.stream.TupleStream; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -141,7 +140,7 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per /* * Only necessary for SolrJ JDBC driver since metadata has to be passed back */ - private static class SqlHandlerStream extends JDBCStream { + private static class SqlHandlerStream extends CalciteJDBCStream { private final boolean includeMetadata; private boolean firstTuple = true; List metadataFields = new ArrayList<>(); diff --git a/solr/example/example-DIH/ivy.xml b/solr/example/example-DIH/ivy.xml index 4060e3766bc..3f67a79574e 100644 --- a/solr/example/example-DIH/ivy.xml +++ b/solr/example/example-DIH/ivy.xml @@ -22,7 +22,7 @@ - + diff --git a/solr/licenses/hsqldb-1.8.0.10.jar.sha1 b/solr/licenses/hsqldb-1.8.0.10.jar.sha1 deleted file mode 100644 index 8661e6995be..00000000000 --- a/solr/licenses/hsqldb-1.8.0.10.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e9978fdb754bce5fcd5161133e7734ecb683036 diff --git a/solr/licenses/hsqldb-2.4.0.jar.sha1 b/solr/licenses/hsqldb-2.4.0.jar.sha1 new file mode 100644 index 00000000000..72520ee928a --- /dev/null +++ b/solr/licenses/hsqldb-2.4.0.jar.sha1 @@ -0,0 +1 @@ +195957160ed990dbc798207c0d577280d9919208 \ No newline at end of file diff --git a/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt b/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt index 953bfa0a295..b8ac4f54daf 100644 --- a/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt +++ b/solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt @@ -1,4 +1,4 @@ -/* Copyright (c) 2001-2005, The HSQL Development Group +/* Copyright (c) 2001-2017, The HSQL Development Group * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -18,9 +18,9 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, - * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT @@ -28,4 +28,3 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ - diff --git a/solr/licenses/hsqldb-NOTICE.txt b/solr/licenses/hsqldb-NOTICE.txt index 0929e9b8fe7..829f8a24e3e 100644 --- a/solr/licenses/hsqldb-NOTICE.txt +++ b/solr/licenses/hsqldb-NOTICE.txt @@ -1,68 +1,69 @@ -========================================================================= -== HSQLDB Notice == -========================================================================= +/* + * For work developed by the HSQL Development Group: + * + * Copyright (c) 2001-2017, The HSQL Development Group + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of the HSQL Development Group nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * + * + * For work originally developed by the Hypersonic SQL Group: + * + * Copyright (c) 1995-2000, The Hypersonic SQL Group. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * Neither the name of the Hypersonic SQL Group nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, + * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * This software consists of voluntary contributions made by many individuals + * on behalf of the Hypersonic SQL Group. + */ -For content, code, and products originally developed by Thomas Mueller and the Hypersonic SQL Group: - -Copyright (c) 1995-2000 by the Hypersonic SQL Group. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the Hypersonic SQL Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -This software consists of voluntary contributions made by many individuals on behalf of the -Hypersonic SQL Group. - -For work added by the HSQL Development Group (a.k.a. hsqldb_lic.txt): - -Copyright (c) 2001-2005, The HSQL Development Group -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the HSQL Development Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java index 172b9ef7994..99b090f8662 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java @@ -17,13 +17,19 @@ package org.apache.solr.client.solrj.io.stream; import java.io.IOException; -import java.sql.Array; +import java.math.BigDecimal; +import java.sql.Clob; import java.sql.Connection; +import java.sql.Date; import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -51,16 +57,64 @@ import static org.apache.solr.common.params.CommonParams.SORT; * Connects to a datasource using a registered JDBC driver and execute a query. The results of * that query will be returned as tuples. An EOF tuple will indicate that all have been read. * - * Supported Datatypes - * JDBC Type | Tuple Type - * --------------|--------------- - * String | String - * Short | Long - * Integer | Long - * Long | Long - * Float | Double - * Double | Double - * Boolean | Boolean + * Supported Datatypes for most types vary by JDBC driver based on the specific + * java type as reported by {@link java.sql.ResultSetMetaData#getColumnClassName(int)}. + * The exception are {@link Types#DATE}, {@link Types#TIME} or {@link Types#TIMESTAMP} + * which are determined by the JDBC type. + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Java or JDBC TypeTuple TypeNotes
BooleanBoolean
StringString
Short, Integer, LongLong
Float, DoubleDouble
{@link Clob} and subclassesStringClobs up to length 231-1 are supported.
Other subclasses of {@link Number}Long, DoubleTuple Type based on {@link BigDecimal#scale()}.
JDBC {@link Types#DATE}Stringyyyy-MM-dd, calls {@link Date#toString}
JDBC {@link Types#TIME}Stringhh:mm:ss, calls {@link Time#toString}
JDBC {@link Types#TIMESTAMP}StringSee {@link DateTimeFormatter#ISO_INSTANT}
+ * **/ public class JDBCStream extends TupleStream implements Expressible { @@ -227,93 +281,202 @@ public class JDBCStream extends TupleStream implements Expressible { } private ResultSetValueSelector[] constructValueSelectors(ResultSetMetaData metadata) throws SQLException{ - ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()]; - - for(int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx){ - final int columnNumber = columnIdx + 1; // cause it starts at 1 - // Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as picked up properly - final String columnName = metadata.getColumnLabel(columnNumber); - String className = metadata.getColumnClassName(columnNumber); - String typeName = metadata.getColumnTypeName(columnNumber); - - if(directSupportedTypes.contains(className)){ - valueSelectors[columnIdx] = new ResultSetValueSelector() { - public Object selectValue(ResultSet resultSet) throws SQLException { - Object obj = resultSet.getObject(columnNumber); - if(resultSet.wasNull()){ return null; } - if(obj instanceof String) { - String s = (String)obj; - if(s.indexOf(sep) > -1) { - s = s.substring(1); - return s.split(sep); - } - } - - return obj; - } - public String getColumnName() { - return columnName; - } - }; - } else if(Short.class.getName().equals(className)) { - valueSelectors[columnIdx] = new ResultSetValueSelector() { - public Object selectValue(ResultSet resultSet) throws SQLException { - Short obj = resultSet.getShort(columnNumber); - if(resultSet.wasNull()){ return null; } - return obj.longValue(); - } - public String getColumnName() { - return columnName; - } - }; - } else if(Integer.class.getName().equals(className)) { - valueSelectors[columnIdx] = new ResultSetValueSelector() { - public Object selectValue(ResultSet resultSet) throws SQLException { - Integer obj = resultSet.getInt(columnNumber); - if(resultSet.wasNull()){ return null; } - return obj.longValue(); - } - public String getColumnName() { - return columnName; - } - }; - } else if(Float.class.getName().equals(className)) { - valueSelectors[columnIdx] = new ResultSetValueSelector() { - public Object selectValue(ResultSet resultSet) throws SQLException { - Float obj = resultSet.getFloat(columnNumber); - if(resultSet.wasNull()){ return null; } - return obj.doubleValue(); - } - public String getColumnName() { - return columnName; - } - }; - } else if(Array.class.getName().equals(className)) { - valueSelectors[columnIdx] = new ResultSetValueSelector() { - public Object selectValue(ResultSet resultSet) throws SQLException { - Object o = resultSet.getObject(columnNumber); - if(resultSet.wasNull()){ return null; } - if(o instanceof Array) { - Array array = (Array)o; - return array.getArray(); - } else { - return o; - } - } - public String getColumnName() { - return columnName; - } - }; - } else { + ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()]; + for (int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx) { + ResultSetValueSelector valueSelector = determineValueSelector(columnIdx, metadata); + if(valueSelector==null) { + int columnNumber = columnIdx + 1; + String columnName = metadata.getColumnLabel(columnNumber); + String className = metadata.getColumnClassName(columnNumber); + String typeName = metadata.getColumnTypeName(columnNumber); throw new SQLException(String.format(Locale.ROOT, "Unable to determine the valueSelector for column '%s' (col #%d) of java class '%s' and type '%s'", columnName, columnNumber, className, typeName)); } - } - + valueSelectors[columnIdx] = valueSelector; + } return valueSelectors; } + protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData metadata) throws SQLException { + final int columnNumber = columnIdx + 1; // cause it starts at 1 + // Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as picked up properly + final String columnName = metadata.getColumnLabel(columnNumber); + final int jdbcType = metadata.getColumnType(columnNumber); + final String className = metadata.getColumnClassName(columnNumber); + ResultSetValueSelector valueSelector = null; + + // Directly supported types can be just directly returned - no conversion really necessary + if(directSupportedTypes.contains(className)){ + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Object obj = resultSet.getObject(columnNumber); + if(resultSet.wasNull()){ return null; } + if(obj instanceof String) { + String s = (String)obj; + if(s.indexOf(sep) > -1) { + s = s.substring(1); + return s.split(sep); + } + } + + return obj; + } + @Override + public String getColumnName() { + return columnName; + } + }; + } + // We're checking the Java class names because there are lots of SQL types across + // lots of database drivers that can be mapped to standard Java types. Basically, + // this makes it easier and we don't have to worry about esoteric type names in the + // JDBC family of types + else if(Short.class.getName().equals(className)) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Short obj = resultSet.getShort(columnNumber); + if(resultSet.wasNull()){ return null; } + return obj.longValue(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } else if(Integer.class.getName().equals(className)) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Integer obj = resultSet.getInt(columnNumber); + if(resultSet.wasNull()){ return null; } + return obj.longValue(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } else if(Float.class.getName().equals(className)) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Float obj = resultSet.getFloat(columnNumber); + if(resultSet.wasNull()){ return null; } + return obj.doubleValue(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } + // Here we are switching to check against the SQL type because date/times are + // notorious for not being consistent. We don't know if the driver is mapping + // to a java.time.* type or some old-school type. + else if (jdbcType == Types.DATE) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Date sqlDate = resultSet.getDate(columnNumber); + return resultSet.wasNull() ? null : sqlDate.toString(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } else if (jdbcType == Types.TIME ) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Time sqlTime = resultSet.getTime(columnNumber); + return resultSet.wasNull() ? null : sqlTime.toString(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } else if (jdbcType == Types.TIMESTAMP) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Timestamp sqlTimestamp = resultSet.getTimestamp(columnNumber); + return resultSet.wasNull() ? null : sqlTimestamp.toInstant().toString(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } + // Now we're going to start seeing if things are assignable from the returned type + // to a more general type - this allows us to cover cases where something we weren't + // explicitly expecting, but can handle, is being returned. + else { + Class clazz; + try { + clazz = Class.forName(className, false, getClass().getClassLoader()); + } catch (Exception e) { + throw new RuntimeException(e); + } + final int scale = metadata.getScale(columnNumber); + if (Number.class.isAssignableFrom(clazz)) { + if (scale > 0) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + BigDecimal bd = resultSet.getBigDecimal(columnNumber); + return resultSet.wasNull() ? null : bd.doubleValue(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } else { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + BigDecimal bd = resultSet.getBigDecimal(columnNumber); + return resultSet.wasNull() ? null : bd.longValue(); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } + } else if (Clob.class.isAssignableFrom(clazz)) { + valueSelector = new ResultSetValueSelector() { + @Override + public Object selectValue(ResultSet resultSet) throws SQLException { + Clob c = resultSet.getClob(columnNumber); + if (resultSet.wasNull()) { + return null; + } + long length = c.length(); + int lengthInt = (int) length; + if (length != lengthInt) { + throw new SQLException(String.format(Locale.ROOT, + "Encountered a clob of length #%l in column '%s' (col #%d). Max supported length is #%i.", + length, columnName, columnNumber, Integer.MAX_VALUE)); + } + return c.getSubString(1, lengthInt); + } + @Override + public String getColumnName() { + return columnName; + } + }; + } + } + return valueSelector; + } + /** * Closes the JDBCStream **/ @@ -432,9 +595,10 @@ public class JDBCStream extends TupleStream implements Expressible { // it's already in the sqlQuery but there's no way we can reliably determine the sort from the query. return definedSort; } + + public interface ResultSetValueSelector { + String getColumnName(); + Object selectValue(ResultSet resultSet) throws SQLException; + } } -interface ResultSetValueSelector { - String getColumnName(); - Object selectValue(ResultSet resultSet) throws SQLException; -} \ No newline at end of file diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java index 9fff33a083a..8813f24ba28 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java @@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.stream; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; @@ -96,6 +97,8 @@ public class JDBCStreamTest extends SolrCloudTestCase { statement.executeUpdate("create table PEOPLE(ID int not null primary key, NAME varchar(50), COUNTRY_CODE char(2), DELETED char(1) default 'N')"); statement.executeUpdate("create table PEOPLE_SPORTS(ID int not null primary key, PERSON_ID int, SPORT_NAME varchar(50), DELETED char(1) default 'N')"); statement.executeUpdate("create table UNSUPPORTED_COLUMNS(ID int not null primary key, UNSP binary)"); + statement.executeUpdate("create table DUAL(ID int not null primary key)"); + statement.executeUpdate("insert into DUAL values(1)"); } @@ -158,8 +161,29 @@ public class JDBCStreamTest extends SolrCloudTestCase { assertOrderOf(tuples, "CODE", "NP", "NL", "NO", "US"); assertOrderOf(tuples, "COUNTRY_NAME", "Nepal", "Netherlands", "Norway", "United States"); + // Additional Types + String query = "select 1 as ID1, {ts '2017-02-18 12:34:56.789'} as TS1, {t '01:02:03'} as T1, " + + "{d '1593-03-14'} as D1, cast(12.34 AS DECIMAL(4,2)) as DEC4_2, " + + "cast(1234 AS DECIMAL(4,0)) as DEC4_0, cast('big stuff' as CLOB(100)) as CLOB1 " + + "from DUAL order by ID1"; + stream = new JDBCStream("jdbc:hsqldb:mem:.", query, new FieldComparator("ID1", ComparatorOrder.ASCENDING)); + tuples = getTuples(stream); + assertEquals(1, tuples.size()); + Tuple t; + try (Connection connection = DriverManager.getConnection("jdbc:hsqldb:mem:."); + Statement statement = connection.createStatement()) { + ResultSet rs = statement.executeQuery(query); + rs.next(); + t = tuples.iterator().next(); + assertString(t, "CLOB1", rs.getString("CLOB1")); + assertString(t, "TS1", rs.getTimestamp("TS1").toInstant().toString()); + assertString(t, "T1", rs.getTime("T1").toString()); + assertString(t, "D1", rs.getDate("D1").toString()); + assertDouble(t, "DEC4_2", rs.getDouble("DEC4_2")); + assertLong(t, "DEC4_0", rs.getLong("DEC4_0")); + } } - + @Test public void testJDBCJoin() throws Exception { @@ -643,6 +667,14 @@ public class JDBCStreamTest extends SolrCloudTestCase { return true; } + public boolean assertDouble(Tuple tuple, String fieldName, double d) throws Exception { + double dv = (double)tuple.get(fieldName); + if(dv != d) { + throw new Exception("Doubles not equal:"+d+" : "+dv); + } + return true; + } + public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception { String actual = (String)tuple.get(fieldName);