Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-10617: JDBCStream to support additional types, minor refactoring to separate out CalciteJDBCStream, upgrade hsqldb for JDBCStream & DIH tests.

This commit is contained in:
jdyer1 2017-05-12 08:44:16 -05:00
parent 904df0eb61
commit e61b5b34bf
14 changed files with 456 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-tools/idea/.idea/libraries/HSQLDB.xml
View File

@ -1,7 +1,7 @@
<component name="libraryTable">
<library name="HSQLDB">
<CLASSES>
<root url="jar://$PROJECT_DIR$/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar!/" />
<root url="jar://$PROJECT_DIR$/solr/example/example-DIH/solr/db/lib/hsqldb-2.4.0.jar!/" />
</CLASSES>
<JAVADOC />
<SOURCES />

3
lucene/ivy-versions.properties
View File

@ -53,7 +53,6 @@ com.sun.jersey.version = 1.9
/commons-logging/commons-logging = 1.1.3
/de.l3s.boilerpipe/boilerpipe = 1.1.0
/dom4j/dom4j = 1.6.1
/hsqldb/hsqldb = 1.8.0.10
/info.ganglia.gmetric4j/gmetric4j = 1.0.7
io.dropwizard.metrics.version = 3.2.2
@ -242,6 +241,8 @@ org.gagravarr.vorbis.java.version = 0.8
/org.gagravarr/vorbis-java-core = ${org.gagravarr.vorbis.java.version}
/org.gagravarr/vorbis-java-tika = ${org.gagravarr.vorbis.java.version}
/org.hsqldb/hsqldb = 2.4.0
/org.jsoup/jsoup = 1.8.2
/org.locationtech.spatial4j/spatial4j = 0.6

7
solr/CHANGES.txt
View File

@ -158,9 +158,14 @@ Apache UIMA 2.3.1
Apache ZooKeeper 3.4.10
Jetty 9.3.14.v20161028
Detailed Change List
----------------------
(No Changes)
Other Changes
----------------------
* SOLR-10617: JDBCStream accepts columns of type TIME, DATE & TIMESTAMP as well as CLOBs and decimal
numeric types (James Dyer)
================== 6.6.0 ==================

2
solr/contrib/dataimporthandler/ivy.xml
View File

@ -23,7 +23,7 @@
<conf name="test" transitive="false"/>
</configurations>
<dependencies>
<dependency org="hsqldb" name="hsqldb" rev="${/hsqldb/hsqldb}" conf="test"/>
<dependency org="org.hsqldb" name="hsqldb" rev="${/org.hsqldb/hsqldb}" conf="test"/>
<dependency org="org.apache.derby" name="derby" rev="${/org.apache.derby/derby}" conf="test"/>
<dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}" conf="test"/>

8
solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
View File

@ -73,14 +73,14 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
"select " +
" 1 as id, " +
" 'SELECT' as SELECT_KEYWORD, " +
" CURRENT_TIMESTAMP as FIRST_TS " +
" {ts '2017-02-18 12:34:56'} as FIRST_TS " +
"from DUAL \" >\n");
sb.append(" <field column=\"SELECT_KEYWORD\" name=\"select_keyword_s\" /> \n");
sb.append(" <entity name=\"SECOND\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" transformer=\"TemplateTransformer\" ");
sb.append(" query=\"" +
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
" 1 as SORT, " +
" CURRENT_TIMESTAMP as SECOND_TS, " +
" {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
" 'PORK' AS MEAT, " +
" 'GRILL' AS METHOD, " +
@ -91,7 +91,7 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
"UNION " +
"${dataimporter.functions.encodeUrl(FIRST.SELECT_KEYWORD)} " +
" 2 as SORT, " +
" CURRENT_TIMESTAMP as SECOND_TS, " +
" {ts '2017-02-18 12:34:56'} as SECOND_TS, " +
" '${dataimporter.functions.formatDate(FIRST.FIRST_TS, 'yyyy'" + thirdLocaleParam + ")}' as SECOND1_S, " +
" 'FISH' AS MEAT, " +
" 'FRY' AS METHOD, " +
@ -112,7 +112,7 @@ public class TestVariableResolverEndToEnd extends AbstractDIHJdbcTestCase {
sb.append("</document> \n");
sb.append("</dataConfig> \n");
String config = sb.toString();
log.debug(config);
log.info(config);
return config;
}
@Override

75
solr/core/src/java/org/apache/solr/handler/CalciteJDBCStream.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import java.io.IOException;
import java.sql.Array;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.Properties;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.stream.JDBCStream;
/**
* Used with o.a.s.Handler.SQLHandler.
*
* @lucene.internal
*/
public class CalciteJDBCStream extends JDBCStream {
private static final long serialVersionUID = 1L;
public CalciteJDBCStream(String connectionUrl, String sqlQuery, StreamComparator definedSort,
Properties connectionProperties, String driverClassName) throws IOException {
super(connectionUrl, sqlQuery, definedSort, connectionProperties, driverClassName);
}
@Override
protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData metadata)
throws SQLException {
ResultSetValueSelector valueSelector = super.determineValueSelector(columnIdx, metadata);
if (valueSelector == null) {
final int columnNumber = columnIdx + 1;
final String columnName = metadata.getColumnLabel(columnNumber);
final String className = metadata.getColumnClassName(columnNumber);
if (Array.class.getName().equals(className)) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Object o = resultSet.getObject(columnNumber);
if (resultSet.wasNull()) {
return null;
}
if (o instanceof Array) {
Array array = (Array) o;
return array.getArray();
} else {
return o;
}
}
@Override
public String getColumnName() {
return columnName;
}
};
}
}
return valueSelector;
}
}

3
solr/core/src/java/org/apache/solr/handler/SQLHandler.java
View File

@ -31,7 +31,6 @@ import org.apache.calcite.config.Lex;
import org.apache.solr.client.solrj.io.Tuple;
import org.apache.solr.client.solrj.io.comp.StreamComparator;
import org.apache.solr.client.solrj.io.stream.ExceptionStream;
import org.apache.solr.client.solrj.io.stream.JDBCStream;
import org.apache.solr.client.solrj.io.stream.TupleStream;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
@ -141,7 +140,7 @@ public class SQLHandler extends RequestHandlerBase implements SolrCoreAware, Per
/*
* Only necessary for SolrJ JDBC driver since metadata has to be passed back
*/
private static class SqlHandlerStream extends JDBCStream {
private static class SqlHandlerStream extends CalciteJDBCStream {
private final boolean includeMetadata;
private boolean firstTuple = true;
List<String> metadataFields = new ArrayList<>();

2
solr/example/example-DIH/ivy.xml
View File

@ -22,7 +22,7 @@
<conf name="compile" transitive="false"/>
</configurations>
<dependencies>
<dependency org="hsqldb" name="hsqldb" rev="${/hsqldb/hsqldb}" conf="compile"/>
<dependency org="org.hsqldb" name="hsqldb" rev="${/org.hsqldb/hsqldb}" conf="compile"/>
<dependency org="org.apache.derby" name="derby" rev="${/org.apache.derby/derby}" conf="compile"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>

1
solr/licenses/hsqldb-1.8.0.10.jar.sha1
View File

@ -1 +0,0 @@
7e9978fdb754bce5fcd5161133e7734ecb683036

1
solr/licenses/hsqldb-2.4.0.jar.sha1 Normal file
View File

@ -0,0 +1 @@
195957160ed990dbc798207c0d577280d9919208

9
solr/licenses/hsqldb-LICENSE-BSD_LIKE.txt
View File

@ -1,4 +1,4 @@
/* Copyright (c) 2001-2005, The HSQL Development Group
/* Copyright (c) 2001-2017, The HSQL Development Group
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -18,9 +18,9 @@
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
@ -28,4 +28,3 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

135
solr/licenses/hsqldb-NOTICE.txt
View File

@ -1,68 +1,69 @@
=========================================================================
== HSQLDB Notice ==
=========================================================================
/*
* For work developed by the HSQL Development Group:
*
* Copyright (c) 2001-2017, The HSQL Development Group
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of the HSQL Development Group nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*
*
* For work originally developed by the Hypersonic SQL Group:
*
* Copyright (c) 1995-2000, The Hypersonic SQL Group.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* Neither the name of the Hypersonic SQL Group nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
* OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This software consists of voluntary contributions made by many individuals
* on behalf of the Hypersonic SQL Group.
*/
For content, code, and products originally developed by Thomas Mueller and the Hypersonic SQL Group:
Copyright (c) 1995-2000 by the Hypersonic SQL Group.
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
Neither the name of the Hypersonic SQL Group nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
This software consists of voluntary contributions made by many individuals on behalf of the
Hypersonic SQL Group.
For work added by the HSQL Development Group (a.k.a. hsqldb_lic.txt):
Copyright (c) 2001-2005, The HSQL Development Group
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
Neither the name of the HSQL Development Group nor the names of its
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

354
solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
View File

@ -17,13 +17,19 @@
package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
import java.sql.Array;
import java.math.BigDecimal;
import java.sql.Clob;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.sql.Types;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -51,16 +57,64 @@ import static org.apache.solr.common.params.CommonParams.SORT;
* Connects to a datasource using a registered JDBC driver and execute a query. The results of
* that query will be returned as tuples. An EOF tuple will indicate that all have been read.
*
* Supported Datatypes
* JDBC Type | Tuple Type
* --------------|---------------
* String | String
* Short | Long
* Integer | Long
* Long | Long
* Float | Double
* Double | Double
* Boolean | Boolean
* Supported Datatypes for most types vary by JDBC driver based on the specific
* java type as reported by {@link java.sql.ResultSetMetaData#getColumnClassName(int)}.
* The exception are {@link Types#DATE}, {@link Types#TIME} or {@link Types#TIMESTAMP}
* which are determined by the JDBC type.
*
* <table rules="all" frame="box" cellpadding="3" summary="Supported Java Types">
* <tr>
* <th>Java or JDBC Type</th>
* <th>Tuple Type</th>
* <th>Notes</th>
* </tr>
* <tr>
* <td>Boolean</td>
* <td>Boolean</td>
* <td></td>
* </tr>
* <tr>
* <td>String</td>
* <td>String</td>
* <td></td>
* </tr>
* <tr>
* <td>Short, Integer, Long</td>
* <td>Long</td>
* <td></td>
* </tr>
* <tr>
* <td>Float, Double</td>
* <td>Double</td>
* <td></td>
* </tr>
* <tr>
* <td>{@link Clob} and subclasses</td>
* <td>String</td>
* <td>Clobs up to length 2<sup>31</sup>-1 are supported.</td>
* </tr>
* <tr>
* <td>Other subclasses of {@link Number}</td>
* <td>Long, Double</td>
* <td>Tuple Type based on {@link BigDecimal#scale()}.</td>
* </tr>
* <tr>
* <td>JDBC {@link Types#DATE}</td>
* <td>String</td>
* <td>yyyy-MM-dd, calls {@link Date#toString}</td>
* </tr>
* <tr>
* <td>JDBC {@link Types#TIME}</td>
* <td>String</td>
* <td>hh:mm:ss, calls {@link Time#toString}</td>
* </tr>
* <tr>
* <td>JDBC {@link Types#TIMESTAMP}</td>
* <td>String</td>
* <td>See {@link DateTimeFormatter#ISO_INSTANT}</td>
* </tr>
* </table>
*
**/
public class JDBCStream extends TupleStream implements Expressible {
@ -227,93 +281,202 @@ public class JDBCStream extends TupleStream implements Expressible {
}
private ResultSetValueSelector[] constructValueSelectors(ResultSetMetaData metadata) throws SQLException{
ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()];
for(int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx){
final int columnNumber = columnIdx + 1; // cause it starts at 1
// Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as picked up properly
final String columnName = metadata.getColumnLabel(columnNumber);
String className = metadata.getColumnClassName(columnNumber);
String typeName = metadata.getColumnTypeName(columnNumber);
if(directSupportedTypes.contains(className)){
valueSelectors[columnIdx] = new ResultSetValueSelector() {
public Object selectValue(ResultSet resultSet) throws SQLException {
Object obj = resultSet.getObject(columnNumber);
if(resultSet.wasNull()){ return null; }
if(obj instanceof String) {
String s = (String)obj;
if(s.indexOf(sep) > -1) {
s = s.substring(1);
return s.split(sep);
}
}
return obj;
}
public String getColumnName() {
return columnName;
}
};
} else if(Short.class.getName().equals(className)) {
valueSelectors[columnIdx] = new ResultSetValueSelector() {
public Object selectValue(ResultSet resultSet) throws SQLException {
Short obj = resultSet.getShort(columnNumber);
if(resultSet.wasNull()){ return null; }
return obj.longValue();
}
public String getColumnName() {
return columnName;
}
};
} else if(Integer.class.getName().equals(className)) {
valueSelectors[columnIdx] = new ResultSetValueSelector() {
public Object selectValue(ResultSet resultSet) throws SQLException {
Integer obj = resultSet.getInt(columnNumber);
if(resultSet.wasNull()){ return null; }
return obj.longValue();
}
public String getColumnName() {
return columnName;
}
};
} else if(Float.class.getName().equals(className)) {
valueSelectors[columnIdx] = new ResultSetValueSelector() {
public Object selectValue(ResultSet resultSet) throws SQLException {
Float obj = resultSet.getFloat(columnNumber);
if(resultSet.wasNull()){ return null; }
return obj.doubleValue();
}
public String getColumnName() {
return columnName;
}
};
} else if(Array.class.getName().equals(className)) {
valueSelectors[columnIdx] = new ResultSetValueSelector() {
public Object selectValue(ResultSet resultSet) throws SQLException {
Object o = resultSet.getObject(columnNumber);
if(resultSet.wasNull()){ return null; }
if(o instanceof Array) {
Array array = (Array)o;
return array.getArray();
} else {
return o;
}
}
public String getColumnName() {
return columnName;
}
};
} else {
ResultSetValueSelector[] valueSelectors = new ResultSetValueSelector[metadata.getColumnCount()];
for (int columnIdx = 0; columnIdx < metadata.getColumnCount(); ++columnIdx) {
ResultSetValueSelector valueSelector = determineValueSelector(columnIdx, metadata);
if(valueSelector==null) {
int columnNumber = columnIdx + 1;
String columnName = metadata.getColumnLabel(columnNumber);
String className = metadata.getColumnClassName(columnNumber);
String typeName = metadata.getColumnTypeName(columnNumber);
throw new SQLException(String.format(Locale.ROOT,
"Unable to determine the valueSelector for column '%s' (col #%d) of java class '%s' and type '%s'",
columnName, columnNumber, className, typeName));
}
}
valueSelectors[columnIdx] = valueSelector;
}
return valueSelectors;
}
protected ResultSetValueSelector determineValueSelector(int columnIdx, ResultSetMetaData metadata) throws SQLException {
final int columnNumber = columnIdx + 1; // cause it starts at 1
// Use getColumnLabel instead of getColumnName to make sure fields renamed with AS as picked up properly
final String columnName = metadata.getColumnLabel(columnNumber);
final int jdbcType = metadata.getColumnType(columnNumber);
final String className = metadata.getColumnClassName(columnNumber);
ResultSetValueSelector valueSelector = null;
// Directly supported types can be just directly returned - no conversion really necessary
if(directSupportedTypes.contains(className)){
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Object obj = resultSet.getObject(columnNumber);
if(resultSet.wasNull()){ return null; }
if(obj instanceof String) {
String s = (String)obj;
if(s.indexOf(sep) > -1) {
s = s.substring(1);
return s.split(sep);
}
}
return obj;
}
@Override
public String getColumnName() {
return columnName;
}
};
}
// We're checking the Java class names because there are lots of SQL types across
// lots of database drivers that can be mapped to standard Java types. Basically,
// this makes it easier and we don't have to worry about esoteric type names in the
// JDBC family of types
else if(Short.class.getName().equals(className)) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Short obj = resultSet.getShort(columnNumber);
if(resultSet.wasNull()){ return null; }
return obj.longValue();
}
@Override
public String getColumnName() {
return columnName;
}
};
} else if(Integer.class.getName().equals(className)) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Integer obj = resultSet.getInt(columnNumber);
if(resultSet.wasNull()){ return null; }
return obj.longValue();
}
@Override
public String getColumnName() {
return columnName;
}
};
} else if(Float.class.getName().equals(className)) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Float obj = resultSet.getFloat(columnNumber);
if(resultSet.wasNull()){ return null; }
return obj.doubleValue();
}
@Override
public String getColumnName() {
return columnName;
}
};
}
// Here we are switching to check against the SQL type because date/times are
// notorious for not being consistent. We don't know if the driver is mapping
// to a java.time.* type or some old-school type.
else if (jdbcType == Types.DATE) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Date sqlDate = resultSet.getDate(columnNumber);
return resultSet.wasNull() ? null : sqlDate.toString();
}
@Override
public String getColumnName() {
return columnName;
}
};
} else if (jdbcType == Types.TIME ) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Time sqlTime = resultSet.getTime(columnNumber);
return resultSet.wasNull() ? null : sqlTime.toString();
}
@Override
public String getColumnName() {
return columnName;
}
};
} else if (jdbcType == Types.TIMESTAMP) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Timestamp sqlTimestamp = resultSet.getTimestamp(columnNumber);
return resultSet.wasNull() ? null : sqlTimestamp.toInstant().toString();
}
@Override
public String getColumnName() {
return columnName;
}
};
}
// Now we're going to start seeing if things are assignable from the returned type
// to a more general type - this allows us to cover cases where something we weren't
// explicitly expecting, but can handle, is being returned.
else {
Class<?> clazz;
try {
clazz = Class.forName(className, false, getClass().getClassLoader());
} catch (Exception e) {
throw new RuntimeException(e);
}
final int scale = metadata.getScale(columnNumber);
if (Number.class.isAssignableFrom(clazz)) {
if (scale > 0) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
BigDecimal bd = resultSet.getBigDecimal(columnNumber);
return resultSet.wasNull() ? null : bd.doubleValue();
}
@Override
public String getColumnName() {
return columnName;
}
};
} else {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
BigDecimal bd = resultSet.getBigDecimal(columnNumber);
return resultSet.wasNull() ? null : bd.longValue();
}
@Override
public String getColumnName() {
return columnName;
}
};
}
} else if (Clob.class.isAssignableFrom(clazz)) {
valueSelector = new ResultSetValueSelector() {
@Override
public Object selectValue(ResultSet resultSet) throws SQLException {
Clob c = resultSet.getClob(columnNumber);
if (resultSet.wasNull()) {
return null;
}
long length = c.length();
int lengthInt = (int) length;
if (length != lengthInt) {
throw new SQLException(String.format(Locale.ROOT,
"Encountered a clob of length #%l in column '%s' (col #%d). Max supported length is #%i.",
length, columnName, columnNumber, Integer.MAX_VALUE));
}
return c.getSubString(1, lengthInt);
}
@Override
public String getColumnName() {
return columnName;
}
};
}
}
return valueSelector;
}
/**
* Closes the JDBCStream
**/
@ -432,9 +595,10 @@ public class JDBCStream extends TupleStream implements Expressible {
// it's already in the sqlQuery but there's no way we can reliably determine the sort from the query.
return definedSort;
}
public interface ResultSetValueSelector {
String getColumnName();
Object selectValue(ResultSet resultSet) throws SQLException;
}
}
interface ResultSetValueSelector {
String getColumnName();
Object selectValue(ResultSet resultSet) throws SQLException;
}

34
solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.io.stream;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
@ -96,6 +97,8 @@ public class JDBCStreamTest extends SolrCloudTestCase {
statement.executeUpdate("create table PEOPLE(ID int not null primary key, NAME varchar(50), COUNTRY_CODE char(2), DELETED char(1) default 'N')");
statement.executeUpdate("create table PEOPLE_SPORTS(ID int not null primary key, PERSON_ID int, SPORT_NAME varchar(50), DELETED char(1) default 'N')");
statement.executeUpdate("create table UNSUPPORTED_COLUMNS(ID int not null primary key, UNSP binary)");
statement.executeUpdate("create table DUAL(ID int not null primary key)");
statement.executeUpdate("insert into DUAL values(1)");
}
@ -158,8 +161,29 @@ public class JDBCStreamTest extends SolrCloudTestCase {
assertOrderOf(tuples, "CODE", "NP", "NL", "NO", "US");
assertOrderOf(tuples, "COUNTRY_NAME", "Nepal", "Netherlands", "Norway", "United States");
// Additional Types
String query = "select 1 as ID1, {ts '2017-02-18 12:34:56.789'} as TS1, {t '01:02:03'} as T1, "
+ "{d '1593-03-14'} as D1, cast(12.34 AS DECIMAL(4,2)) as DEC4_2, "
+ "cast(1234 AS DECIMAL(4,0)) as DEC4_0, cast('big stuff' as CLOB(100)) as CLOB1 "
+ "from DUAL order by ID1";
stream = new JDBCStream("jdbc:hsqldb:mem:.", query, new FieldComparator("ID1", ComparatorOrder.ASCENDING));
tuples = getTuples(stream);
assertEquals(1, tuples.size());
Tuple t;
try (Connection connection = DriverManager.getConnection("jdbc:hsqldb:mem:.");
Statement statement = connection.createStatement()) {
ResultSet rs = statement.executeQuery(query);
rs.next();
t = tuples.iterator().next();
assertString(t, "CLOB1", rs.getString("CLOB1"));
assertString(t, "TS1", rs.getTimestamp("TS1").toInstant().toString());
assertString(t, "T1", rs.getTime("T1").toString());
assertString(t, "D1", rs.getDate("D1").toString());
assertDouble(t, "DEC4_2", rs.getDouble("DEC4_2"));
assertLong(t, "DEC4_0", rs.getLong("DEC4_0"));
}
}
@Test
public void testJDBCJoin() throws Exception {
@ -643,6 +667,14 @@ public class JDBCStreamTest extends SolrCloudTestCase {
return true;
}
public boolean assertDouble(Tuple tuple, String fieldName, double d) throws Exception {
double dv = (double)tuple.get(fieldName);
if(dv != d) {
throw new Exception("Doubles not equal:"+d+" : "+dv);
}
return true;
}
public boolean assertString(Tuple tuple, String fieldName, String expected) throws Exception {
String actual = (String)tuple.get(fieldName);