svn merge -c 1530816 FIXES: MAPREDUCE-5102. fix coverage org.apache.hadoop.mapreduce.lib.db and org.apache.hadoop.mapred.lib.db. Contributed by Aleksey Gorshkov, Andrey Klochkov, and Nathan Roberts
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1530818 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jason Darrell Lowe
parent
5f2d5faa7f
commit
9b5a023fc2
|
|
@ -50,6 +50,10 @@ Release 2.3.0 - UNRELEASED
|
||||||
|
|
||||||
MAPREDUCE-5514. Fix TestRMContainerAllocator. (Zhijie Shen via acmurthy)
|
MAPREDUCE-5514. Fix TestRMContainerAllocator. (Zhijie Shen via acmurthy)
|
||||||
|
|
||||||
|
MAPREDUCE-5102. fix coverage org.apache.hadoop.mapreduce.lib.db and
|
||||||
|
org.apache.hadoop.mapred.lib.db (Aleksey Gorshkov, Andrey Klochkov, and
|
||||||
|
Nathan Roberts via jlowe)
|
||||||
|
|
||||||
Release 2.2.1 - UNRELEASED
|
Release 2.2.1 - UNRELEASED
|
||||||
|
|
||||||
INCOMPATIBLE CHANGES
|
INCOMPATIBLE CHANGES
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,157 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.mapred.lib.db;
|
||||||
|
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
|
||||||
|
import org.apache.hadoop.io.LongWritable;
|
||||||
|
import org.apache.hadoop.mapred.JobConf;
|
||||||
|
import org.apache.hadoop.mapred.RecordReader;
|
||||||
|
import org.apache.hadoop.mapred.Reporter;
|
||||||
|
import org.apache.hadoop.mapred.lib.db.DBInputFormat.DBInputSplit;
|
||||||
|
import org.apache.hadoop.mapred.lib.db.DBInputFormat.DBRecordReader;
|
||||||
|
import org.apache.hadoop.mapred.lib.db.DBInputFormat.NullDBWritable;
|
||||||
|
import org.apache.hadoop.mapred.InputSplit;
|
||||||
|
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||||
|
import org.apache.hadoop.mapred.lib.db.DBConfiguration;
|
||||||
|
import org.apache.hadoop.mapreduce.lib.db.DriverForTest;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
public class TestDBInputFormat {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* test DBInputFormat class. Class should split result for chunks
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@Test(timeout = 10000)
|
||||||
|
public void testDBInputFormat() throws Exception {
|
||||||
|
JobConf configuration = new JobConf();
|
||||||
|
setupDriver(configuration);
|
||||||
|
|
||||||
|
DBInputFormat<NullDBWritable> format = new DBInputFormat<NullDBWritable>();
|
||||||
|
format.setConf(configuration);
|
||||||
|
format.setConf(configuration);
|
||||||
|
DBInputFormat.DBInputSplit splitter = new DBInputFormat.DBInputSplit(1, 10);
|
||||||
|
Reporter reporter = mock(Reporter.class);
|
||||||
|
RecordReader<LongWritable, NullDBWritable> reader = format.getRecordReader(
|
||||||
|
splitter, configuration, reporter);
|
||||||
|
|
||||||
|
configuration.setInt(MRJobConfig.NUM_MAPS, 3);
|
||||||
|
InputSplit[] lSplits = format.getSplits(configuration, 3);
|
||||||
|
assertEquals(5, lSplits[0].getLength());
|
||||||
|
assertEquals(3, lSplits.length);
|
||||||
|
|
||||||
|
// test reader .Some simple tests
|
||||||
|
assertEquals(LongWritable.class, reader.createKey().getClass());
|
||||||
|
assertEquals(0, reader.getPos());
|
||||||
|
assertEquals(0, reader.getProgress(), 0.001);
|
||||||
|
reader.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* test configuration for db. should works DBConfiguration.* parameters.
|
||||||
|
*/
|
||||||
|
@Test (timeout = 5000)
|
||||||
|
public void testSetInput() {
|
||||||
|
JobConf configuration = new JobConf();
|
||||||
|
|
||||||
|
String[] fieldNames = { "field1", "field2" };
|
||||||
|
DBInputFormat.setInput(configuration, NullDBWritable.class, "table",
|
||||||
|
"conditions", "orderBy", fieldNames);
|
||||||
|
assertEquals(
|
||||||
|
"org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable",
|
||||||
|
configuration.getClass(DBConfiguration.INPUT_CLASS_PROPERTY, null)
|
||||||
|
.getName());
|
||||||
|
assertEquals("table",
|
||||||
|
configuration.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, null));
|
||||||
|
|
||||||
|
String[] fields = configuration
|
||||||
|
.getStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY);
|
||||||
|
assertEquals("field1", fields[0]);
|
||||||
|
assertEquals("field2", fields[1]);
|
||||||
|
|
||||||
|
assertEquals("conditions",
|
||||||
|
configuration.get(DBConfiguration.INPUT_CONDITIONS_PROPERTY, null));
|
||||||
|
assertEquals("orderBy",
|
||||||
|
configuration.get(DBConfiguration.INPUT_ORDER_BY_PROPERTY, null));
|
||||||
|
|
||||||
|
configuration = new JobConf();
|
||||||
|
|
||||||
|
DBInputFormat.setInput(configuration, NullDBWritable.class, "query",
|
||||||
|
"countQuery");
|
||||||
|
assertEquals("query", configuration.get(DBConfiguration.INPUT_QUERY, null));
|
||||||
|
assertEquals("countQuery",
|
||||||
|
configuration.get(DBConfiguration.INPUT_COUNT_QUERY, null));
|
||||||
|
|
||||||
|
JobConf jConfiguration = new JobConf();
|
||||||
|
DBConfiguration.configureDB(jConfiguration, "driverClass", "dbUrl", "user",
|
||||||
|
"password");
|
||||||
|
assertEquals("driverClass",
|
||||||
|
jConfiguration.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
|
||||||
|
assertEquals("dbUrl", jConfiguration.get(DBConfiguration.URL_PROPERTY));
|
||||||
|
assertEquals("user", jConfiguration.get(DBConfiguration.USERNAME_PROPERTY));
|
||||||
|
assertEquals("password",
|
||||||
|
jConfiguration.get(DBConfiguration.PASSWORD_PROPERTY));
|
||||||
|
jConfiguration = new JobConf();
|
||||||
|
DBConfiguration.configureDB(jConfiguration, "driverClass", "dbUrl");
|
||||||
|
assertEquals("driverClass",
|
||||||
|
jConfiguration.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
|
||||||
|
assertEquals("dbUrl", jConfiguration.get(DBConfiguration.URL_PROPERTY));
|
||||||
|
assertNull(jConfiguration.get(DBConfiguration.USERNAME_PROPERTY));
|
||||||
|
assertNull(jConfiguration.get(DBConfiguration.PASSWORD_PROPERTY));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
* test DBRecordReader. This reader should creates keys, values, know about position..
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Test (timeout = 5000)
|
||||||
|
public void testDBRecordReader() throws Exception {
|
||||||
|
|
||||||
|
JobConf job = mock(JobConf.class);
|
||||||
|
DBConfiguration dbConfig = mock(DBConfiguration.class);
|
||||||
|
String[] fields = { "field1", "filed2" };
|
||||||
|
|
||||||
|
@SuppressWarnings("rawtypes")
|
||||||
|
DBRecordReader reader = new DBInputFormat<NullDBWritable>().new DBRecordReader(
|
||||||
|
new DBInputSplit(), NullDBWritable.class, job,
|
||||||
|
DriverForTest.getConnection(), dbConfig, "condition", fields, "table");
|
||||||
|
LongWritable key = reader.createKey();
|
||||||
|
assertEquals(0, key.get());
|
||||||
|
DBWritable value = reader.createValue();
|
||||||
|
assertEquals(
|
||||||
|
"org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable", value
|
||||||
|
.getClass().getName());
|
||||||
|
assertEquals(0, reader.getPos());
|
||||||
|
assertFalse(reader.next(key, value));
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private void setupDriver(JobConf configuration) throws Exception {
|
||||||
|
configuration.set(DBConfiguration.URL_PROPERTY, "testUrl");
|
||||||
|
DriverManager.registerDriver(new DriverForTest());
|
||||||
|
configuration.set(DBConfiguration.DRIVER_CLASS_PROPERTY,
|
||||||
|
DriverForTest.class.getCanonicalName());
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,112 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.mapreduce.lib.db;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DatabaseMetaData;
|
||||||
|
import java.sql.Driver;
|
||||||
|
import java.sql.DriverPropertyInfo;
|
||||||
|
import java.sql.PreparedStatement;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.SQLFeatureNotSupportedException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.Properties;
|
||||||
|
import java.util.logging.Logger;
|
||||||
|
|
||||||
|
import static org.mockito.Matchers.any;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* class emulates a connection to database
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public class DriverForTest implements Driver {
|
||||||
|
|
||||||
|
public static Connection getConnection() {
|
||||||
|
Connection connection = mock(FakeConnection.class);
|
||||||
|
try {
|
||||||
|
Statement statement = mock(Statement.class);
|
||||||
|
ResultSet results = mock(ResultSet.class);
|
||||||
|
when(results.getLong(1)).thenReturn(15L);
|
||||||
|
when(statement.executeQuery(any(String.class))).thenReturn(results);
|
||||||
|
when(connection.createStatement()).thenReturn(statement);
|
||||||
|
|
||||||
|
DatabaseMetaData metadata = mock(DatabaseMetaData.class);
|
||||||
|
when(metadata.getDatabaseProductName()).thenReturn("Test");
|
||||||
|
when(connection.getMetaData()).thenReturn(metadata);
|
||||||
|
|
||||||
|
PreparedStatement reparedStatement0= mock(PreparedStatement.class);
|
||||||
|
when(connection.prepareStatement(anyString())).thenReturn(
|
||||||
|
reparedStatement0);
|
||||||
|
|
||||||
|
PreparedStatement preparedStatement = mock(PreparedStatement.class);
|
||||||
|
ResultSet resultSet = mock(ResultSet.class);
|
||||||
|
when(resultSet.next()).thenReturn(false);
|
||||||
|
when(preparedStatement.executeQuery()).thenReturn(resultSet);
|
||||||
|
|
||||||
|
when(connection.prepareStatement(anyString(), anyInt(), anyInt()))
|
||||||
|
.thenReturn(preparedStatement);
|
||||||
|
} catch (SQLException e) {
|
||||||
|
;
|
||||||
|
}
|
||||||
|
return connection;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean acceptsURL(String arg0) throws SQLException {
|
||||||
|
return "testUrl".equals(arg0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Connection connect(String arg0, Properties arg1) throws SQLException {
|
||||||
|
|
||||||
|
return getConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getMajorVersion() {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int getMinorVersion() {
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DriverPropertyInfo[] getPropertyInfo(String arg0, Properties arg1)
|
||||||
|
throws SQLException {
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean jdbcCompliant() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Logger getParentLogger() throws SQLFeatureNotSupportedException {
|
||||||
|
throw new SQLFeatureNotSupportedException();
|
||||||
|
}
|
||||||
|
|
||||||
|
private interface FakeConnection extends Connection{
|
||||||
|
public void setSessionTimeZone(String arg);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,170 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.mapreduce.lib.db;
|
||||||
|
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.Types;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.mapreduce.Job;
|
||||||
|
import org.apache.hadoop.mapreduce.InputSplit;
|
||||||
|
import org.apache.hadoop.mapreduce.JobContext;
|
||||||
|
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||||
|
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.DBInputSplit;
|
||||||
|
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable;
|
||||||
|
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat.DataDrivenDBInputSplit;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import static org.junit.Assert.*;
|
||||||
|
import static org.mockito.Mockito.*;
|
||||||
|
|
||||||
|
public class TestDbClasses {
|
||||||
|
/**
|
||||||
|
* test splitters from DataDrivenDBInputFormat. For different data types may
|
||||||
|
* be different splitter
|
||||||
|
*/
|
||||||
|
@Test(timeout = 1000)
|
||||||
|
public void testDataDrivenDBInputFormatSplitter() {
|
||||||
|
DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
|
||||||
|
testCommonSplitterTypes(format);
|
||||||
|
assertEquals(DateSplitter.class, format.getSplitter(Types.TIMESTAMP)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(DateSplitter.class, format.getSplitter(Types.DATE).getClass());
|
||||||
|
assertEquals(DateSplitter.class, format.getSplitter(Types.TIME).getClass());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 1000)
|
||||||
|
public void testDataDrivenDBInputFormat() throws Exception {
|
||||||
|
JobContext jobContext = mock(JobContext.class);
|
||||||
|
Configuration configuration = new Configuration();
|
||||||
|
configuration.setInt(MRJobConfig.NUM_MAPS, 1);
|
||||||
|
|
||||||
|
when(jobContext.getConfiguration()).thenReturn(configuration);
|
||||||
|
DataDrivenDBInputFormat<NullDBWritable> format = new DataDrivenDBInputFormat<NullDBWritable>();
|
||||||
|
List<InputSplit> splits = format.getSplits(jobContext);
|
||||||
|
assertEquals(1, splits.size());
|
||||||
|
DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(0);
|
||||||
|
assertEquals("1=1", split.getLowerClause());
|
||||||
|
assertEquals("1=1", split.getUpperClause());
|
||||||
|
|
||||||
|
// 2
|
||||||
|
configuration.setInt(MRJobConfig.NUM_MAPS, 2);
|
||||||
|
|
||||||
|
DataDrivenDBInputFormat.setBoundingQuery(configuration, "query");
|
||||||
|
assertEquals("query",
|
||||||
|
configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));
|
||||||
|
|
||||||
|
Job job = mock(Job.class);
|
||||||
|
when(job.getConfiguration()).thenReturn(configuration);
|
||||||
|
DataDrivenDBInputFormat.setInput(job, NullDBWritable.class, "query",
|
||||||
|
"Bounding Query");
|
||||||
|
|
||||||
|
assertEquals("Bounding Query",
|
||||||
|
configuration.get(DBConfiguration.INPUT_BOUNDING_QUERY));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout = 1000)
|
||||||
|
public void testOracleDataDrivenDBInputFormat() throws Exception {
|
||||||
|
OracleDataDrivenDBInputFormat<NullDBWritable> format =
|
||||||
|
new OracleDataDrivenDBInputFormatForTest();
|
||||||
|
testCommonSplitterTypes(format);
|
||||||
|
assertEquals(OracleDateSplitter.class, format.getSplitter(Types.TIMESTAMP)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(OracleDateSplitter.class,
|
||||||
|
format.getSplitter(Types.DATE).getClass());
|
||||||
|
assertEquals(OracleDateSplitter.class,
|
||||||
|
format.getSplitter(Types.TIME).getClass());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* test generate sql script for OracleDBRecordReader.
|
||||||
|
*/
|
||||||
|
|
||||||
|
@Test(timeout = 2000)
|
||||||
|
public void testOracleDBRecordReader() throws Exception {
|
||||||
|
DBInputSplit splitter = new DBInputSplit(1, 10);
|
||||||
|
Configuration configuration = new Configuration();
|
||||||
|
Connection connect = DriverForTest.getConnection();
|
||||||
|
|
||||||
|
DBConfiguration dbConfiguration = new DBConfiguration(configuration);
|
||||||
|
dbConfiguration.setInputOrderBy("Order");
|
||||||
|
String[] fields = { "f1", "f2" };
|
||||||
|
|
||||||
|
OracleDBRecordReader<NullDBWritable> recorder = new OracleDBRecordReader<NullDBWritable>(
|
||||||
|
splitter, NullDBWritable.class, configuration, connect,
|
||||||
|
dbConfiguration, "condition", fields, "table");
|
||||||
|
assertEquals(
|
||||||
|
"SELECT * FROM (SELECT a.*,ROWNUM dbif_rno FROM ( SELECT f1, f2 FROM table WHERE condition ORDER BY Order ) a WHERE rownum <= 1 + 9 ) WHERE dbif_rno >= 1",
|
||||||
|
recorder.getSelectQuery());
|
||||||
|
}
|
||||||
|
|
||||||
|
private void testCommonSplitterTypes(
|
||||||
|
DataDrivenDBInputFormat<NullDBWritable> format) {
|
||||||
|
assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.DECIMAL)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(BigDecimalSplitter.class, format.getSplitter(Types.NUMERIC)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(BooleanSplitter.class, format.getSplitter(Types.BOOLEAN)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(BooleanSplitter.class, format.getSplitter(Types.BIT)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(IntegerSplitter.class, format.getSplitter(Types.BIGINT)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(IntegerSplitter.class, format.getSplitter(Types.TINYINT)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(IntegerSplitter.class, format.getSplitter(Types.SMALLINT)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(IntegerSplitter.class, format.getSplitter(Types.INTEGER)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(FloatSplitter.class, format.getSplitter(Types.DOUBLE)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(FloatSplitter.class, format.getSplitter(Types.REAL).getClass());
|
||||||
|
assertEquals(FloatSplitter.class, format.getSplitter(Types.FLOAT)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(TextSplitter.class, format.getSplitter(Types.LONGVARCHAR)
|
||||||
|
.getClass());
|
||||||
|
assertEquals(TextSplitter.class, format.getSplitter(Types.CHAR).getClass());
|
||||||
|
assertEquals(TextSplitter.class, format.getSplitter(Types.VARCHAR)
|
||||||
|
.getClass());
|
||||||
|
// if unknown data type splitter is null
|
||||||
|
assertNull(format.getSplitter(Types.BINARY));
|
||||||
|
}
|
||||||
|
|
||||||
|
private class OracleDataDrivenDBInputFormatForTest extends
|
||||||
|
OracleDataDrivenDBInputFormat<NullDBWritable> {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DBConfiguration getDBConf() {
|
||||||
|
|
||||||
|
String[] names = { "field1", "field2" };
|
||||||
|
DBConfiguration result = mock(DBConfiguration.class);
|
||||||
|
when(result.getInputConditions()).thenReturn("conditions");
|
||||||
|
when(result.getInputFieldNames()).thenReturn(names);
|
||||||
|
when(result.getInputTableName()).thenReturn("table");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Connection getConnection() {
|
||||||
|
return DriverForTest.getConnection();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,164 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.mapreduce.lib.db;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertEquals;
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.math.BigDecimal;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.mapreduce.InputSplit;
|
||||||
|
import org.apache.hadoop.mapreduce.MRJobConfig;
|
||||||
|
import org.apache.hadoop.mapreduce.lib.db.DataDrivenDBInputFormat.DataDrivenDBInputSplit;
|
||||||
|
import org.junit.Before;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test Splitters. Splitters should build parts of sql sentences for split result.
|
||||||
|
*/
|
||||||
|
public class TestSplitters {
|
||||||
|
|
||||||
|
private Configuration configuration;
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void setup() {
|
||||||
|
configuration = new Configuration();
|
||||||
|
configuration.setInt(MRJobConfig.NUM_MAPS, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=2000)
|
||||||
|
public void testBooleanSplitter() throws Exception{
|
||||||
|
BooleanSplitter splitter = new BooleanSplitter();
|
||||||
|
ResultSet result = mock(ResultSet.class);
|
||||||
|
when(result.getString(1)).thenReturn("result1");
|
||||||
|
|
||||||
|
List<InputSplit> splits=splitter.split(configuration, result, "column");
|
||||||
|
assertSplits(new String[] {"column = FALSE column = FALSE",
|
||||||
|
"column IS NULL column IS NULL"}, splits);
|
||||||
|
|
||||||
|
when(result.getString(1)).thenReturn("result1");
|
||||||
|
when(result.getString(2)).thenReturn("result2");
|
||||||
|
when(result.getBoolean(1)).thenReturn(true);
|
||||||
|
when(result.getBoolean(2)).thenReturn(false);
|
||||||
|
|
||||||
|
splits=splitter.split(configuration, result, "column");
|
||||||
|
assertEquals(0, splits.size());
|
||||||
|
|
||||||
|
when(result.getString(1)).thenReturn("result1");
|
||||||
|
when(result.getString(2)).thenReturn("result2");
|
||||||
|
when(result.getBoolean(1)).thenReturn(false);
|
||||||
|
when(result.getBoolean(2)).thenReturn(true);
|
||||||
|
|
||||||
|
splits = splitter.split(configuration, result, "column");
|
||||||
|
assertSplits(new String[] {
|
||||||
|
"column = FALSE column = FALSE", ".*column = TRUE"}, splits);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=2000)
|
||||||
|
public void testFloatSplitter() throws Exception{
|
||||||
|
FloatSplitter splitter = new FloatSplitter();
|
||||||
|
|
||||||
|
ResultSet results = mock(ResultSet.class);
|
||||||
|
|
||||||
|
List<InputSplit> splits = splitter.split(configuration, results, "column");
|
||||||
|
assertSplits(new String[] {".*column IS NULL"}, splits);
|
||||||
|
|
||||||
|
when(results.getString(1)).thenReturn("result1");
|
||||||
|
when(results.getString(2)).thenReturn("result2");
|
||||||
|
when(results.getDouble(1)).thenReturn(5.0);
|
||||||
|
when(results.getDouble(2)).thenReturn(7.0);
|
||||||
|
|
||||||
|
splits = splitter.split(configuration, results, "column1");
|
||||||
|
assertSplits(new String[] {"column1 >= 5.0 column1 < 6.0",
|
||||||
|
"column1 >= 6.0 column1 <= 7.0"}, splits);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=2000)
|
||||||
|
public void testBigDecimalSplitter() throws Exception{
|
||||||
|
BigDecimalSplitter splitter = new BigDecimalSplitter();
|
||||||
|
ResultSet result = mock(ResultSet.class);
|
||||||
|
|
||||||
|
List<InputSplit> splits = splitter.split(configuration, result, "column");
|
||||||
|
assertSplits(new String[] {".*column IS NULL"}, splits);
|
||||||
|
|
||||||
|
when(result.getString(1)).thenReturn("result1");
|
||||||
|
when(result.getString(2)).thenReturn("result2");
|
||||||
|
when(result.getBigDecimal(1)).thenReturn(new BigDecimal(10));
|
||||||
|
when(result.getBigDecimal(2)).thenReturn(new BigDecimal(12));
|
||||||
|
|
||||||
|
splits = splitter.split(configuration, result, "column1");
|
||||||
|
assertSplits(new String[] {"column1 >= 10 column1 < 11",
|
||||||
|
"column1 >= 11 column1 <= 12"}, splits);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=2000)
|
||||||
|
public void testIntegerSplitter() throws Exception{
|
||||||
|
IntegerSplitter splitter = new IntegerSplitter();
|
||||||
|
ResultSet result = mock(ResultSet.class);
|
||||||
|
|
||||||
|
List<InputSplit> splits = splitter.split(configuration, result, "column");
|
||||||
|
assertSplits(new String[] {".*column IS NULL"}, splits);
|
||||||
|
|
||||||
|
when(result.getString(1)).thenReturn("result1");
|
||||||
|
when(result.getString(2)).thenReturn("result2");
|
||||||
|
when(result.getLong(1)).thenReturn(8L);
|
||||||
|
when(result.getLong(2)).thenReturn(19L);
|
||||||
|
|
||||||
|
splits = splitter.split(configuration, result, "column1");
|
||||||
|
assertSplits(new String[] {"column1 >= 8 column1 < 13",
|
||||||
|
"column1 >= 13 column1 < 18", "column1 >= 18 column1 <= 19"}, splits);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test(timeout=2000)
|
||||||
|
public void testTextSplitter() throws Exception{
|
||||||
|
TextSplitter splitter = new TextSplitter();
|
||||||
|
ResultSet result = mock(ResultSet.class);
|
||||||
|
|
||||||
|
List<InputSplit> splits = splitter.split(configuration, result, "column");
|
||||||
|
assertSplits(new String[] {"column IS NULL column IS NULL"}, splits);
|
||||||
|
|
||||||
|
when(result.getString(1)).thenReturn("result1");
|
||||||
|
when(result.getString(2)).thenReturn("result2");
|
||||||
|
|
||||||
|
splits = splitter.split(configuration, result, "column1");
|
||||||
|
assertSplits(new String[] {"column1 >= 'result1' column1 < 'result1.'",
|
||||||
|
"column1 >= 'result1' column1 <= 'result2'"}, splits);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void assertSplits(String[] expectedSplitRE,
|
||||||
|
List<InputSplit> splits) throws IOException {
|
||||||
|
assertEquals(expectedSplitRE.length, splits.size());
|
||||||
|
for (int i = 0; i < expectedSplitRE.length; i++) {
|
||||||
|
DataDrivenDBInputSplit split = (DataDrivenDBInputSplit) splits.get(i);
|
||||||
|
String actualExpr = split.getLowerClause() + " " + split.getUpperClause();
|
||||||
|
assertTrue("Split #" + (i+1) + " expression is wrong."
|
||||||
|
+ " Expected " + expectedSplitRE[i]
|
||||||
|
+ " Actual " + actualExpr,
|
||||||
|
Pattern.matches(expectedSplitRE[i], actualExpr));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue