Change time column name when reading from external sources in MSQ (#14148)

When ingesting from an external source which already contains a column "__time", currently, the value is dropped. Changing the time column name in the external input slice reader resolves this.
This commit is contained in:
Adarsh Sanjeev 2023-04-25 11:13:59 +05:30 committed by GitHub
parent 9d4cc501f7
commit cade80b582
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 42 additions and 1 deletions

View File

@ -52,6 +52,7 @@ import org.apache.druid.msq.querykit.LazyResourceHolder;
import org.apache.druid.msq.util.DimensionSchemaUtils;
import org.apache.druid.segment.RowAdapters;
import org.apache.druid.segment.RowBasedSegment;
import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.incremental.SimpleRowIngestionMeters;
import org.apache.druid.timeline.SegmentId;
@ -120,7 +121,7 @@ public class ExternalInputSliceReader implements InputSliceReader
)
{
final InputRowSchema schema = new InputRowSchema(
new TimestampSpec("__dummy__", "auto", DateTimes.utc(0)),
new TimestampSpec(ColumnHolder.TIME_COLUMN_NAME, "auto", DateTimes.utc(0)),
new DimensionsSpec(
signature.getColumnNames().stream().map(
column ->

View File

@ -137,6 +137,43 @@ public class MSQInsertTest extends MSQTestBase
}
@Test
public void testInsertWithExistingTimeColumn() throws IOException
{
List<Object[]> expectedRows = ImmutableList.of(
new Object[] {1678897351000L, "A"},
new Object[] {1679588551000L, "B"},
new Object[] {1682266951000L, "C"}
);
RowSignature rowSignature = RowSignature.builder()
.add("__time", ColumnType.LONG)
.add("flags", ColumnType.STRING)
.build();
final File toRead = MSQTestFileUtils.getResourceAsTemporaryFile(temporaryFolder, this,
"/dataset-with-time-column.json"
);
final String toReadFileNameAsJson = queryFramework().queryJsonMapper().writeValueAsString(toRead.getAbsolutePath());
testIngestQuery().setSql(" INSERT INTO foo1 SELECT\n"
+ " __time,\n"
+ " flags\n"
+ "FROM TABLE(\n"
+ " EXTERN(\n"
+ " '{ \"files\": [" + toReadFileNameAsJson + "],\"type\":\"local\"}',\n"
+ " '{\"type\": \"json\"}',\n"
+ " '[{\"name\": \"__time\", \"type\": \"long\"}, {\"name\": \"flags\", \"type\": \"string\"}]'\n"
+ " )\n"
+ ") PARTITIONED BY day")
.setQueryContext(context)
.setExpectedResultRows(expectedRows)
.setExpectedDataSource("foo1")
.setExpectedRowSignature(rowSignature)
.verifyResults();
}
@Test
public void testInsertOnExternalDataSource() throws IOException
{

View File

@ -0,0 +1,3 @@
{"__time":"1678897351000","flags":"A"}
{"__time":"1679588551000","flags":"B"}
{"__time":"1682266951000","flags":"C"}