mirror of https://github.com/apache/druid.git
log.warn anytime a column is relying on ArrayIngestMode.MVD (#17164)
* log.warn anytime a column is relying on ArrayIngestMode.MVD
This commit is contained in:
parent
a2b011cdcd
commit
d77637344d
|
@ -387,7 +387,7 @@ public class MSQTaskSqlEngine implements SqlEngine
|
||||||
final ColumnType oldDruidType = Calcites.getColumnTypeForRelDataType(oldSqlTypeField.getType());
|
final ColumnType oldDruidType = Calcites.getColumnTypeForRelDataType(oldSqlTypeField.getType());
|
||||||
final RelDataType newSqlType = rootRel.getRowType().getFieldList().get(columnIndex).getType();
|
final RelDataType newSqlType = rootRel.getRowType().getFieldList().get(columnIndex).getType();
|
||||||
final ColumnType newDruidType =
|
final ColumnType newDruidType =
|
||||||
DimensionSchemaUtils.getDimensionType(Calcites.getColumnTypeForRelDataType(newSqlType), arrayIngestMode);
|
DimensionSchemaUtils.getDimensionType(columnName, Calcites.getColumnTypeForRelDataType(newSqlType), arrayIngestMode);
|
||||||
|
|
||||||
if (newDruidType.isArray() && oldDruidType.is(ValueType.STRING)
|
if (newDruidType.isArray() && oldDruidType.is(ValueType.STRING)
|
||||||
|| (newDruidType.is(ValueType.STRING) && oldDruidType.isArray())) {
|
|| (newDruidType.is(ValueType.STRING) && oldDruidType.isArray())) {
|
||||||
|
|
|
@ -27,6 +27,8 @@ import org.apache.druid.data.input.impl.StringDimensionSchema;
|
||||||
import org.apache.druid.error.InvalidInput;
|
import org.apache.druid.error.InvalidInput;
|
||||||
import org.apache.druid.java.util.common.ISE;
|
import org.apache.druid.java.util.common.ISE;
|
||||||
import org.apache.druid.java.util.common.StringUtils;
|
import org.apache.druid.java.util.common.StringUtils;
|
||||||
|
import org.apache.druid.java.util.emitter.EmittingLogger;
|
||||||
|
import org.apache.druid.java.util.emitter.service.AlertEvent;
|
||||||
import org.apache.druid.segment.AutoTypeColumnSchema;
|
import org.apache.druid.segment.AutoTypeColumnSchema;
|
||||||
import org.apache.druid.segment.DimensionHandlerUtils;
|
import org.apache.druid.segment.DimensionHandlerUtils;
|
||||||
import org.apache.druid.segment.column.ColumnCapabilities;
|
import org.apache.druid.segment.column.ColumnCapabilities;
|
||||||
|
@ -42,6 +44,7 @@ import javax.annotation.Nullable;
|
||||||
*/
|
*/
|
||||||
public class DimensionSchemaUtils
|
public class DimensionSchemaUtils
|
||||||
{
|
{
|
||||||
|
private static final EmittingLogger LOG = new EmittingLogger(DimensionSchemaUtils.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates a dimension schema for creating {@link org.apache.druid.data.input.InputSourceReader}.
|
* Creates a dimension schema for creating {@link org.apache.druid.data.input.InputSourceReader}.
|
||||||
|
@ -89,7 +92,7 @@ public class DimensionSchemaUtils
|
||||||
return new AutoTypeColumnSchema(column, null);
|
return new AutoTypeColumnSchema(column, null);
|
||||||
} else {
|
} else {
|
||||||
// dimensionType may not be identical to queryType, depending on arrayIngestMode.
|
// dimensionType may not be identical to queryType, depending on arrayIngestMode.
|
||||||
final ColumnType dimensionType = getDimensionType(queryType, arrayIngestMode);
|
final ColumnType dimensionType = getDimensionType(column, queryType, arrayIngestMode);
|
||||||
|
|
||||||
if (dimensionType.getType() == ValueType.STRING) {
|
if (dimensionType.getType() == ValueType.STRING) {
|
||||||
return new StringDimensionSchema(
|
return new StringDimensionSchema(
|
||||||
|
@ -121,6 +124,7 @@ public class DimensionSchemaUtils
|
||||||
* @throws org.apache.druid.error.DruidException if there is some problem
|
* @throws org.apache.druid.error.DruidException if there is some problem
|
||||||
*/
|
*/
|
||||||
public static ColumnType getDimensionType(
|
public static ColumnType getDimensionType(
|
||||||
|
final String columnName,
|
||||||
@Nullable final ColumnType queryType,
|
@Nullable final ColumnType queryType,
|
||||||
final ArrayIngestMode arrayIngestMode
|
final ArrayIngestMode arrayIngestMode
|
||||||
)
|
)
|
||||||
|
@ -132,6 +136,13 @@ public class DimensionSchemaUtils
|
||||||
ValueType elementType = queryType.getElementType().getType();
|
ValueType elementType = queryType.getElementType().getType();
|
||||||
if (elementType == ValueType.STRING) {
|
if (elementType == ValueType.STRING) {
|
||||||
if (arrayIngestMode == ArrayIngestMode.MVD) {
|
if (arrayIngestMode == ArrayIngestMode.MVD) {
|
||||||
|
final String msgFormat = "Inserting a multi-value string column[%s] relying on deprecated"
|
||||||
|
+ " ArrayIngestMode.MVD. This query should be rewritten to use the ARRAY_TO_MV"
|
||||||
|
+ " operator to insert ARRAY types as multi-value strings.";
|
||||||
|
LOG.makeWarningAlert(msgFormat, columnName)
|
||||||
|
.severity(AlertEvent.Severity.DEPRECATED)
|
||||||
|
.addData("feature", "ArrayIngestMode.MVD")
|
||||||
|
.emit();
|
||||||
return ColumnType.STRING;
|
return ColumnType.STRING;
|
||||||
} else {
|
} else {
|
||||||
return queryType;
|
return queryType;
|
||||||
|
|
|
@ -1073,7 +1073,7 @@ public class RemoteTaskRunnerTest
|
||||||
EasyMock.expect(worker.getHost()).andReturn("host").atLeastOnce();
|
EasyMock.expect(worker.getHost()).andReturn("host").atLeastOnce();
|
||||||
EasyMock.replay(worker);
|
EasyMock.replay(worker);
|
||||||
ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class);
|
ServiceEmitter emitter = EasyMock.createMock(ServiceEmitter.class);
|
||||||
Capture<EmittingLogger.EmittingAlertBuilder> capturedArgument = Capture.newInstance();
|
Capture<EmittingLogger.LoggingAlertBuilder> capturedArgument = Capture.newInstance();
|
||||||
emitter.emit(EasyMock.capture(capturedArgument));
|
emitter.emit(EasyMock.capture(capturedArgument));
|
||||||
EasyMock.expectLastCall().atLeastOnce();
|
EasyMock.expectLastCall().atLeastOnce();
|
||||||
EmittingLogger.registerEmitter(emitter);
|
EmittingLogger.registerEmitter(emitter);
|
||||||
|
|
|
@ -29,8 +29,11 @@ import org.apache.druid.java.util.emitter.service.AlertBuilder;
|
||||||
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
|
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* {@link Logger} which also has an {@link ServiceEmitter}. Primarily useful for constructing and emitting "alerts" in
|
||||||
|
* the form of {@link AlertBuilder}, which will log when {@link AlertBuilder#emit()} is called.
|
||||||
*/
|
*/
|
||||||
public class EmittingLogger extends Logger
|
public class EmittingLogger extends Logger
|
||||||
{
|
{
|
||||||
|
@ -62,12 +65,34 @@ public class EmittingLogger extends Logger
|
||||||
return new EmittingLogger(getSlf4jLogger(), false);
|
return new EmittingLogger(getSlf4jLogger(), false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public AlertBuilder makeAlert(String message, Object... objects)
|
/**
|
||||||
|
* Make an {@link AlertBuilder} which will call {@link #warn(String, Object...)} when {@link AlertBuilder#emit()} is
|
||||||
|
* called.
|
||||||
|
*/
|
||||||
|
public AlertBuilder makeWarningAlert(String message, Object... objects)
|
||||||
{
|
{
|
||||||
return makeAlert(null, message, objects);
|
return makeAlert(null, false, message, objects);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make an {@link AlertBuilder} which will call {@link #error(String, Object...)} when {@link AlertBuilder#emit()} is
|
||||||
|
* called.
|
||||||
|
*/
|
||||||
|
public AlertBuilder makeAlert(String message, Object... objects)
|
||||||
|
{
|
||||||
|
return makeAlert(null, true, message, objects);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make an {@link AlertBuilder} which will call {@link #error(Throwable, String, Object...)} when
|
||||||
|
* {@link AlertBuilder#emit()} is called.
|
||||||
|
*/
|
||||||
public AlertBuilder makeAlert(@Nullable Throwable t, String message, Object... objects)
|
public AlertBuilder makeAlert(@Nullable Throwable t, String message, Object... objects)
|
||||||
|
{
|
||||||
|
return makeAlert(t, true, message, objects);
|
||||||
|
}
|
||||||
|
|
||||||
|
public AlertBuilder makeAlert(@Nullable Throwable t, boolean isError, String message, Object... objects)
|
||||||
{
|
{
|
||||||
if (emitter == null) {
|
if (emitter == null) {
|
||||||
final String errorMessage = StringUtils.format(
|
final String errorMessage = StringUtils.format(
|
||||||
|
@ -87,20 +112,22 @@ public class EmittingLogger extends Logger
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
|
|
||||||
return new EmittingAlertBuilder(t, StringUtils.format(message, objects), emitter)
|
return new LoggingAlertBuilder(t, StringUtils.format(message, objects), emitter, isError)
|
||||||
.addData("class", className);
|
.addData("class", className);
|
||||||
}
|
}
|
||||||
|
|
||||||
public class EmittingAlertBuilder extends AlertBuilder
|
public class LoggingAlertBuilder extends AlertBuilder
|
||||||
{
|
{
|
||||||
private final Throwable t;
|
private final Throwable t;
|
||||||
|
private final boolean isError;
|
||||||
|
|
||||||
private volatile boolean emitted = false;
|
private volatile boolean emitted = false;
|
||||||
|
|
||||||
private EmittingAlertBuilder(Throwable t, String description, ServiceEmitter emitter)
|
private LoggingAlertBuilder(Throwable t, String description, ServiceEmitter emitter, boolean isError)
|
||||||
{
|
{
|
||||||
super(description, emitter);
|
super(description, emitter);
|
||||||
this.t = t;
|
this.t = t;
|
||||||
|
this.isError = isError;
|
||||||
addThrowable(t);
|
addThrowable(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -126,15 +153,22 @@ public class EmittingLogger extends Logger
|
||||||
private void logIt(String format)
|
private void logIt(String format)
|
||||||
{
|
{
|
||||||
if (t == null) {
|
if (t == null) {
|
||||||
|
if (isError) {
|
||||||
error(format, description, dataMap);
|
error(format, description, dataMap);
|
||||||
|
} else {
|
||||||
|
warn(format, description, dataMap);
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// Filter out the stack trace from the message, because it should be in the logline already if it's wanted.
|
// Filter out the stack trace from the message, because it should be in the logline already if it's wanted.
|
||||||
error(
|
final Map<String, Object> filteredDataMap = Maps.filterKeys(
|
||||||
t,
|
dataMap,
|
||||||
format,
|
Predicates.not(Predicates.equalTo("exceptionStackTrace"))
|
||||||
description,
|
|
||||||
Maps.filterKeys(dataMap, Predicates.not(Predicates.equalTo("exceptionStackTrace")))
|
|
||||||
);
|
);
|
||||||
|
if (isError) {
|
||||||
|
error(t, format, description, filteredDataMap);
|
||||||
|
} else {
|
||||||
|
warn(t, format, description, filteredDataMap);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -186,6 +186,13 @@ public class AlertEvent implements Event
|
||||||
{
|
{
|
||||||
return "service-failure";
|
return "service-failure";
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
DEPRECATED {
|
||||||
|
@Override
|
||||||
|
public String toString()
|
||||||
|
{
|
||||||
|
return "deprecated";
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
public static final Severity DEFAULT = COMPONENT_FAILURE;
|
public static final Severity DEFAULT = COMPONENT_FAILURE;
|
||||||
|
|
Loading…
Reference in New Issue