lazy create descriptor in ProtobufInputRowParser (#6678)

This commit is contained in:
陈春斌 2018-11-29 13:59:29 +08:00 committed by Fangjin Yang
parent c81cb94226
commit 624f328ea1
2 changed files with 16 additions and 5 deletions

View File

@ -22,6 +22,7 @@ package org.apache.druid.data.input.protobuf;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.github.os72.protobuf.dynamic.DynamicSchema; import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
@ -53,7 +54,7 @@ public class ProtobufInputRowParser implements ByteBufferInputRowParser
private final ParseSpec parseSpec; private final ParseSpec parseSpec;
private final String descriptorFilePath; private final String descriptorFilePath;
private final String protoMessageType; private final String protoMessageType;
private final Descriptor descriptor; private Descriptor descriptor;
private Parser<String, Object> parser; private Parser<String, Object> parser;
private final List<String> dimensions; private final List<String> dimensions;
@ -67,7 +68,6 @@ public class ProtobufInputRowParser implements ByteBufferInputRowParser
this.parseSpec = parseSpec; this.parseSpec = parseSpec;
this.descriptorFilePath = descriptorFilePath; this.descriptorFilePath = descriptorFilePath;
this.protoMessageType = protoMessageType; this.protoMessageType = protoMessageType;
this.descriptor = getDescriptor(descriptorFilePath);
this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames();
} }
@ -83,6 +83,14 @@ public class ProtobufInputRowParser implements ByteBufferInputRowParser
return new ProtobufInputRowParser(parseSpec, descriptorFilePath, protoMessageType); return new ProtobufInputRowParser(parseSpec, descriptorFilePath, protoMessageType);
} }
@VisibleForTesting
void initDescriptor()
{
if (this.descriptor == null) {
this.descriptor = getDescriptor(descriptorFilePath);
}
}
@Override @Override
public List<InputRow> parseBatch(ByteBuffer input) public List<InputRow> parseBatch(ByteBuffer input)
{ {
@ -90,6 +98,7 @@ public class ProtobufInputRowParser implements ByteBufferInputRowParser
// parser should be created when it is really used to avoid unnecessary initialization of the underlying // parser should be created when it is really used to avoid unnecessary initialization of the underlying
// parseSpec. // parseSpec.
parser = parseSpec.makeParser(); parser = parseSpec.makeParser();
initDescriptor();
} }
String json; String json;
try { try {

View File

@ -83,7 +83,7 @@ public class ProtobufInputRowParserTest
//configure parser with desc file, and specify which file name to use //configure parser with desc file, and specify which file name to use
@SuppressWarnings("unused") // expected to create parser without exception @SuppressWarnings("unused") // expected to create parser without exception
ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", "ProtoTestEvent"); ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", "ProtoTestEvent");
parser.initDescriptor();
} }
@ -93,7 +93,7 @@ public class ProtobufInputRowParserTest
//configure parser with desc file, and specify which file name to use //configure parser with desc file, and specify which file name to use
@SuppressWarnings("unused") // expected to create parser without exception @SuppressWarnings("unused") // expected to create parser without exception
ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", "prototest.ProtoTestEvent"); ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", "prototest.ProtoTestEvent");
parser.initDescriptor();
} }
@ -103,7 +103,7 @@ public class ProtobufInputRowParserTest
//configure parser with desc file //configure parser with desc file
@SuppressWarnings("unused") // expected exception @SuppressWarnings("unused") // expected exception
ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", "BadName"); ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", "BadName");
parser.initDescriptor();
} }
@Test(expected = ParseException.class) @Test(expected = ParseException.class)
@ -112,6 +112,7 @@ public class ProtobufInputRowParserTest
//configure parser with non existent desc file //configure parser with non existent desc file
@SuppressWarnings("unused") // expected exception @SuppressWarnings("unused") // expected exception
ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "file:/nonexist.desc", "BadName"); ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "file:/nonexist.desc", "BadName");
parser.initDescriptor();
} }
@Test @Test
@ -120,6 +121,7 @@ public class ProtobufInputRowParserTest
// For the backward compatibility, protoMessageType allows null when the desc file has only one message type. // For the backward compatibility, protoMessageType allows null when the desc file has only one message type.
@SuppressWarnings("unused") // expected to create parser without exception @SuppressWarnings("unused") // expected to create parser without exception
ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", null); ProtobufInputRowParser parser = new ProtobufInputRowParser(parseSpec, "prototest.desc", null);
parser.initDescriptor();
} }
@Test @Test