Fix missing option serialization after backport

Relates #29465
This commit is contained in:
Jim Ferenczi 2018-05-30 12:55:31 +02:00
parent 3c21e46fa3
commit f582418ada
18 changed files with 33 additions and 152 deletions

View File

@ -9,4 +9,9 @@ These `execution_hint` are removed and should be replaced by `global_ordinals`.
The dynamic cluster setting named `search.max_buckets` now defaults
to 10,000 (instead of unlimited in the previous version).
Requests that try to return more than the limit will fail with an exception.
Requests that try to return more than the limit will fail with an exception.
==== `missing` option of the `composite` aggregation has been removed
The `missing` option of the `composite` aggregation, deprecated in 6.x,
has been removed. `missing_bucket` should be used instead.

View File

@ -98,7 +98,7 @@ public class NoriAnalysisTests extends ESTokenStreamTestCase {
.build();
TestAnalysis analysis = createTestAnalysis(settings);
Tokenizer tokenizer = analysis.tokenizer.get("my_tokenizer").create();
tokenizer.setReader(new StringReader("뿌리가 깊은 나무"));
tokenizer.setReader(new StringReader(""));
assertTokenStreamContents(tokenizer, new String[] {"뿌리", "", "", "", "나무"});
tokenizer.setReader(new StringReader("가늠표"));
assertTokenStreamContents(tokenizer, new String[] {"가늠표", "가늠", ""});

View File

@ -327,8 +327,8 @@ setup:
---
"Composite aggregation and array size":
- skip:
version: " - 6.99.99"
reason: starting in 7.0 the composite sources do not allocate arrays eagerly.
version: " - 6.3.99"
reason: starting in 6.4 the composite sources do not allocate arrays eagerly.
- do:
search:

View File

@ -50,8 +50,8 @@ class BinaryValuesSource extends SingleDimensionValuesSource<BytesRef> {
BinaryValuesSource(BigArrays bigArrays, LongConsumer breakerConsumer,
MappedFieldType fieldType, CheckedFunction<LeafReaderContext, SortedBinaryDocValues, IOException> docValuesFunc,
DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, missing, size, reverseMul);
DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, size, reverseMul);
this.breakerConsumer = breakerConsumer;
this.docValuesFunc = docValuesFunc;
this.values = bigArrays.newObjectArray(Math.min(size, 100));

View File

@ -271,7 +271,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::globalOrdinalsValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -288,7 +287,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::bytesValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -304,7 +302,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::bytesValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -318,7 +315,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::doubleValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -337,7 +333,6 @@ final class CompositeAggregator extends BucketsAggregator {
rounding,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);

View File

@ -23,8 +23,6 @@ import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryShardException;
@ -42,15 +40,12 @@ import java.util.Objects;
* A {@link ValuesSource} builder for {@link CompositeAggregationBuilder}
*/
public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSourceBuilder<AB>> implements Writeable, ToXContentFragment {
private static final DeprecationLogger DEPRECATION_LOGGER =
new DeprecationLogger(Loggers.getLogger(CompositeValuesSourceBuilder.class));
protected final String name;
private String field = null;
private Script script = null;
private ValueType valueType = null;
private boolean missingBucket = false;
private Object missing = null;
private SortOrder order = SortOrder.ASC;
private String format = null;
@ -72,12 +67,15 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
if (in.readBoolean()) {
this.valueType = ValueType.readFromStream(in);
}
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
this.missingBucket = in.readBoolean();
} else {
this.missingBucket = false;
}
this.missing = in.readGenericValue();
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
// skip missing value
in.readGenericValue();
}
this.order = SortOrder.readFromStream(in);
if (in.getVersion().onOrAfter(Version.V_6_3_0)) {
this.format = in.readOptionalString();
@ -103,7 +101,9 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeBoolean(missingBucket);
}
out.writeGenericValue(missing);
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeGenericValue(null);
}
order.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_6_3_0)) {
out.writeOptionalString(format);
@ -125,9 +125,6 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
builder.field("script", script);
}
builder.field("missing_bucket", missingBucket);
if (missing != null) {
builder.field("missing", missing);
}
if (valueType != null) {
builder.field("value_type", valueType.getPreferredName());
}
@ -142,7 +139,7 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
@Override
public final int hashCode() {
return Objects.hash(field, missingBucket, missing, script, valueType, order, format, innerHashCode());
return Objects.hash(field, missingBucket, script, valueType, order, format, innerHashCode());
}
protected abstract int innerHashCode();
@ -158,7 +155,6 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
Objects.equals(script, that.script()) &&
Objects.equals(valueType, that.valueType()) &&
Objects.equals(missingBucket, that.missingBucket()) &&
Objects.equals(missing, that.missing()) &&
Objects.equals(order, that.order()) &&
Objects.equals(format, that.format()) &&
innerEquals(that);
@ -229,28 +225,6 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
return valueType;
}
/**
* Sets the value to use when the source finds a missing value in a
* document.
*
* @deprecated Use {@link #missingBucket(boolean)} instead.
*/
@SuppressWarnings("unchecked")
@Deprecated
public AB missing(Object missing) {
if (missing == null) {
throw new IllegalArgumentException("[missing] must not be null");
}
DEPRECATION_LOGGER.deprecated("[missing] is deprecated. Please use [missing_bucket] instead.");
this.missing = missing;
return (AB) this;
}
@Deprecated
public Object missing() {
return missing;
}
/**
* If true an explicit `null bucket will represent documents with missing values.
*/
@ -328,18 +302,14 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
public final CompositeValuesSourceConfig build(SearchContext context) throws IOException {
ValuesSourceConfig<?> config = ValuesSourceConfig.resolve(context.getQueryShardContext(),
valueType, field, script, missing, null, format);
valueType, field, script, null,null, format);
if (config.unmapped() && field != null && missing == null && missingBucket == false) {
if (config.unmapped() && field != null && missingBucket == false) {
// this source cannot produce any values so we refuse to build
// since composite buckets are not created on null values by default.
throw new QueryShardException(context.getQueryShardContext(),
"failed to find field [" + field + "] and [missing_bucket] is not set");
}
if (missingBucket && missing != null) {
throw new QueryShardException(context.getQueryShardContext(),
"cannot use [missing] option in conjunction with [missing_bucket]");
}
return innerBuild(context, config);
}
}

View File

@ -32,7 +32,6 @@ class CompositeValuesSourceConfig {
private final ValuesSource vs;
private final DocValueFormat format;
private final int reverseMul;
private final Object missing;
private final boolean missingBucket;
/**
@ -42,18 +41,15 @@ class CompositeValuesSourceConfig {
* @param vs The underlying {@link ValuesSource}.
* @param format The {@link DocValueFormat} of this source.
* @param order The sort order associated with this source.
* @param missing The missing value or null if documents with missing value should be ignored.
*/
CompositeValuesSourceConfig(String name, @Nullable MappedFieldType fieldType, ValuesSource vs, DocValueFormat format,
SortOrder order, boolean missingBucket, @Nullable Object missing) {
SortOrder order, boolean missingBucket) {
this.name = name;
this.fieldType = fieldType;
this.vs = vs;
this.format = format;
this.reverseMul = order == SortOrder.ASC ? 1 : -1;
this.missingBucket = missingBucket;
assert missingBucket == false || missing == null;
this.missing = missing;
}
/**
@ -85,13 +81,6 @@ class CompositeValuesSourceConfig {
return format;
}
/**
* The missing value for this configuration or null if documents with missing value should be ignored.
*/
Object missing() {
return missing;
}
/**
* If true, an explicit `null bucket represents documents with missing values.
*/

View File

@ -38,8 +38,6 @@ class CompositeValuesSourceParserHelper {
ValueType targetValueType) {
objectParser.declareField(VB::field, XContentParser::text,
new ParseField("field"), ObjectParser.ValueType.STRING);
objectParser.declareField(VB::missing, XContentParser::objectText,
new ParseField("missing"), ObjectParser.ValueType.VALUE);
objectParser.declareBoolean(VB::missingBucket, new ParseField("missing_bucket"));
objectParser.declareField(VB::valueType, p -> {

View File

@ -226,7 +226,7 @@ public class DateHistogramValuesSourceBuilder extends CompositeValuesSourceBuild
// is specified in the builder.
final DocValueFormat docValueFormat = format() == null ? DocValueFormat.RAW : config.format();
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
return new CompositeValuesSourceConfig(name, fieldType, vs, docValueFormat, order(), missingBucket(), missing());
return new CompositeValuesSourceConfig(name, fieldType, vs, docValueFormat, order(), missingBucket());
} else {
throw new IllegalArgumentException("invalid source, expected numeric, got " + orig.getClass().getSimpleName());
}

View File

@ -45,8 +45,8 @@ class DoubleValuesSource extends SingleDimensionValuesSource<Double> {
DoubleValuesSource(BigArrays bigArrays, MappedFieldType fieldType,
CheckedFunction<LeafReaderContext, SortedNumericDoubleValues, IOException> docValuesFunc,
DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, missing, size, reverseMul);
DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, size, reverseMul);
this.docValuesFunc = docValuesFunc;
this.bits = missingBucket ? new BitArray(bigArrays, 100) : null;
this.values = bigArrays.newDoubleArray(Math.min(size, 100), false);

View File

@ -54,8 +54,8 @@ class GlobalOrdinalValuesSource extends SingleDimensionValuesSource<BytesRef> {
GlobalOrdinalValuesSource(BigArrays bigArrays, MappedFieldType type,
CheckedFunction<LeafReaderContext, SortedSetDocValues, IOException> docValuesFunc,
DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, type, missingBucket, missing, size, reverseMul);
DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, type, missingBucket, size, reverseMul);
this.docValuesFunc = docValuesFunc;
this.values = bigArrays.newLongArray(Math.min(size, 100), false);
}

View File

@ -115,7 +115,7 @@ public class HistogramValuesSourceBuilder extends CompositeValuesSourceBuilder<H
ValuesSource.Numeric numeric = (ValuesSource.Numeric) orig;
final HistogramValuesSource vs = new HistogramValuesSource(numeric, interval);
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
return new CompositeValuesSourceConfig(name, fieldType, vs, config.format(), order(), missingBucket(), missing());
return new CompositeValuesSourceConfig(name, fieldType, vs, config.format(), order(), missingBucket());
} else {
throw new IllegalArgumentException("invalid source, expected numeric, got " + orig.getClass().getSimpleName());
}

View File

@ -56,8 +56,8 @@ class LongValuesSource extends SingleDimensionValuesSource<Long> {
LongValuesSource(BigArrays bigArrays,
MappedFieldType fieldType, CheckedFunction<LeafReaderContext, SortedNumericDocValues, IOException> docValuesFunc,
LongUnaryOperator rounding, DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, missing, size, reverseMul);
LongUnaryOperator rounding, DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, size, reverseMul);
this.bigArrays = bigArrays;
this.docValuesFunc = docValuesFunc;
this.rounding = rounding;

View File

@ -41,8 +41,6 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
protected final DocValueFormat format;
@Nullable
protected final MappedFieldType fieldType;
@Nullable
protected final Object missing;
protected final boolean missingBucket;
protected final int size;
@ -57,18 +55,15 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
* @param format The format of the source.
* @param fieldType The field type or null if the source is a script.
* @param missingBucket If true, an explicit `null bucket represents documents with missing values.
* @param missing The missing value or null if documents with missing value should be ignored.
* @param size The number of values to record.
* @param reverseMul -1 if the natural order ({@link SortOrder#ASC} should be reversed.
*/
SingleDimensionValuesSource(BigArrays bigArrays, DocValueFormat format,
@Nullable MappedFieldType fieldType, boolean missingBucket, @Nullable Object missing,
@Nullable MappedFieldType fieldType, boolean missingBucket,
int size, int reverseMul) {
assert missing == null || missingBucket == false;
this.bigArrays = bigArrays;
this.format = format;
this.fieldType = fieldType;
this.missing = missing;
this.missingBucket = missingBucket;
this.size = size;
this.reverseMul = reverseMul;
@ -147,7 +142,6 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
*/
protected boolean checkIfSortedDocsIsApplicable(IndexReader reader, MappedFieldType fieldType) {
if (fieldType == null ||
missing != null ||
(missingBucket && afterValue == null) ||
fieldType.indexOptions() == IndexOptions.NONE ||
// inverse of the natural order

View File

@ -93,6 +93,6 @@ public class TermsValuesSourceBuilder extends CompositeValuesSourceBuilder<Terms
} else {
format = config.format();
}
return new CompositeValuesSourceConfig(name, fieldType, vs, format, order(), missingBucket(), missing());
return new CompositeValuesSourceConfig(name, fieldType, vs, format, order(), missingBucket());
}
}

View File

@ -142,19 +142,6 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
createAggregatorFactory(builder, searcher);
}
public void testMissingBucket() throws Exception {
TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10))
.field("unknown")
.missingBucket(true)
.missing("MISSING");
CompositeAggregationBuilder builder = new CompositeAggregationBuilder("test", Collections.singletonList(terms));
IndexSearcher searcher = new IndexSearcher(new MultiReader());
QueryShardException exc =
expectThrows(QueryShardException.class, () -> createAggregator(builder, searcher));
assertWarnings("[missing] is deprecated. Please use [missing_bucket] instead.");
assertThat(exc.getMessage(), containsString("cannot use [missing] option in conjunction with [missing_bucket]"));
}
public void testWithKeyword() throws Exception {
final List<Map<String, List<Object>>> dataset = new ArrayList<>();
dataset.addAll(

View File

@ -224,7 +224,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
value -> value,
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);
@ -235,7 +234,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
context -> FieldData.sortableLongBitsToDoubles(DocValues.getSortedNumeric(context.reader(), fieldType.name())),
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);
@ -249,7 +247,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
context -> DocValues.getSortedSet(context.reader(), fieldType.name()),
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);
@ -261,7 +258,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
context -> FieldData.toString(DocValues.getSortedSet(context.reader(), fieldType.name())),
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);

View File

@ -46,7 +46,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -57,20 +56,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
assertNull(source.createSortedDocsProducerOrNull(reader,
new TermQuery(new Term("keyword", "toto)"))));
source = new BinaryValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
(b) -> {},
keyword,
context -> null,
DocValueFormat.RAW,
false,
"missing_value",
1,
1
);
assertNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
assertNull(source.createSortedDocsProducerOrNull(reader, null));
source = new BinaryValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
(b) -> {},
@ -78,7 +63,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
true,
null,
1,
1
);
@ -92,7 +76,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
0,
-1
);
@ -107,7 +90,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1);
assertNull(source.createSortedDocsProducerOrNull(reader, null));
@ -121,7 +103,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
keyword, context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -132,26 +113,12 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
assertNull(source.createSortedDocsProducerOrNull(reader,
new TermQuery(new Term("keyword", "toto)"))));
source = new GlobalOrdinalValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
keyword,
context -> null,
DocValueFormat.RAW,
false,
"missing_value",
1,
1
);
assertNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
assertNull(source.createSortedDocsProducerOrNull(reader, null));
source = new GlobalOrdinalValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
keyword,
context -> null,
DocValueFormat.RAW,
true,
null,
1,
1
);
@ -164,7 +131,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
-1
);
@ -178,7 +144,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -202,7 +167,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
value -> value,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -214,27 +178,12 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)"))));
LongValuesSource sourceWithMissing = new LongValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
number,
context -> null,
value -> value,
DocValueFormat.RAW,
false,
0d,
1,
1);
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, null));
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)"))));
sourceWithMissing = new LongValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
number,
context -> null,
value -> value,
DocValueFormat.RAW,
true,
null,
1,
1);
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
@ -248,7 +197,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
value -> value,
DocValueFormat.RAW,
false,
null,
1,
-1
);
@ -262,7 +210,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);