Fix missing option serialization after backport

Relates #29465
This commit is contained in:
Jim Ferenczi 2018-05-30 12:55:31 +02:00
parent 3c21e46fa3
commit f582418ada
18 changed files with 33 additions and 152 deletions

View File

@ -10,3 +10,8 @@ These `execution_hint` are removed and should be replaced by `global_ordinals`.
The dynamic cluster setting named `search.max_buckets` now defaults
to 10,000 (instead of unlimited in the previous version).
Requests that try to return more than the limit will fail with an exception.
==== `missing` option of the `composite` aggregation has been removed
The `missing` option of the `composite` aggregation, deprecated in 6.x,
has been removed. `missing_bucket` should be used instead.

View File

@ -98,7 +98,7 @@ public class NoriAnalysisTests extends ESTokenStreamTestCase {
.build();
TestAnalysis analysis = createTestAnalysis(settings);
Tokenizer tokenizer = analysis.tokenizer.get("my_tokenizer").create();
tokenizer.setReader(new StringReader("뿌리가 깊은 나무"));
tokenizer.setReader(new StringReader(""));
assertTokenStreamContents(tokenizer, new String[] {"뿌리", "", "", "", "나무"});
tokenizer.setReader(new StringReader("가늠표"));
assertTokenStreamContents(tokenizer, new String[] {"가늠표", "가늠", ""});

View File

@ -327,8 +327,8 @@ setup:
---
"Composite aggregation and array size":
- skip:
version: " - 6.99.99"
reason: starting in 7.0 the composite sources do not allocate arrays eagerly.
version: " - 6.3.99"
reason: starting in 6.4 the composite sources do not allocate arrays eagerly.
- do:
search:

View File

@ -50,8 +50,8 @@ class BinaryValuesSource extends SingleDimensionValuesSource<BytesRef> {
BinaryValuesSource(BigArrays bigArrays, LongConsumer breakerConsumer,
MappedFieldType fieldType, CheckedFunction<LeafReaderContext, SortedBinaryDocValues, IOException> docValuesFunc,
DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, missing, size, reverseMul);
DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, size, reverseMul);
this.breakerConsumer = breakerConsumer;
this.docValuesFunc = docValuesFunc;
this.values = bigArrays.newObjectArray(Math.min(size, 100));

View File

@ -271,7 +271,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::globalOrdinalsValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -288,7 +287,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::bytesValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -304,7 +302,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::bytesValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -318,7 +315,6 @@ final class CompositeAggregator extends BucketsAggregator {
vs::doubleValues,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);
@ -337,7 +333,6 @@ final class CompositeAggregator extends BucketsAggregator {
rounding,
config.format(),
config.missingBucket(),
config.missing(),
size,
reverseMul
);

View File

@ -23,8 +23,6 @@ import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryShardException;
@ -42,15 +40,12 @@ import java.util.Objects;
* A {@link ValuesSource} builder for {@link CompositeAggregationBuilder}
*/
public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSourceBuilder<AB>> implements Writeable, ToXContentFragment {
private static final DeprecationLogger DEPRECATION_LOGGER =
new DeprecationLogger(Loggers.getLogger(CompositeValuesSourceBuilder.class));
protected final String name;
private String field = null;
private Script script = null;
private ValueType valueType = null;
private boolean missingBucket = false;
private Object missing = null;
private SortOrder order = SortOrder.ASC;
private String format = null;
@ -72,12 +67,15 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
if (in.readBoolean()) {
this.valueType = ValueType.readFromStream(in);
}
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
this.missingBucket = in.readBoolean();
} else {
this.missingBucket = false;
}
this.missing = in.readGenericValue();
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
// skip missing value
in.readGenericValue();
}
this.order = SortOrder.readFromStream(in);
if (in.getVersion().onOrAfter(Version.V_6_3_0)) {
this.format = in.readOptionalString();
@ -103,7 +101,9 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeBoolean(missingBucket);
}
out.writeGenericValue(missing);
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeGenericValue(null);
}
order.writeTo(out);
if (out.getVersion().onOrAfter(Version.V_6_3_0)) {
out.writeOptionalString(format);
@ -125,9 +125,6 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
builder.field("script", script);
}
builder.field("missing_bucket", missingBucket);
if (missing != null) {
builder.field("missing", missing);
}
if (valueType != null) {
builder.field("value_type", valueType.getPreferredName());
}
@ -142,7 +139,7 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
@Override
public final int hashCode() {
return Objects.hash(field, missingBucket, missing, script, valueType, order, format, innerHashCode());
return Objects.hash(field, missingBucket, script, valueType, order, format, innerHashCode());
}
protected abstract int innerHashCode();
@ -158,7 +155,6 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
Objects.equals(script, that.script()) &&
Objects.equals(valueType, that.valueType()) &&
Objects.equals(missingBucket, that.missingBucket()) &&
Objects.equals(missing, that.missing()) &&
Objects.equals(order, that.order()) &&
Objects.equals(format, that.format()) &&
innerEquals(that);
@ -229,28 +225,6 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
return valueType;
}
/**
* Sets the value to use when the source finds a missing value in a
* document.
*
* @deprecated Use {@link #missingBucket(boolean)} instead.
*/
@SuppressWarnings("unchecked")
@Deprecated
public AB missing(Object missing) {
if (missing == null) {
throw new IllegalArgumentException("[missing] must not be null");
}
DEPRECATION_LOGGER.deprecated("[missing] is deprecated. Please use [missing_bucket] instead.");
this.missing = missing;
return (AB) this;
}
@Deprecated
public Object missing() {
return missing;
}
/**
* If true an explicit `null bucket will represent documents with missing values.
*/
@ -328,18 +302,14 @@ public abstract class CompositeValuesSourceBuilder<AB extends CompositeValuesSou
public final CompositeValuesSourceConfig build(SearchContext context) throws IOException {
ValuesSourceConfig<?> config = ValuesSourceConfig.resolve(context.getQueryShardContext(),
valueType, field, script, missing, null, format);
valueType, field, script, null,null, format);
if (config.unmapped() && field != null && missing == null && missingBucket == false) {
if (config.unmapped() && field != null && missingBucket == false) {
// this source cannot produce any values so we refuse to build
// since composite buckets are not created on null values by default.
throw new QueryShardException(context.getQueryShardContext(),
"failed to find field [" + field + "] and [missing_bucket] is not set");
}
if (missingBucket && missing != null) {
throw new QueryShardException(context.getQueryShardContext(),
"cannot use [missing] option in conjunction with [missing_bucket]");
}
return innerBuild(context, config);
}
}

View File

@ -32,7 +32,6 @@ class CompositeValuesSourceConfig {
private final ValuesSource vs;
private final DocValueFormat format;
private final int reverseMul;
private final Object missing;
private final boolean missingBucket;
/**
@ -42,18 +41,15 @@ class CompositeValuesSourceConfig {
* @param vs The underlying {@link ValuesSource}.
* @param format The {@link DocValueFormat} of this source.
* @param order The sort order associated with this source.
* @param missing The missing value or null if documents with missing value should be ignored.
*/
CompositeValuesSourceConfig(String name, @Nullable MappedFieldType fieldType, ValuesSource vs, DocValueFormat format,
SortOrder order, boolean missingBucket, @Nullable Object missing) {
SortOrder order, boolean missingBucket) {
this.name = name;
this.fieldType = fieldType;
this.vs = vs;
this.format = format;
this.reverseMul = order == SortOrder.ASC ? 1 : -1;
this.missingBucket = missingBucket;
assert missingBucket == false || missing == null;
this.missing = missing;
}
/**
@ -85,13 +81,6 @@ class CompositeValuesSourceConfig {
return format;
}
/**
* The missing value for this configuration or null if documents with missing value should be ignored.
*/
Object missing() {
return missing;
}
/**
* If true, an explicit `null bucket represents documents with missing values.
*/

View File

@ -38,8 +38,6 @@ class CompositeValuesSourceParserHelper {
ValueType targetValueType) {
objectParser.declareField(VB::field, XContentParser::text,
new ParseField("field"), ObjectParser.ValueType.STRING);
objectParser.declareField(VB::missing, XContentParser::objectText,
new ParseField("missing"), ObjectParser.ValueType.VALUE);
objectParser.declareBoolean(VB::missingBucket, new ParseField("missing_bucket"));
objectParser.declareField(VB::valueType, p -> {

View File

@ -226,7 +226,7 @@ public class DateHistogramValuesSourceBuilder extends CompositeValuesSourceBuild
// is specified in the builder.
final DocValueFormat docValueFormat = format() == null ? DocValueFormat.RAW : config.format();
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
return new CompositeValuesSourceConfig(name, fieldType, vs, docValueFormat, order(), missingBucket(), missing());
return new CompositeValuesSourceConfig(name, fieldType, vs, docValueFormat, order(), missingBucket());
} else {
throw new IllegalArgumentException("invalid source, expected numeric, got " + orig.getClass().getSimpleName());
}

View File

@ -45,8 +45,8 @@ class DoubleValuesSource extends SingleDimensionValuesSource<Double> {
DoubleValuesSource(BigArrays bigArrays, MappedFieldType fieldType,
CheckedFunction<LeafReaderContext, SortedNumericDoubleValues, IOException> docValuesFunc,
DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, missing, size, reverseMul);
DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, size, reverseMul);
this.docValuesFunc = docValuesFunc;
this.bits = missingBucket ? new BitArray(bigArrays, 100) : null;
this.values = bigArrays.newDoubleArray(Math.min(size, 100), false);

View File

@ -54,8 +54,8 @@ class GlobalOrdinalValuesSource extends SingleDimensionValuesSource<BytesRef> {
GlobalOrdinalValuesSource(BigArrays bigArrays, MappedFieldType type,
CheckedFunction<LeafReaderContext, SortedSetDocValues, IOException> docValuesFunc,
DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, type, missingBucket, missing, size, reverseMul);
DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, type, missingBucket, size, reverseMul);
this.docValuesFunc = docValuesFunc;
this.values = bigArrays.newLongArray(Math.min(size, 100), false);
}

View File

@ -115,7 +115,7 @@ public class HistogramValuesSourceBuilder extends CompositeValuesSourceBuilder<H
ValuesSource.Numeric numeric = (ValuesSource.Numeric) orig;
final HistogramValuesSource vs = new HistogramValuesSource(numeric, interval);
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
return new CompositeValuesSourceConfig(name, fieldType, vs, config.format(), order(), missingBucket(), missing());
return new CompositeValuesSourceConfig(name, fieldType, vs, config.format(), order(), missingBucket());
} else {
throw new IllegalArgumentException("invalid source, expected numeric, got " + orig.getClass().getSimpleName());
}

View File

@ -56,8 +56,8 @@ class LongValuesSource extends SingleDimensionValuesSource<Long> {
LongValuesSource(BigArrays bigArrays,
MappedFieldType fieldType, CheckedFunction<LeafReaderContext, SortedNumericDocValues, IOException> docValuesFunc,
LongUnaryOperator rounding, DocValueFormat format, boolean missingBucket, Object missing, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, missing, size, reverseMul);
LongUnaryOperator rounding, DocValueFormat format, boolean missingBucket, int size, int reverseMul) {
super(bigArrays, format, fieldType, missingBucket, size, reverseMul);
this.bigArrays = bigArrays;
this.docValuesFunc = docValuesFunc;
this.rounding = rounding;

View File

@ -41,8 +41,6 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
protected final DocValueFormat format;
@Nullable
protected final MappedFieldType fieldType;
@Nullable
protected final Object missing;
protected final boolean missingBucket;
protected final int size;
@ -57,18 +55,15 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
* @param format The format of the source.
* @param fieldType The field type or null if the source is a script.
* @param missingBucket If true, an explicit `null bucket represents documents with missing values.
* @param missing The missing value or null if documents with missing value should be ignored.
* @param size The number of values to record.
* @param reverseMul -1 if the natural order ({@link SortOrder#ASC} should be reversed.
*/
SingleDimensionValuesSource(BigArrays bigArrays, DocValueFormat format,
@Nullable MappedFieldType fieldType, boolean missingBucket, @Nullable Object missing,
@Nullable MappedFieldType fieldType, boolean missingBucket,
int size, int reverseMul) {
assert missing == null || missingBucket == false;
this.bigArrays = bigArrays;
this.format = format;
this.fieldType = fieldType;
this.missing = missing;
this.missingBucket = missingBucket;
this.size = size;
this.reverseMul = reverseMul;
@ -147,7 +142,6 @@ abstract class SingleDimensionValuesSource<T extends Comparable<T>> implements R
*/
protected boolean checkIfSortedDocsIsApplicable(IndexReader reader, MappedFieldType fieldType) {
if (fieldType == null ||
missing != null ||
(missingBucket && afterValue == null) ||
fieldType.indexOptions() == IndexOptions.NONE ||
// inverse of the natural order

View File

@ -93,6 +93,6 @@ public class TermsValuesSourceBuilder extends CompositeValuesSourceBuilder<Terms
} else {
format = config.format();
}
return new CompositeValuesSourceConfig(name, fieldType, vs, format, order(), missingBucket(), missing());
return new CompositeValuesSourceConfig(name, fieldType, vs, format, order(), missingBucket());
}
}

View File

@ -142,19 +142,6 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
createAggregatorFactory(builder, searcher);
}
public void testMissingBucket() throws Exception {
TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10))
.field("unknown")
.missingBucket(true)
.missing("MISSING");
CompositeAggregationBuilder builder = new CompositeAggregationBuilder("test", Collections.singletonList(terms));
IndexSearcher searcher = new IndexSearcher(new MultiReader());
QueryShardException exc =
expectThrows(QueryShardException.class, () -> createAggregator(builder, searcher));
assertWarnings("[missing] is deprecated. Please use [missing_bucket] instead.");
assertThat(exc.getMessage(), containsString("cannot use [missing] option in conjunction with [missing_bucket]"));
}
public void testWithKeyword() throws Exception {
final List<Map<String, List<Object>>> dataset = new ArrayList<>();
dataset.addAll(

View File

@ -224,7 +224,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
value -> value,
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);
@ -235,7 +234,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
context -> FieldData.sortableLongBitsToDoubles(DocValues.getSortedNumeric(context.reader(), fieldType.name())),
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);
@ -249,7 +247,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
context -> DocValues.getSortedSet(context.reader(), fieldType.name()),
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);
@ -261,7 +258,6 @@ public class CompositeValuesCollectorQueueTests extends AggregatorTestCase {
context -> FieldData.toString(DocValues.getSortedSet(context.reader(), fieldType.name())),
DocValueFormat.RAW,
missingBucket,
null,
size,
1
);

View File

@ -46,7 +46,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -57,20 +56,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
assertNull(source.createSortedDocsProducerOrNull(reader,
new TermQuery(new Term("keyword", "toto)"))));
source = new BinaryValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
(b) -> {},
keyword,
context -> null,
DocValueFormat.RAW,
false,
"missing_value",
1,
1
);
assertNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
assertNull(source.createSortedDocsProducerOrNull(reader, null));
source = new BinaryValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
(b) -> {},
@ -78,7 +63,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
true,
null,
1,
1
);
@ -92,7 +76,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
0,
-1
);
@ -107,7 +90,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1);
assertNull(source.createSortedDocsProducerOrNull(reader, null));
@ -121,7 +103,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
keyword, context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -132,26 +113,12 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
assertNull(source.createSortedDocsProducerOrNull(reader,
new TermQuery(new Term("keyword", "toto)"))));
source = new GlobalOrdinalValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
keyword,
context -> null,
DocValueFormat.RAW,
false,
"missing_value",
1,
1
);
assertNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
assertNull(source.createSortedDocsProducerOrNull(reader, null));
source = new GlobalOrdinalValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
keyword,
context -> null,
DocValueFormat.RAW,
true,
null,
1,
1
);
@ -164,7 +131,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
-1
);
@ -178,7 +144,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -202,7 +167,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
value -> value,
DocValueFormat.RAW,
false,
null,
1,
1
);
@ -214,27 +178,12 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)"))));
LongValuesSource sourceWithMissing = new LongValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
number,
context -> null,
value -> value,
DocValueFormat.RAW,
false,
0d,
1,
1);
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, null));
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)"))));
sourceWithMissing = new LongValuesSource(
BigArrays.NON_RECYCLING_INSTANCE,
number,
context -> null,
value -> value,
DocValueFormat.RAW,
true,
null,
1,
1);
assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery()));
@ -248,7 +197,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
value -> value,
DocValueFormat.RAW,
false,
null,
1,
-1
);
@ -262,7 +210,6 @@ public class SingleDimensionValuesSourceTests extends ESTestCase {
context -> null,
DocValueFormat.RAW,
false,
null,
1,
1
);