honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-09 14:35:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix IncludeExclude parsing

Browse Source 
`include` / `exclude` in terms / sig-terms aggs seems completely broken
and massively untested. This commit makes the TermsTests pass again that
randomly use `include` / `exclude`. This class must be tested individually
and we need real integ tests that use xcontent that use this feature.
This commit is contained in:
Simon Willnauer 2016-12-11 09:55:53 +01:00
parent e6b10ca4db
commit 20ff703e07
1 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
core/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/support/IncludeExclude.java
View File

@ -103,8 +103,11 @@ public class IncludeExclude implements Writeable, ToXContent {
String currentFieldName = null; String currentFieldName = null;
Integer partition = null, numPartitions = null; Integer partition = null, numPartitions = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else
// This "include":{"pattern":"foo.*"} syntax is undocumented since 2.0 // This "include":{"pattern":"foo.*"} syntax is undocumented since 2.0
// Regexes should be "include":"foo.*" // Regexes should be "include":"foo.*"
if (parseFieldMatcher.match(currentFieldName, PATTERN_FIELD)) { if (parseFieldMatcher.match(currentFieldName, PATTERN_FIELD)) {
return new IncludeExclude(parser.text(), null); return new IncludeExclude(parser.text(), null);
} else if (parseFieldMatcher.match(currentFieldName, NUM_PARTITIONS_FIELD)) { } else if (parseFieldMatcher.match(currentFieldName, NUM_PARTITIONS_FIELD)) {
@ -146,12 +149,10 @@ public class IncludeExclude implements Writeable, ToXContent {
// in the index. // in the index.
public abstract static class LongFilter { public abstract static class LongFilter {
public abstract boolean accept(long value); public abstract boolean accept(long value);
}
public class PartitionedLongFilter extends LongFilter {
private final ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES);
}
public class PartitionedLongFilter extends LongFilter {
@Override @Override
public boolean accept(long value) { public boolean accept(long value) {
// hash the value to keep even distributions // hash the value to keep even distributions
@ -159,8 +160,8 @@ public class IncludeExclude implements Writeable, ToXContent {
return Math.floorMod(hashCode, incNumPartitions) == incZeroBasedPartition; return Math.floorMod(hashCode, incNumPartitions) == incZeroBasedPartition;
} }
} }
public static class SetBackedLongFilter extends LongFilter { public static class SetBackedLongFilter extends LongFilter {
private LongSet valids; private LongSet valids;
private LongSet invalids; private LongSet invalids;
@ -197,8 +198,8 @@ public class IncludeExclude implements Writeable, ToXContent {
public boolean accept(BytesRef value) { public boolean accept(BytesRef value) {
return Math.floorMod(value.hashCode(), incNumPartitions) == incZeroBasedPartition; return Math.floorMod(value.hashCode(), incNumPartitions) == incZeroBasedPartition;
} }
} }
static class AutomatonBackedStringFilter extends StringFilter { static class AutomatonBackedStringFilter extends StringFilter {
private final ByteRunAutomaton runAutomaton; private final ByteRunAutomaton runAutomaton;
@ -361,7 +362,7 @@ public class IncludeExclude implements Writeable, ToXContent {
this.include = null; this.include = null;
this.exclude = null; this.exclude = null;
this.incZeroBasedPartition = 0; this.incZeroBasedPartition = 0;
this.incNumPartitions = 0; this.incNumPartitions = 0;
this.includeValues = includeValues; this.includeValues = includeValues;
this.excludeValues = excludeValues; this.excludeValues = excludeValues;
} }
@ -388,11 +389,11 @@ public class IncludeExclude implements Writeable, ToXContent {
this.exclude = null; this.exclude = null;
this.includeValues = null; this.includeValues = null;
this.excludeValues = null; this.excludeValues = null;
} }
/** /**
* Read from a stream. * Read from a stream.
*/ */
@ -580,7 +581,7 @@ public class IncludeExclude implements Writeable, ToXContent {
public boolean isPartitionBased() { public boolean isPartitionBased() {
return incNumPartitions > 0; return incNumPartitions > 0;
} }
private Automaton toAutomaton() { private Automaton toAutomaton() {
Automaton a = null; Automaton a = null;
if (include != null) { if (include != null) {
@ -629,16 +630,16 @@ public class IncludeExclude implements Writeable, ToXContent {
if (isPartitionBased()){ if (isPartitionBased()){
return new PartitionedOrdinalsFilter(); return new PartitionedOrdinalsFilter();
} }
return new TermListBackedOrdinalsFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format)); return new TermListBackedOrdinalsFilter(parseForDocValues(includeValues, format), parseForDocValues(excludeValues, format));
} }
public LongFilter convertToLongFilter(DocValueFormat format) { public LongFilter convertToLongFilter(DocValueFormat format) {
if(isPartitionBased()){ if(isPartitionBased()){
return new PartitionedLongFilter(); return new PartitionedLongFilter();
} }
int numValids = includeValues == null ? 0 : includeValues.size(); int numValids = includeValues == null ? 0 : includeValues.size();
int numInvalids = excludeValues == null ? 0 : excludeValues.size(); int numInvalids = excludeValues == null ? 0 : excludeValues.size();
SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids); SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids);
@ -659,7 +660,7 @@ public class IncludeExclude implements Writeable, ToXContent {
if(isPartitionBased()){ if(isPartitionBased()){
return new PartitionedLongFilter(); return new PartitionedLongFilter();
} }
int numValids = includeValues == null ? 0 : includeValues.size(); int numValids = includeValues == null ? 0 : includeValues.size();
int numInvalids = excludeValues == null ? 0 : excludeValues.size(); int numInvalids = excludeValues == null ? 0 : excludeValues.size();
SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids); SetBackedLongFilter result = new SetBackedLongFilter(numValids, numInvalids);
@ -682,22 +683,21 @@ public class IncludeExclude implements Writeable, ToXContent {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (include != null) { if (include != null) {
builder.field(INCLUDE_FIELD.getPreferredName(), include.getOriginalString()); builder.field(INCLUDE_FIELD.getPreferredName(), include.getOriginalString());
} } else if (includeValues != null) {
if (includeValues != null) {
builder.startArray(INCLUDE_FIELD.getPreferredName()); builder.startArray(INCLUDE_FIELD.getPreferredName());
for (BytesRef value : includeValues) { for (BytesRef value : includeValues) {
builder.value(value.utf8ToString()); builder.value(value.utf8ToString());
} }
builder.endArray(); builder.endArray();
} } else if (isPartitionBased()) {
if (isPartitionBased()) { builder.startObject(INCLUDE_FIELD.getPreferredName());
builder.field(PARTITION_FIELD.getPreferredName(), incZeroBasedPartition); builder.field(PARTITION_FIELD.getPreferredName(), incZeroBasedPartition);
builder.field(NUM_PARTITIONS_FIELD.getPreferredName(), incNumPartitions); builder.field(NUM_PARTITIONS_FIELD.getPreferredName(), incNumPartitions);
builder.endObject();
} }
if (exclude != null) { if (exclude != null) {
builder.field(EXCLUDE_FIELD.getPreferredName(), exclude.getOriginalString()); builder.field(EXCLUDE_FIELD.getPreferredName(), exclude.getOriginalString());
} } else if (excludeValues != null) {
if (excludeValues != null) {
builder.startArray(EXCLUDE_FIELD.getPreferredName()); builder.startArray(EXCLUDE_FIELD.getPreferredName());
for (BytesRef value : excludeValues) { for (BytesRef value : excludeValues) {
builder.value(value.utf8ToString()); builder.value(value.utf8ToString());