honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor BytesValues to be reused as the interface for HashedBytesValues and remove HashBytesValues.

This commit is contained in:
Simon Willnauer 2013-03-27 13:45:20 +01:00
parent 129f02623b
commit 7f81469137
26 changed files with 676 additions and 1750 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java
View File

@ -48,11 +48,9 @@ public interface AtomicFieldData<Script extends ScriptDocValues> {
* Use a non thread safe (lightweight) view of the values as bytes.
*/
BytesValues getBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
HashedBytesValues getHashedBytesValues();
BytesValues getHashedBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as strings.
@ -75,11 +73,9 @@ public interface AtomicFieldData<Script extends ScriptDocValues> {
* Use a non thread safe (lightweight) view of the values as bytes.
*/
BytesValues.WithOrdinals getBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
HashedBytesValues.WithOrdinals getHashedBytesValues();
BytesValues.WithOrdinals getHashedBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as strings.

16
src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java
View File

@ -21,8 +21,20 @@ package org.elasticsearch.index.fielddata;
/**
*/
public interface AtomicGeoPointFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
public abstract class AtomicGeoPointFieldData<Script extends ScriptDocValues> implements AtomicFieldData<Script> {
GeoPointValues getGeoPointValues();
public abstract GeoPointValues getGeoPointValues();
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public BytesValues getHashedBytesValues() {
return getBytesValues();
}
}

23
src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java
View File

@ -21,10 +21,27 @@ package org.elasticsearch.index.fielddata;
/**
*/
public interface AtomicNumericFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
public abstract class AtomicNumericFieldData<Script extends ScriptDocValues> implements AtomicFieldData<Script> {
LongValues getLongValues();
public abstract LongValues getLongValues();
public abstract DoubleValues getDoubleValues();
@Override
public BytesValues getHashedBytesValues() {
return getBytesValues();
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());
}
DoubleValues getDoubleValues();
}

204
src/main/java/org/elasticsearch/index/fielddata/BytesValues.java
View File

@ -24,8 +24,6 @@ import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
@ -74,34 +72,29 @@ public abstract class BytesValues {
* Note, the bytes are not "safe".
*/
public abstract BytesRef getValueScratch(int docId, BytesRef ret);
/**
* Returns an array wrapping all the bytes values for a doc. The content is guaranteed not to be shared.
* Fills the given spare for the given doc ID and returns the hashcode of the reference as defined by
* {@link BytesRef#hashCode()}
*/
public abstract BytesRefArrayRef getValues(int docId);
public int getValueHashed(int docId, BytesRef spare) {
return getValueScratch(docId, spare).hashCode();
}
/**
* Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
*/
public abstract Iter getIter(int docId);
public abstract Iter getIter(int docId); // TODO: maybe this should return null for no values so we can safe one call?
/**
* Go over all the possible values in their BytesRef format for a specific doc.
*/
public abstract void forEachValueInDoc(int docId, ValueInDocProc proc);
public static interface ValueInDocProc {
void onValue(int docId, BytesRef value);
void onMissing(int docId);
}
public static interface Iter {
boolean hasNext();
BytesRef next();
int hash();
public static class Empty implements Iter {
@ -116,15 +109,22 @@ public abstract class BytesValues {
public BytesRef next() {
throw new ElasticSearchIllegalStateException();
}
@Override
public int hash() {
return 0;
}
}
public final static class Single implements Iter {
public static class Single implements Iter {
public BytesRef value;
public boolean done;
protected BytesRef value;
protected int ord;
protected boolean done;
public Single reset(BytesRef value) {
public Single reset(BytesRef value, int ord) {
this.value = value;
this.ord = ord;
this.done = false;
return this;
}
@ -140,11 +140,15 @@ public abstract class BytesValues {
done = true;
return value;
}
public int hash() {
return value.hashCode();
}
}
static final class Multi implements Iter {
static class Multi implements Iter {
private int ord;
protected int ord;
private BytesValues.WithOrdinals withOrds;
private Ordinals.Docs.Iter ordsIter;
private final BytesRef scratch = new BytesRef();
@ -171,6 +175,10 @@ public abstract class BytesValues {
ord = ordsIter.next();
return scratch;
}
public int hash() {
return scratch.hashCode();
}
}
}
@ -185,21 +193,11 @@ public abstract class BytesValues {
return false;
}
@Override
public BytesRefArrayRef getValues(int docId) {
return BytesRefArrayRef.EMPTY;
}
@Override
public Iter getIter(int docId) {
return Iter.Empty.INSTANCE;
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
proc.onMissing(docId);
}
@Override
public BytesRef getValueScratch(int docId, BytesRef ret) {
ret.length = 0;
@ -214,7 +212,6 @@ public abstract class BytesValues {
private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
private final ValueIter valueIter = new ValueIter();
private final Proc proc = new Proc();
public StringBased(StringValues values) {
super(values.isMultiValued());
@ -237,31 +234,11 @@ public abstract class BytesValues {
return ret;
}
@Override
public BytesRefArrayRef getValues(int docId) {
StringArrayRef arrayRef = values.getValues(docId);
int size = arrayRef.size();
if (size == 0) {
return BytesRefArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = arrayRef.start; i < arrayRef.end; i++) {
String value = arrayRef.values[i];
arrayScratch.values[arrayScratch.end++] = value == null ? null : new BytesRef(value);
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return valueIter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
public static class ValueIter implements Iter {
private final BytesRef scratch = new BytesRef();
@ -282,29 +259,13 @@ public abstract class BytesValues {
scratch.copyChars(iter.next());
return scratch;
}
}
public static class Proc implements StringValues.ValueInDocProc {
private final BytesRef scratch = new BytesRef();
private BytesValues.ValueInDocProc proc;
public Proc reset(BytesValues.ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, String value) {
scratch.copyChars(value);
proc.onValue(docId, scratch);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
public int hash() {
return scratch.hashCode();
}
}
}
/**
@ -327,66 +288,29 @@ public abstract class BytesValues {
public BytesRef getValueByOrd(int ord) {
return getValueScratchByOrd(ord, scratch);
}
protected Iter.Multi newMultiIter() {
assert this.isMultiValued();
return new Iter.Multi(this);
}
protected Iter.Single newSingleIter() {
assert !this.isMultiValued();
return new Iter.Single();
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public BytesRefArrayRef getValues(int docId) {
assert !isMultiValued();
int ord = ordinals.getOrd(docId);
if (ord == 0) return BytesRefArrayRef.EMPTY;
arrayScratch.values[0] = getSafeValueByOrd(ord);
arrayScratch.end = 1;
arrayScratch.start = 0;
return arrayScratch;
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
assert !isMultiValued();
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
} else {
proc.onValue(docId, getValue(docId));
}
}
protected BytesRefArrayRef getValuesMulti(int docId) {
assert isMultiValued();
IntArrayRef ords = ordinals.getOrds(docId);
int size = ords.size();
if (size == 0) {
return BytesRefArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = ords.start; i < ords.end; i++) {
arrayScratch.values[arrayScratch.end++] = getValueScratchByOrd(ords.values[i], new BytesRef());
}
return arrayScratch;
}
protected void forEachValueInDocMulti(int docId, ValueInDocProc proc) {
assert isMultiValued();
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
getValueScratchByOrd(ord, scratch);
proc.onValue(docId, scratch);
} while ((ord = iter.next()) != 0);
}
@Override
public BytesRef getValue(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return null;
final int ord = ordinals.getOrd(docId);
if (ord == 0) {
return null;
}
return getValueScratchByOrd(ord, scratch);
}
@ -412,47 +336,17 @@ public abstract class BytesValues {
super(ordinals);
}
@Override
public BytesRef getValueByOrd(int ord) {
return null;
}
@Override
public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
ret.length = 0;
return ret;
}
@Override
public BytesRef getSafeValueByOrd(int ord) {
return null;
}
@Override
public boolean hasValue(int docId) {
return false;
}
@Override
public BytesRefArrayRef getValues(int docId) {
return BytesRefArrayRef.EMPTY;
}
@Override
public Iter getIter(int docId) {
return Iter.Empty.INSTANCE;
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
proc.onMissing(docId);
}
@Override
public BytesRef getValueScratch(int docId, BytesRef ret) {
ret.length = 0;
return ret;
}
}
}
}

420
src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java
View File

@ -1,420 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public interface HashedBytesValues {
static final HashedBytesValues EMPTY = new Empty();
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
/**
* Converts the provided bytes to "safe" ones from a "non" safe call made (if needed).
*/
HashedBytesRef makeSafe(HashedBytesRef bytes);
/**
* Returns a bytes value for a docId. Note, the content of it might be shared across invocation,
* call {@link #makeSafe(org.elasticsearch.common.lucene.HashedBytesRef)} to converts it to a "safe"
* option (if needed).
*/
HashedBytesRef getValue(int docId);
/**
* Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
*/
Iter getIter(int docId);
/**
* Go over all the possible values in their BytesRef format for a specific doc.
*/
void forEachValueInDoc(int docId, ValueInDocProc proc);
public static interface ValueInDocProc {
void onValue(int docId, HashedBytesRef value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
HashedBytesRef next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public HashedBytesRef next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public HashedBytesRef value;
public boolean done;
public Single reset(HashedBytesRef value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public HashedBytesRef next() {
assert !done;
done = true;
return value;
}
}
}
static class Empty implements HashedBytesValues {
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return false;
}
@Override
public HashedBytesRef getValue(int docId) {
return null;
}
@Override
public Iter getIter(int docId) {
return Iter.Empty.INSTANCE;
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
proc.onMissing(docId);
}
@Override
public HashedBytesRef makeSafe(HashedBytesRef bytes) {
//todo maybe better to throw an excepiton here as the only value this method accepts is a scratch value...
//todo ...extracted from this ByteValues, in our case, there are not values, so this should never be called!?!?
return HashedBytesRef.deepCopyOf(bytes);
}
}
/**
* A {@link BytesValues} based implementation.
*/
static class BytesBased implements HashedBytesValues {
private final BytesValues values;
protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private final ValueIter valueIter = new ValueIter();
private final Proc proc = new Proc();
public BytesBased(BytesValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public HashedBytesRef makeSafe(HashedBytesRef bytes) {
return new HashedBytesRef(values.makeSafe(bytes.bytes), bytes.hash);
}
@Override
public HashedBytesRef getValue(int docId) {
BytesRef value = values.getValue(docId);
if (value == null) return null;
scratch.bytes = value;
return scratch.resetHashCode();
}
@Override
public Iter getIter(int docId) {
return valueIter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
static class ValueIter implements Iter {
private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private BytesValues.Iter iter;
public ValueIter reset(BytesValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public HashedBytesRef next() {
scratch.bytes = iter.next();
return scratch.resetHashCode();
}
}
static class Proc implements BytesValues.ValueInDocProc {
private final HashedBytesRef scratch = new HashedBytesRef();
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, BytesRef value) {
scratch.bytes = value;
proc.onValue(docId, scratch.resetHashCode());
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
static class StringBased implements HashedBytesValues {
private final StringValues values;
protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private final ValueIter valueIter = new ValueIter();
private final Proc proc = new Proc();
public StringBased(StringValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public HashedBytesRef makeSafe(HashedBytesRef bytes) {
// we use scratch to provide it, so just need to copy it over to a new instance
return new HashedBytesRef(bytes.bytes, bytes.hash);
}
@Override
public HashedBytesRef getValue(int docId) {
String value = values.getValue(docId);
if (value == null) return null;
scratch.bytes.copyChars(value);
return scratch.resetHashCode();
}
@Override
public Iter getIter(int docId) {
return valueIter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
static class ValueIter implements Iter {
private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private StringValues.Iter iter;
public ValueIter reset(StringValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public HashedBytesRef next() {
scratch.bytes.copyChars(iter.next());
return scratch.resetHashCode();
}
}
static class Proc implements StringValues.ValueInDocProc {
private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, String value) {
scratch.bytes.copyChars(value);
proc.onValue(docId, scratch);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
public interface WithOrdinals extends HashedBytesValues {
Ordinals.Docs ordinals();
HashedBytesRef getValueByOrd(int ord);
HashedBytesRef getSafeValueByOrd(int ord);
public static class Empty extends HashedBytesValues.Empty implements WithOrdinals {
private final Ordinals ordinals;
public Empty(Ordinals ordinals) {
this.ordinals = ordinals;
}
@Override
public Ordinals.Docs ordinals() {
return ordinals.ordinals();
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
return null;
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return null;
}
}
static class BytesBased extends HashedBytesValues.BytesBased implements WithOrdinals {
private final BytesValues.WithOrdinals values;
public BytesBased(BytesValues.WithOrdinals values) {
super(values);
this.values = values;
}
@Override
public Ordinals.Docs ordinals() {
return values.ordinals();
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
scratch.bytes = values.getValueByOrd(ord);
return scratch.resetHashCode();
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(values.getSafeValueByOrd(ord));
}
}
static class StringBased extends HashedBytesValues.StringBased implements WithOrdinals {
private final StringValues.WithOrdinals values;
public StringBased(StringValues.WithOrdinals values) {
super(values);
this.values = values;
}
@Override
public Ordinals.Docs ordinals() {
return values.ordinals();
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
scratch.bytes.copyChars(values.getValueByOrd(ord));
return scratch.resetHashCode();
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(new BytesRef(values.getValueByOrd(ord)));
}
}
}
}

2
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
View File

@ -61,7 +61,7 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Ind
buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
.put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
.put(Tuple.tuple("string", "paged_bytes"), new PagedBytesIndexFieldData.Builder())
.put(Tuple.tuple("string", "fst"), new FSTPackedIndexFieldData.Builder())
.put(Tuple.tuple("string", "fst"), new FSTBytesIndexFieldData.Builder())
.put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
.put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
.put(Tuple.tuple("byte", "array"), new ByteArrayIndexFieldData.Builder())

18
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java
View File

@ -128,18 +128,10 @@ public final class BytesRefValComparator extends FieldComparator<BytesRef> {
return delegate.getValueScratch(docId, ret);
}
public BytesRefArrayRef getValues(int docId) {
return delegate.getValues(docId);
}
public Iter getIter(int docId) {
return delegate.getIter(docId);
}
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
delegate.forEachValueInDoc(docId, proc);
}
}
private static final class MultiValuedBytesWrapper extends FilteredByteValues {
@ -177,16 +169,6 @@ public final class BytesRefValComparator extends FieldComparator<BytesRef> {
currentVal = iter.next();
}
return relevantVal;
/*if (reversed) {
BytesRefArrayRef ref = readerValues.getValues(docId);
if (ref.isEmpty()) {
return null;
} else {
return ref.values[ref.end - 1]; // last element is the highest value.
}
} else {
return readerValues.getValue(docId); // returns the lowest value
}*/
}
}

33
src/main/java/org/elasticsearch/index/fielddata/plain/ByteArrayAtomicFieldData.java
View File

@ -29,7 +29,7 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData {
public abstract class ByteArrayAtomicFieldData extends AtomicNumericFieldData {
public static final ByteArrayAtomicFieldData EMPTY = new Empty();
@ -89,11 +89,6 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public StringValues getStringValues() {
return StringValues.EMPTY;
@ -131,27 +126,10 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
}
return size;
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.BytesBased(getBytesValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericLong(getLongValues());
}
@Override
public LongValues getLongValues() {
return new LongValues(values, ordinals.ordinals());
@ -404,10 +382,6 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.BytesBased(getBytesValues());
}
@Override
public StringValues getStringValues() {
@ -600,11 +574,6 @@ public abstract class ByteArrayAtomicFieldData implements AtomicNumericFieldData
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());

224
src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java
View File

@ -21,12 +21,12 @@ package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.StringValues;
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
/**
@ -41,7 +41,7 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdi
private final BytesRef[] values;
protected final Ordinals ordinals;
private int[] hashes;
private volatile int[] hashes;
private long size = -1;
public ConcreteBytesRefAtomicFieldData(BytesRef[] values, Ordinals ordinals) {
@ -90,7 +90,7 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdi
}
@Override
public HashedBytesValues.WithOrdinals getHashedBytesValues() {
public BytesValues getHashedBytesValues() {
if (hashes == null) {
int[] hashes = new int[values.length];
for (int i = 0; i < values.length; i++) {
@ -99,7 +99,7 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdi
}
this.hashes = hashes;
}
return ordinals.isMultiValued() ? new HashedBytesValues.Multi(values, hashes, ordinals.ordinals()) : new HashedBytesValues.Single(values, hashes, ordinals.ordinals());
return ordinals.isMultiValued() ? new BytesValues.MultiHashed(values, ordinals.ordinals(), hashes) : new BytesValues.SingleHashed(values, ordinals.ordinals(), hashes);
}
@Override
@ -163,35 +163,55 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdi
return ret;
}
static final class Single extends BytesValues {
static class Single extends BytesValues {
private final Iter.Single iter = new Iter.Single();
private final Iter.Single iter;
Single(BytesRef[] values, Ordinals.Docs ordinals) {
super(values, ordinals);
this.iter = newSingleIter();
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(values[ord]);
return iter.reset(values[ord], ord);
}
}
static final class SingleHashed extends Single {
private final int[] hashes;
SingleHashed(BytesRef[] values, Docs ordinals, int[] hashes) {
super(values, ordinals);
this.hashes = hashes;
}
@Override
protected Iter.Single newSingleIter() {
return new Iter.Single() {
public int hash() {
return hashes[ord];
}
};
}
@Override
public int getValueHashed(int docId, BytesRef ret) {
final int ord = ordinals.getOrd(docId);
getValueScratchByOrd(ord, ret);
return hashes[ord];
}
}
static final class Multi extends BytesValues {
static class Multi extends BytesValues {
private final Iter.Multi iter;
Multi(BytesRef[] values, Ordinals.Docs ordinals) {
super(values, ordinals);
assert ordinals.isMultiValued();
this.iter = new Iter.Multi(this);
}
@Override
public BytesRefArrayRef getValues(int docId) {
return getValuesMulti(docId);
this.iter = newMultiIter();
}
@Override
@ -199,160 +219,35 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdi
return iter.reset(ordinals.getIter(docId));
}
}
static final class MultiHashed extends Multi {
private final int[] hashes;
MultiHashed(BytesRef[] values, Ordinals.Docs ordinals, int[] hashes) {
super(values, ordinals);
this.hashes = hashes;
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
forEachValueInDocMulti(docId, proc);
protected Iter.Multi newMultiIter() {
return new Iter.Multi(this) {
public int hash() {
return hashes[ord];
}
};
}
@Override
public int getValueHashed(int docId, BytesRef ret) {
final int ord = ordinals.getOrd(docId);
getValueScratchByOrd(ord, ret);
return hashes[ord];
}
}
}
static abstract class HashedBytesValues implements org.elasticsearch.index.fielddata.HashedBytesValues.WithOrdinals {
protected final BytesRef[] values;
protected final int[] hashes;
protected final Ordinals.Docs ordinals;
protected final HashedBytesRef scratch = new HashedBytesRef();
HashedBytesValues(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
this.values = values;
this.hashes = hashes;
this.ordinals = ordinals;
}
@Override
public Ordinals.Docs ordinals() {
return this.ordinals;
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
return scratch.reset(values[ord], hashes[ord]);
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(values[ord], hashes[ord]);
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public HashedBytesRef makeSafe(HashedBytesRef bytes) {
// we just need to create a copy of the bytes ref, no need
// to create a copy of the actual BytesRef, as its concrete
return new HashedBytesRef(bytes.bytes, bytes.hash);
}
@Override
public HashedBytesRef getValue(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return null;
return scratch.reset(values[ord], hashes[ord]);
}
static class Single extends HashedBytesValues {
private final Iter.Single iter = new Iter.Single();
Single(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
super(values, hashes, ordinals);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(scratch.reset(values[ord], hashes[ord]));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
} else {
proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
}
}
}
static class Multi extends HashedBytesValues {
private final ValuesIter iter;
Multi(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
super(values, hashes, ordinals);
this.iter = new ValuesIter(values, hashes);
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements Iter {
private final BytesRef[] values;
private final int[] hashes;
private Ordinals.Docs.Iter ordsIter;
private int ord;
private final HashedBytesRef scratch = new HashedBytesRef();
ValuesIter(BytesRef[] values, int[] hashes) {
this.values = values;
this.hashes = hashes;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public HashedBytesRef next() {
HashedBytesRef value = scratch.reset(values[ord], hashes[ord]);
ord = ordsIter.next();
return value;
}
}
}
}
static class Empty extends ConcreteBytesRefAtomicFieldData {
@ -385,11 +280,6 @@ public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdi
return new BytesValues.WithOrdinals.Empty(ordinals.ordinals());
}
@Override
public HashedBytesValues.WithOrdinals getHashedBytesValues() {
return new HashedBytesValues.WithOrdinals.Empty((EmptyOrdinals) ordinals);
}
@Override
public StringValues.WithOrdinals getStringValues() {
return new StringValues.WithOrdinals.Empty((EmptyOrdinals) ordinals);

26
src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java
View File

@ -30,7 +30,7 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldData {
public abstract class DoubleArrayAtomicFieldData extends AtomicNumericFieldData {
public static final DoubleArrayAtomicFieldData EMPTY = new Empty();
@ -89,11 +89,6 @@ public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldDa
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public StringValues getStringValues() {
return StringValues.EMPTY;
@ -137,11 +132,6 @@ public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldDa
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values, ordinals.ordinals());
@ -498,11 +488,6 @@ public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldDa
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.DoubleBased(getDoubleValues());
@ -685,15 +670,6 @@ public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldDa
return new ScriptDocValues.NumericDouble(getDoubleValues());
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {

118
src/main/java/org/elasticsearch/index/fielddata/plain/FSTPackedBytesAtomicFieldData.java → src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesAtomicFieldData.java
View File

@ -30,30 +30,30 @@ import org.apache.lucene.util.fst.FST.Arc;
import org.apache.lucene.util.fst.FST.BytesReader;
import org.apache.lucene.util.fst.Util;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.HashedBytesValues;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.StringValues;
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
/**
*/
public class FSTPackedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public class FSTBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
public static FSTPackedBytesAtomicFieldData empty(int numDocs) {
public static FSTBytesAtomicFieldData empty(int numDocs) {
return new Empty(numDocs);
}
// 0 ordinal in values means no value (its null)
protected final Ordinals ordinals;
private int[] hashes;
private volatile int[] hashes;
private long size = -1;
private final FST<Long> fst;
public FSTPackedBytesAtomicFieldData(FST<Long> fst, Ordinals ordinals) {
public FSTBytesAtomicFieldData(FST<Long> fst, Ordinals ordinals) {
this.ordinals = ordinals;
this.fst = fst;
}
@ -95,8 +95,19 @@ public class FSTPackedBytesAtomicFieldData implements AtomicFieldData.WithOrdina
}
@Override
public HashedBytesValues.WithOrdinals getHashedBytesValues() {
public StringValues.WithOrdinals getStringValues() {
assert fst != null;
return StringValues.BytesValuesWrapper.wrap(getBytesValues());
}
@Override
public ScriptDocValues.Strings getScriptValues() {
assert fst != null;
return new ScriptDocValues.Strings(getStringValues());
}
@Override
public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getHashedBytesValues() {
if (hashes == null) {
BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
int[] hashes = new int[ordinals.getMaxOrd()];
@ -111,19 +122,7 @@ public class FSTPackedBytesAtomicFieldData implements AtomicFieldData.WithOrdina
}
this.hashes = hashes;
}
return ordinals.isMultiValued() ? new HashedBytesValuesWithOrds.Multi(getBytesValues(), hashes) : new HashedBytesValuesWithOrds.Single(getBytesValues(), hashes);
}
@Override
public StringValues.WithOrdinals getStringValues() {
assert fst != null;
return StringValues.BytesValuesWrapper.wrap(getBytesValues());
}
@Override
public ScriptDocValues.Strings getScriptValues() {
assert fst != null;
return new ScriptDocValues.Strings(getStringValues());
return ordinals.isMultiValued() ? new BytesValues.MultiHashed(fst, ordinals.ordinals(), hashes) : new BytesValues.SingleHashed(fst, ordinals.ordinals(), hashes);
}
static abstract class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
@ -160,52 +159,94 @@ public class FSTPackedBytesAtomicFieldData implements AtomicFieldData.WithOrdina
return ret;
}
static final class Single extends BytesValues {
private final Iter.Single iter = new Iter.Single();
static class Single extends BytesValues {
private final Iter.Single iter;
Single(FST<Long> fst, Ordinals.Docs ordinals) {
super(fst, ordinals);
assert !ordinals.isMultiValued();
this.iter = newSingleIter();
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(getValueByOrd(ord));
return iter.reset(getValueByOrd(ord), ord);
}
}
static final class SingleHashed extends Single {
private final int[] hashes;
SingleHashed(FST<Long> fst, Docs ordinals, int[] hashes) {
super(fst, ordinals);
this.hashes = hashes;
}
@Override
protected Iter.Single newSingleIter() {
return new Iter.Single() {
public int hash() {
return hashes[ord];
}
};
}
@Override
public int getValueHashed(int docId, BytesRef ret) {
final int ord = ordinals.getOrd(docId);
getValueScratchByOrd(ord, ret);
return hashes[ord];
}
}
static final class Multi extends BytesValues {
static class Multi extends BytesValues {
private final Iter.Multi iter;
Multi(FST<Long> fst, Ordinals.Docs ordinals) {
super(fst, ordinals);
assert ordinals.isMultiValued();
this.iter = new Iter.Multi(this);
this.iter = newMultiIter();
}
@Override
public BytesRefArrayRef getValues(int docId) {
return getValuesMulti(docId);
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
}
static final class MultiHashed extends Multi {
private final int[] hashes;
MultiHashed(FST<Long> fst, Docs ordinals, int[] hashes) {
super(fst, ordinals);
this.hashes = hashes;
}
@Override
protected Iter.Multi newMultiIter() {
return new Iter.Multi(this) {
public int hash() {
return hashes[ord];
}
};
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
forEachValueInDocMulti(docId, proc);
public int getValueHashed(int docId, BytesRef ret) {
final int ord = ordinals.getOrd(docId);
getValueScratchByOrd(ord, ret);
return hashes[ord];
}
}
}
static class Empty extends FSTPackedBytesAtomicFieldData {
static class Empty extends FSTBytesAtomicFieldData {
Empty(int numDocs) {
super(null, new EmptyOrdinals(numDocs));
@ -231,11 +272,6 @@ public class FSTPackedBytesAtomicFieldData implements AtomicFieldData.WithOrdina
return new BytesValues.WithOrdinals.Empty(ordinals.ordinals());
}
@Override
public HashedBytesValues.WithOrdinals getHashedBytesValues() {
return new HashedBytesValuesWithOrds.Empty(ordinals);
}
@Override
public StringValues.WithOrdinals getStringValues() {
return new StringValues.WithOrdinals.Empty(ordinals);
@ -247,4 +283,8 @@ public class FSTPackedBytesAtomicFieldData implements AtomicFieldData.WithOrdina
}
}
}

14
src/main/java/org/elasticsearch/index/fielddata/plain/FSTPackedIndexFieldData.java → src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesIndexFieldData.java
View File

@ -47,17 +47,17 @@ import org.elasticsearch.index.settings.IndexSettings;
/**
*/
public class FSTPackedIndexFieldData extends AbstractIndexFieldData<FSTPackedBytesAtomicFieldData> implements IndexFieldData.WithOrdinals<FSTPackedBytesAtomicFieldData> {
public class FSTBytesIndexFieldData extends AbstractIndexFieldData<FSTBytesAtomicFieldData> implements IndexFieldData.WithOrdinals<FSTBytesAtomicFieldData> {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
return new FSTPackedIndexFieldData(index, indexSettings, fieldNames, type, cache);
return new FSTBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
}
}
public FSTPackedIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
public FSTBytesIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
super(index, indexSettings, fieldNames, fieldDataType, cache);
}
@ -67,7 +67,7 @@ public class FSTPackedIndexFieldData extends AbstractIndexFieldData<FSTPackedByt
}
@Override
public FSTPackedBytesAtomicFieldData load(AtomicReaderContext context) {
public FSTBytesAtomicFieldData load(AtomicReaderContext context) {
try {
return cache.load(context, this);
} catch (Throwable e) {
@ -80,12 +80,12 @@ public class FSTPackedIndexFieldData extends AbstractIndexFieldData<FSTPackedByt
}
@Override
public FSTPackedBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
public FSTBytesAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldNames().indexName());
if (terms == null) {
return FSTPackedBytesAtomicFieldData.empty(reader.maxDoc());
return FSTBytesAtomicFieldData.empty(reader.maxDoc());
}
PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
org.apache.lucene.util.fst.Builder<Long> fstBuilder = new org.apache.lucene.util.fst.Builder<Long>(INPUT_TYPE.BYTE1, outputs);
@ -111,7 +111,7 @@ public class FSTPackedIndexFieldData extends AbstractIndexFieldData<FSTPackedByt
final Ordinals ordinals = builder.build(fieldDataType.getSettings());
return new FSTPackedBytesAtomicFieldData(fst, ordinals);
return new FSTBytesAtomicFieldData(fst, ordinals);
} finally {
builder.close();
}

27
src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayAtomicFieldData.java
View File

@ -29,7 +29,7 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldData {
public abstract class FloatArrayAtomicFieldData extends AtomicNumericFieldData {
public static final FloatArrayAtomicFieldData EMPTY = new Empty();
@ -88,11 +88,6 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public StringValues getStringValues() {
return StringValues.EMPTY;
@ -131,16 +126,6 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
return size;
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.DoubleBased(getDoubleValues());
@ -403,11 +388,6 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.DoubleBased(getDoubleValues());
@ -599,11 +579,6 @@ public abstract class FloatArrayAtomicFieldData implements AtomicNumericFieldDat
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.DoubleBased(getDoubleValues());

22
src/main/java/org/elasticsearch/index/fielddata/plain/GeoPointDoubleArrayAtomicFieldData.java
View File

@ -31,7 +31,7 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPointFieldData {
public abstract class GeoPointDoubleArrayAtomicFieldData extends AtomicGeoPointFieldData {
public static final GeoPointDoubleArrayAtomicFieldData EMPTY = new Empty();
@ -87,11 +87,6 @@ public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPoi
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public GeoPointValues getGeoPointValues() {
return GeoPointValues.EMPTY;
@ -140,11 +135,6 @@ public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPoi
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(lon, lat, ordinals.ordinals());
@ -466,11 +456,6 @@ public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPoi
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(lon, lat, set);
@ -677,11 +662,6 @@ public abstract class GeoPointDoubleArrayAtomicFieldData implements AtomicGeoPoi
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(lon, lat);

179
src/main/java/org/elasticsearch/index/fielddata/plain/HashedBytesValuesWithOrds.java
View File

@ -1,179 +0,0 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.plain.FSTPackedBytesAtomicFieldData.BytesValues;
/**
* shared utils class - should be factored into HashedBytesValues
*/
abstract class HashedBytesValuesWithOrds implements org.elasticsearch.index.fielddata.HashedBytesValues.WithOrdinals {
protected final int[] hashes;
protected final Ordinals.Docs ordinals;
protected final BytesRef scratch1 = new BytesRef();
protected final HashedBytesRef scratch = new HashedBytesRef();
protected final BytesValues.WithOrdinals withOrds;
HashedBytesValuesWithOrds(BytesValues.WithOrdinals withOrds, int[] hashes) {
this.hashes = hashes;
this.ordinals = withOrds.ordinals();
this.withOrds = withOrds;
}
@Override
public boolean isMultiValued() {
return withOrds.isMultiValued();
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
} else {
proc.onValue(docId, scratch.reset(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]));
}
}
protected final void forEachValueInDocMulti(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, scratch.reset(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]));
} while ((ord = iter.next()) != 0);
}
@Override
public Ordinals.Docs ordinals() {
return this.ordinals;
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
return scratch.reset(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]);
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]);
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public HashedBytesRef makeSafe(HashedBytesRef bytes) {
return new HashedBytesRef(BytesRef.deepCopyOf(bytes.bytes), bytes.hash);
}
@Override
public HashedBytesRef getValue(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return null;
return scratch.reset(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]);
}
final static class Single extends HashedBytesValuesWithOrds {
private final Iter.Single iter = new Iter.Single();
Single(BytesValues.WithOrdinals withOrds, int[] hashes) {
super(withOrds, hashes);
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(scratch.reset(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]));
}
}
final static class Multi extends HashedBytesValuesWithOrds {
private final HashedBytesValuesWithOrds.Multi.MultiIter iter;
Multi(BytesValues.WithOrdinals withOrds, int[] hashes) {
super(withOrds, hashes);
this.iter = new MultiIter(withOrds, hashes);
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
forEachValueInDocMulti(docId, proc);
}
final static class MultiIter implements Iter {
private final int[] hashes;
private Ordinals.Docs.Iter ordsIter;
private int ord;
private final BytesRef scratch1 = new BytesRef();
private final HashedBytesRef scratch = new HashedBytesRef();
private final BytesValues.WithOrdinals withOrds;
MultiIter(BytesValues.WithOrdinals withOrds, int[] hashes) {
this.hashes = hashes;
this.withOrds = withOrds;
}
public HashedBytesValuesWithOrds.Multi.MultiIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public HashedBytesRef next() {
HashedBytesRef value = scratch.reset(withOrds.getValueScratchByOrd(ord, scratch1), hashes[ord]);
ord = ordsIter.next();
return value;
}
}
}
}

27
src/main/java/org/elasticsearch/index/fielddata/plain/IntArrayAtomicFieldData.java
View File

@ -29,7 +29,7 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData {
public abstract class IntArrayAtomicFieldData extends AtomicNumericFieldData {
public static final IntArrayAtomicFieldData EMPTY = new Empty();
@ -88,11 +88,6 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public StringValues getStringValues() {
return StringValues.EMPTY;
@ -131,16 +126,6 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
return size;
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());
@ -403,11 +388,6 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());
@ -597,11 +577,6 @@ public abstract class IntArrayAtomicFieldData implements AtomicNumericFieldData
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());

27
src/main/java/org/elasticsearch/index/fielddata/plain/LongArrayAtomicFieldData.java
View File

@ -30,7 +30,7 @@ import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData {
public abstract class LongArrayAtomicFieldData extends AtomicNumericFieldData {
public static final LongArrayAtomicFieldData EMPTY = new Empty();
@ -89,11 +89,6 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public StringValues getStringValues() {
return StringValues.EMPTY;
@ -132,16 +127,6 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
return size;
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values, ordinals.ordinals());
@ -498,11 +483,6 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values, set);
@ -753,11 +733,6 @@ public abstract class LongArrayAtomicFieldData implements AtomicNumericFieldData
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values);

110
src/main/java/org/elasticsearch/index/fielddata/plain/PagedBytesAtomicFieldData.java
View File

@ -21,14 +21,17 @@ package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.PagedBytes.Reader;
import org.apache.lucene.util.packed.GrowableWriter;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.index.fielddata.AtomicFieldData;
import org.elasticsearch.index.fielddata.HashedBytesValues;
import org.elasticsearch.index.fielddata.BytesValues.Iter;
import org.elasticsearch.index.fielddata.BytesValues.Iter.Multi;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.StringValues;
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
/**
@ -44,7 +47,7 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
private final PackedInts.Reader termOrdToBytesOffset;
protected final Ordinals ordinals;
private int[] hashes;
private volatile int[] hashes;
private long size = -1;
private final long readerBytesSize;
@ -86,14 +89,8 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
}
return size;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return ordinals.isMultiValued() ? new BytesValues.Multi(bytes, termOrdToBytesOffset, ordinals.ordinals()) : new BytesValues.Single(bytes, termOrdToBytesOffset, ordinals.ordinals());
}
@Override
public HashedBytesValues.WithOrdinals getHashedBytesValues() {
private final int[] getHashes() {
if (hashes == null) {
int numberOfValues = termOrdToBytesOffset.size();
int[] hashes = new int[numberOfValues];
@ -104,9 +101,24 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
}
this.hashes = hashes;
}
return ordinals.isMultiValued() ? new HashedBytesValuesWithOrds.Multi(getBytesValues(), hashes) : new HashedBytesValuesWithOrds.Single(getBytesValues(), hashes);
return hashes;
}
@Override
public BytesValues.WithOrdinals getBytesValues() {
return ordinals.isMultiValued() ? new BytesValues.Multi(bytes, termOrdToBytesOffset, ordinals.ordinals()) : new BytesValues.Single(
bytes, termOrdToBytesOffset, ordinals.ordinals());
}
@Override
public org.elasticsearch.index.fielddata.BytesValues.WithOrdinals getHashedBytesValues() {
final int[] hashes = getHashes();
return ordinals.isMultiValued() ? new BytesValues.MultiHashed(hashes, bytes, termOrdToBytesOffset, ordinals.ordinals())
: new BytesValues.SingleHashed(hashes, bytes, termOrdToBytesOffset, ordinals.ordinals());
}
@Override
public StringValues.WithOrdinals getStringValues() {
return StringValues.BytesValuesWrapper.wrap(getBytesValues());
@ -144,49 +156,94 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
}
static final class Single extends BytesValues {
static class Single extends BytesValues {
private final Iter.Single iter = new Iter.Single();
private final Iter.Single iter;
Single(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
super(bytes, termOrdToBytesOffset, ordinals);
assert !ordinals.isMultiValued();
iter = newSingleIter();
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
bytes.fill(scratch, termOrdToBytesOffset.get(ord));
return iter.reset(scratch);
return iter.reset(scratch, ord);
}
}
static final class SingleHashed extends Single {
private final int[] hashes;
static final class Multi extends BytesValues {
SingleHashed(int[] hashes, Reader bytes, org.apache.lucene.util.packed.PackedInts.Reader termOrdToBytesOffset, Docs ordinals) {
super(bytes, termOrdToBytesOffset, ordinals);
this.hashes = hashes;
}
@Override
protected Iter.Single newSingleIter() {
return new Iter.Single() {
public int hash() {
return hashes[ord];
}
};
}
@Override
public int getValueHashed(int docId, BytesRef ret) {
final int ord = ordinals.getOrd(docId);
getValueScratchByOrd(ord, ret);
return hashes[ord];
}
}
static class Multi extends BytesValues {
private final Iter.Multi iter;
Multi(PagedBytes.Reader bytes, PackedInts.Reader termOrdToBytesOffset, Ordinals.Docs ordinals) {
super(bytes, termOrdToBytesOffset, ordinals);
assert ordinals.isMultiValued();
this.iter = new Iter.Multi(this);
}
@Override
public BytesRefArrayRef getValues(int docId) {
return getValuesMulti(docId);
this.iter = newMultiIter();
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
}
static final class MultiHashed extends Multi {
private final int[] hashes;
MultiHashed(int[] hashes, Reader bytes, org.apache.lucene.util.packed.PackedInts.Reader termOrdToBytesOffset, Docs ordinals) {
super(bytes, termOrdToBytesOffset, ordinals);
this.hashes = hashes;
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
forEachValueInDocMulti(docId, proc);
protected Iter.Multi newMultiIter() {
return new Iter.Multi(this) {
public int hash() {
return hashes[ord];
}
};
}
@Override
public int getValueHashed(int docId, BytesRef ret) {
int ord = ordinals.getOrd(docId);
getValueScratchByOrd(ord, ret);
return hashes[ord];
}
}
}
@ -222,11 +279,6 @@ public class PagedBytesAtomicFieldData implements AtomicFieldData.WithOrdinals<S
return new BytesValues.WithOrdinals.Empty(ordinals.ordinals());
}
@Override
public HashedBytesValues.WithOrdinals getHashedBytesValues() {
return new HashedBytesValuesWithOrds.Empty(ordinals);
}
@Override
public StringValues.WithOrdinals getStringValues() {
return new StringValues.WithOrdinals.Empty((EmptyOrdinals) ordinals);

27
src/main/java/org/elasticsearch/index/fielddata/plain/ShortArrayAtomicFieldData.java
View File

@ -29,7 +29,7 @@ import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldData {
public abstract class ShortArrayAtomicFieldData extends AtomicNumericFieldData {
public static final ShortArrayAtomicFieldData EMPTY = new Empty();
@ -88,11 +88,6 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
return BytesValues.EMPTY;
}
@Override
public HashedBytesValues getHashedBytesValues() {
return HashedBytesValues.EMPTY;
}
@Override
public StringValues getStringValues() {
return StringValues.EMPTY;
@ -131,16 +126,6 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
return size;
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());
@ -403,11 +388,6 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());
@ -598,11 +578,6 @@ public abstract class ShortArrayAtomicFieldData implements AtomicNumericFieldDat
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues.LongBased(getLongValues());

181
src/main/java/org/elasticsearch/search/facet/terms/strings/FieldsTermsStringFacetExecutor.java
View File

@ -19,30 +19,27 @@
package org.elasticsearch.search.facet.terms.strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import gnu.trove.iterator.TObjectIntIterator;
import gnu.trove.map.hash.TObjectIntHashMap;
import java.io.IOException;
import java.util.Arrays;
import java.util.regex.Pattern;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.collect.BoundedTreeSet;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.HashedBytesValues;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.FacetPhaseExecutionException;
import org.elasticsearch.search.facet.InternalFacet;
import org.elasticsearch.search.facet.terms.strings.HashedAggregator.BytesRefCountIterator;
import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
*
@ -51,13 +48,9 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
private final InternalStringTermsFacet.ComparatorType comparatorType;
private final int size;
private final int numberOfShards;
private final IndexFieldData[] indexFieldDatas;
private final SearchScript script;
private final Pattern pattern;
private final ImmutableSet<BytesRef> excluded;
TObjectIntHashMap<HashedBytesRef> facets;
private final HashedAggregator aggregator;
long missing;
long total;
@ -66,14 +59,7 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
this.size = size;
this.comparatorType = comparatorType;
this.numberOfShards = context.numberOfShards();
this.script = script;
this.excluded = excluded;
this.pattern = pattern;
facets = CacheRecycler.popObjectIntMap();
this.indexFieldDatas = new IndexFieldData[fieldsNames.length];
for (int i = 0; i < fieldsNames.length; i++) {
FieldMapper mapper = context.smartNameFieldMapper(fieldsNames[i]);
@ -82,6 +68,11 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
}
indexFieldDatas[i] = context.fieldData().getForField(mapper);
}
if (excluded.isEmpty() && pattern == null && script == null) {
aggregator = new HashedAggregator();
} else {
aggregator = new HashedScriptAggregator(excluded, pattern, script);
}
// TODO: we need to support this flag with the new field data...
// if (allTerms) {
@ -100,54 +91,23 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
@Override
public Collector collector() {
return new Collector();
return new Collector(aggregator);
}
@Override
public InternalFacet buildFacet(String facetName) {
if (facets.isEmpty()) {
CacheRecycler.pushObjectIntMap(facets);
return new InternalStringTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalStringTermsFacet.TermEntry>of(), missing, total);
} else {
if (size < EntryPriorityQueue.LIMIT) {
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
}
InternalStringTermsFacet.TermEntry[] list = new InternalStringTermsFacet.TermEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = ((InternalStringTermsFacet.TermEntry) ordered.pop());
}
CacheRecycler.pushObjectIntMap(facets);
return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
} else {
BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered = new BoundedTreeSet<InternalStringTermsFacet.TermEntry>(comparatorType.comparator(), size);
for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
}
CacheRecycler.pushObjectIntMap(facets);
return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, missing, total);
}
}
return HashedAggregator.buildFacet(facetName, size, missing, total, comparatorType, aggregator);
}
class Collector extends FacetExecutor.Collector {
private final StaticAggregatorValueProc[] aggregators;
private HashedBytesValues[] values;
private final HashedAggregator aggregator;
private BytesValues[] values;
public Collector() {
values = new HashedBytesValues[indexFieldDatas.length];
aggregators = new StaticAggregatorValueProc[indexFieldDatas.length];
for (int i = 0; i < indexFieldDatas.length; i++) {
if (excluded.isEmpty() && pattern == null && script == null) {
aggregators[i] = new StaticAggregatorValueProc(facets);
} else {
aggregators[i] = new AggregatorValueProc(facets, excluded, pattern, script);
}
}
public Collector(HashedAggregator aggregator) {
values = new BytesValues[indexFieldDatas.length];
this.aggregator = aggregator;
}
@Override
@ -160,8 +120,7 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
for (int i = 0; i < indexFieldDatas.length; i++) {
values[i] = indexFieldDatas[i].load(context).getHashedBytesValues();
aggregators[i].values = values[i];
values[i] = indexFieldDatas[i].load(context).getBytesValues();
}
if (script != null) {
script.setNextReader(context);
@ -171,103 +130,15 @@ public class FieldsTermsStringFacetExecutor extends FacetExecutor {
@Override
public void collect(int doc) throws IOException {
for (int i = 0; i < values.length; i++) {
values[i].forEachValueInDoc(doc, aggregators[i]);
aggregator.onDoc(doc, values[i]);
}
}
@Override
public void postCollection() {
long missing = 0;
long total = 0;
for (StaticAggregatorValueProc aggregator : aggregators) {
missing += aggregator.missing();
total += aggregator.total();
}
FieldsTermsStringFacetExecutor.this.missing = missing;
FieldsTermsStringFacetExecutor.this.total = total;
FieldsTermsStringFacetExecutor.this.missing = aggregator.missing();
FieldsTermsStringFacetExecutor.this.total = aggregator.total();
}
}
public static class AggregatorValueProc extends StaticAggregatorValueProc {
private final ImmutableSet<BytesRef> excluded;
private final Matcher matcher;
private final SearchScript script;
public AggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
super(facets);
this.excluded = excluded;
this.matcher = pattern != null ? pattern.matcher("") : null;
this.script = script;
}
@Override
public void onValue(int docId, HashedBytesRef value) {
if (excluded != null && excluded.contains(value.bytes)) {
return;
}
// LUCENE 4 UPGRADE: use Lucene's RegexCapabilities
if (matcher != null && !matcher.reset(value.bytes.utf8ToString()).matches()) {
return;
}
if (script != null) {
script.setNextDocId(docId);
// LUCENE 4 UPGRADE: needs optimization
script.setNextVar("term", value.bytes.utf8ToString());
Object scriptValue = script.run();
if (scriptValue == null) {
return;
}
if (scriptValue instanceof Boolean) {
if (!((Boolean) scriptValue)) {
return;
}
} else {
// LUCENE 4 UPGRADE: make script return BR?
value = new HashedBytesRef(scriptValue.toString());
}
}
super.onValue(docId, value);
}
}
public static class StaticAggregatorValueProc implements HashedBytesValues.ValueInDocProc {
// LUCENE 4 UPGRADE: check if hashcode is not too expensive
private final TObjectIntHashMap<HashedBytesRef> facets;
HashedBytesValues values;
private int missing;
private int total;
public StaticAggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets) {
this.facets = facets;
}
@Override
public void onValue(int docId, HashedBytesRef value) {
facets.adjustOrPutValue(values.makeSafe(value), 1, 1);
total++;
}
@Override
public void onMissing(int docId) {
missing++;
}
public final TObjectIntHashMap<HashedBytesRef> facets() {
return facets;
}
public final int missing() {
return this.missing;
}
public final int total() {
return this.total;
}
}
}

145
src/main/java/org/elasticsearch/search/facet/terms/strings/HashedAggregator.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.facet.terms.strings;
import java.util.Arrays;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.elasticsearch.common.collect.BoundedTreeSet;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.BytesValues.Iter;
import org.elasticsearch.search.facet.InternalFacet;
import org.elasticsearch.search.facet.terms.TermsFacet;
import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
import com.google.common.collect.ImmutableList;
public class HashedAggregator {
private int missing;
private int total;
private final BytesRefHash hash;
private int[] counts = new int[10];
public HashedAggregator() {
this(new BytesRefHash());
}
public HashedAggregator(BytesRefHash hash) {
this.hash = hash;
}
public void onDoc(int docId, BytesValues values) {
if (values.hasValue(docId)) {
final Iter iter = values.getIter(docId);
while(iter.hasNext()) {
onValue(docId, iter.next(), iter.hash(), values);
total++;
}
} else {
missing++;
}
}
protected BytesRef makesSafe(BytesRef ref, BytesValues values) {
return values.makeSafe(ref);
}
protected void onValue(int docId, BytesRef value, int hashCode, BytesValues values) {
int key = hash.add(value, hashCode);
if (key < 0) {
key = ((-key)-1);
} else if (key >= counts.length) {
counts = ArrayUtil.grow(counts, key + 1);
}
counts[key]++;
}
public final int missing() {
return missing;
}
public final int total() {
return total;
}
public final boolean isEmpty() {
return hash.size() == 0;
}
public BytesRefCountIterator getIter() {
return new BytesRefCountIterator();
}
public final class BytesRefCountIterator {
final BytesRef spare = new BytesRef();
private final int size;
private int current = 0;
private int currentCount = -1;
BytesRefCountIterator() {
this.size = hash.size();
}
public BytesRef next() {
if (current < size) {
currentCount = counts[current];
hash.get(current++, spare);
return spare;
}
currentCount = -1;
return null;
}
public int count() {
return currentCount;
}
}
public static InternalFacet buildFacet(String facetName, int size, long missing, long total, TermsFacet.ComparatorType comparatorType, HashedAggregator aggregator) {
if (aggregator.isEmpty()) {
return new InternalStringTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalStringTermsFacet.TermEntry>of(), missing, total);
} else {
if (size < EntryPriorityQueue.LIMIT) {
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
BytesRefCountIterator iter = aggregator.getIter();
BytesRef next = null;
while((next = iter.next()) != null) {
ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(BytesRef.deepCopyOf(next), iter.count()));
// maybe we can survive with a 0-copy here if we keep the bytes ref hash around?
}
InternalStringTermsFacet.TermEntry[] list = new InternalStringTermsFacet.TermEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = ((InternalStringTermsFacet.TermEntry) ordered.pop());
}
return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
} else {
BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered = new BoundedTreeSet<InternalStringTermsFacet.TermEntry>(comparatorType.comparator(), size);
BytesRefCountIterator iter = aggregator.getIter();
BytesRef next = null;
while((next = iter.next()) != null) {
ordered.add(new InternalStringTermsFacet.TermEntry(BytesRef.deepCopyOf(next), iter.count()));
// maybe we can survive with a 0-copy here if we keep the bytes ref hash around?
}
return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, missing, total);
}
}
}
}

94
src/main/java/org/elasticsearch/search/facet/terms/strings/HashedScriptAggregator.java Normal file
View File

@ -0,0 +1,94 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.facet.terms.strings;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.script.SearchScript;
import com.google.common.collect.ImmutableSet;
public final class HashedScriptAggregator extends HashedAggregator {
private final ImmutableSet<BytesRef> excluded;
private final Matcher matcher;
private final SearchScript script;
private final CharsRef spare = new CharsRef();
private final BytesRef scriptSpare = new BytesRef();
private final boolean convert;
public HashedScriptAggregator(ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
this(new BytesRefHash(), excluded, pattern, script);
}
public HashedScriptAggregator(BytesRefHash hash, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
super(hash);
this.excluded = excluded;
this.matcher = pattern != null ? pattern.matcher("") : null;
this.script = script;
this.convert = script == null || matcher == null;
}
@Override
protected void onValue(int docId, BytesRef value, int hashCode, BytesValues values) {
if (excluded != null && excluded.contains(value)) {
return;
}
if (convert) {
// only convert if we need to and only once per doc...
UnicodeUtil.UTF8toUTF16(value, spare);
}
if (matcher != null) {
assert value.utf8ToString().equals(spare.toString());
if (!matcher.reset(spare).matches()) {
return;
}
}
if (script != null) {
assert value.utf8ToString().equals(spare.toString());
script.setNextDocId(docId);
// LUCENE 4 UPGRADE: needs optimization -- maybe a CharSequence does the job here?
// we only creat that string if we really need
script.setNextVar("term", spare.toString());
Object scriptValue = script.run();
if (scriptValue == null) {
return;
}
if (scriptValue instanceof Boolean) {
if (!((Boolean) scriptValue)) {
return;
}
} else {
// LUCENE 4 UPGRADE: should be possible to convert directly to BR
scriptSpare.copyChars(scriptValue.toString());
hashCode = scriptSpare.hashCode();
super.onValue(docId, scriptSpare, hashCode, values);
return;
}
}
super.onValue(docId, value, hashCode, values);
}
}

177
src/main/java/org/elasticsearch/search/facet/terms/strings/TermsStringFacetExecutor.java
View File

@ -19,29 +19,32 @@
package org.elasticsearch.search.facet.terms.strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import gnu.trove.iterator.TObjectIntIterator;
import gnu.trove.map.hash.TObjectIntHashMap;
import java.io.IOException;
import java.util.Arrays;
import java.util.regex.Pattern;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.collect.BoundedTreeSet;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.HashedBytesValues;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.FacetPhaseExecutionException;
import org.elasticsearch.search.facet.InternalFacet;
import org.elasticsearch.search.facet.terms.TermsFacet;
import org.elasticsearch.search.facet.terms.strings.HashedAggregator.BytesRefCountIterator;
import org.elasticsearch.search.facet.terms.support.EntryPriorityQueue;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
/**
*
@ -50,89 +53,50 @@ public class TermsStringFacetExecutor extends FacetExecutor {
private final IndexFieldData indexFieldData;
private final TermsFacet.ComparatorType comparatorType;
private final ImmutableSet<BytesRef> excluded;
private final Pattern pattern;
private final SearchScript script;
private final int size;
private final int numberOfShards;
// the aggregation map
TObjectIntHashMap<HashedBytesRef> facets;
long missing;
long total;
private final boolean allTerms;
private final HashedAggregator aggregator;
public TermsStringFacetExecutor(IndexFieldData indexFieldData, int size, TermsFacet.ComparatorType comparatorType, boolean allTerms, SearchContext context,
ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
this.indexFieldData = indexFieldData;
this.size = size;
this.comparatorType = comparatorType;
this.numberOfShards = context.numberOfShards();
this.script = script;
this.excluded = excluded;
this.pattern = pattern;
this.facets = CacheRecycler.popObjectIntMap();
if (allTerms) {
// TODO: we need to support this back with the new field data!
// try {
// for (AtomicReaderContext readerContext : context.searcher().getTopReaderContext().leaves()) {
// FieldData fieldData = fieldDataCache.cache(fieldDataType, readerContext.reader(), indexFieldName);
// fieldData.forEachValue(aggregator);
// }
// } catch (Exception e) {
// throw new FacetPhaseExecutionException(facetName, "failed to load all terms", e);
// }
this.allTerms = allTerms;
if (excluded.isEmpty() && pattern == null && script == null) {
aggregator = new HashedAggregator();
} else {
aggregator = new HashedScriptAggregator(excluded, pattern, script);
}
}
@Override
public Collector collector() {
return new Collector();
return new Collector(aggregator, allTerms);
}
@Override
public InternalFacet buildFacet(String facetName) {
if (facets.isEmpty()) {
CacheRecycler.pushObjectIntMap(facets);
return new InternalStringTermsFacet(facetName, comparatorType, size, ImmutableList.<InternalStringTermsFacet.TermEntry>of(), missing, total);
} else {
if (size < EntryPriorityQueue.LIMIT) {
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.insertWithOverflow(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
}
InternalStringTermsFacet.TermEntry[] list = new InternalStringTermsFacet.TermEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = ((InternalStringTermsFacet.TermEntry) ordered.pop());
}
CacheRecycler.pushObjectIntMap(facets);
return new InternalStringTermsFacet(facetName, comparatorType, size, Arrays.asList(list), missing, total);
} else {
BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered = new BoundedTreeSet<InternalStringTermsFacet.TermEntry>(comparatorType.comparator(), size);
for (TObjectIntIterator<HashedBytesRef> it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new InternalStringTermsFacet.TermEntry(it.key().bytes, it.value()));
}
CacheRecycler.pushObjectIntMap(facets);
return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, missing, total);
}
}
return HashedAggregator.buildFacet(facetName, size, missing, total, comparatorType, aggregator);
}
class Collector extends FacetExecutor.Collector {
final class Collector extends FacetExecutor.Collector {
private final StaticAggregatorValueProc aggregator;
private HashedBytesValues values;
private final HashedAggregator aggregator;
private final boolean allTerms;
private BytesValues values;
Collector() {
if (excluded.isEmpty() && pattern == null && script == null) {
aggregator = new StaticAggregatorValueProc(facets);
} else {
aggregator = new AggregatorValueProc(facets, excluded, pattern, script);
}
Collector(HashedAggregator aggregator, boolean allTerms) {
this.aggregator = aggregator;
this.allTerms = allTerms;
}
@Override
@ -144,8 +108,7 @@ public class TermsStringFacetExecutor extends FacetExecutor {
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
values = indexFieldData.load(context).getHashedBytesValues();
aggregator.values = values;
values = indexFieldData.load(context).getBytesValues();
if (script != null) {
script.setNextReader(context);
}
@ -153,7 +116,7 @@ public class TermsStringFacetExecutor extends FacetExecutor {
@Override
public void collect(int doc) throws IOException {
values.forEachValueInDoc(doc, aggregator);
aggregator.onDoc(doc, values);
}
@Override
@ -162,85 +125,5 @@ public class TermsStringFacetExecutor extends FacetExecutor {
TermsStringFacetExecutor.this.total = aggregator.total();
}
}
public static class AggregatorValueProc extends StaticAggregatorValueProc {
private final ImmutableSet<BytesRef> excluded;
private final Matcher matcher;
private final SearchScript script;
public AggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets, ImmutableSet<BytesRef> excluded, Pattern pattern, SearchScript script) {
super(facets);
this.excluded = excluded;
this.matcher = pattern != null ? pattern.matcher("") : null;
this.script = script;
}
@Override
public void onValue(int docId, HashedBytesRef value) {
if (excluded != null && excluded.contains(value.bytes)) {
return;
}
// LUCENE 4 UPGRADE: use Lucene's RegexCapabilities
if (matcher != null && !matcher.reset(value.bytes.utf8ToString()).matches()) {
return;
}
if (script != null) {
script.setNextDocId(docId);
// LUCENE 4 UPGRADE: needs optimization
script.setNextVar("term", value.bytes.utf8ToString());
Object scriptValue = script.run();
if (scriptValue == null) {
return;
}
if (scriptValue instanceof Boolean) {
if (!((Boolean) scriptValue)) {
return;
}
} else {
// LUCENE 4 UPGRADE: should be possible to convert directly to BR
value = new HashedBytesRef(scriptValue.toString());
}
}
super.onValue(docId, value);
}
}
public static class StaticAggregatorValueProc implements HashedBytesValues.ValueInDocProc {
// LUCENE 4 UPGRADE: check if hashcode is not too expensive
private final TObjectIntHashMap<HashedBytesRef> facets;
HashedBytesValues values;
private int missing = 0;
private int total = 0;
public StaticAggregatorValueProc(TObjectIntHashMap<HashedBytesRef> facets) {
this.facets = facets;
}
@Override
public void onValue(int docId, HashedBytesRef value) {
// we have to "makeSafe", even if it exists, since it might not..., need to find a way to optimize it
facets.adjustOrPutValue(values.makeSafe(value), 1, 1);
total++;
}
@Override
public void onMissing(int docId) {
missing++;
}
public final TObjectIntHashMap<HashedBytesRef> facets() {
return facets;
}
public final int missing() {
return this.missing;
}
public int total() {
return this.total;
}
}
}

56
src/main/java/org/elasticsearch/search/facet/termsstats/strings/TermsStatsStringFacetExecutor.java
View File

@ -19,27 +19,30 @@
package org.elasticsearch.search.facet.termsstats.strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.common.trove.ExtTHashMap;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.HashedBytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.InternalFacet;
import org.elasticsearch.search.facet.terms.strings.HashedAggregator;
import org.elasticsearch.search.facet.termsstats.TermsStatsFacet;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
public class TermsStatsStringFacetExecutor extends FacetExecutor {
@ -100,7 +103,7 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
class Collector extends FacetExecutor.Collector {
private final Aggregator aggregator;
private HashedBytesValues keyValues;
private BytesValues keyValues;
public Collector() {
if (script != null) {
@ -119,8 +122,7 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
keyValues = keyIndexFieldData.load(context).getHashedBytesValues();
aggregator.keyValues = keyValues;
keyValues = keyIndexFieldData.load(context).getBytesValues();
if (script != null) {
script.setNextReader(context);
} else {
@ -130,7 +132,7 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
@Override
public void collect(int doc) throws IOException {
keyValues.forEachValueInDoc(doc, aggregator);
aggregator.onDoc(doc, keyValues);
}
@Override
@ -139,13 +141,12 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
}
}
public static class Aggregator implements HashedBytesValues.ValueInDocProc {
public static class Aggregator extends HashedAggregator {
final ExtTHashMap<HashedBytesRef, InternalTermsStatsStringFacet.StringEntry> entries;
final HashedBytesRef spare = new HashedBytesRef();
int missing = 0;
HashedBytesValues keyValues;
DoubleValues valueValues;
ValueAggregator valueAggregator = new ValueAggregator();
@ -155,23 +156,19 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
}
@Override
public void onValue(int docId, HashedBytesRef value) {
InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(value);
public void onValue(int docId, BytesRef value, int hashCode, BytesValues values) {
spare.reset(value, hashCode);
InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(spare);
if (stringEntry == null) {
value = keyValues.makeSafe(value);
stringEntry = new InternalTermsStatsStringFacet.StringEntry(value, 0, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
entries.put(value, stringEntry);
HashedBytesRef theValue = new HashedBytesRef(makesSafe(value, values), hashCode);
stringEntry = new InternalTermsStatsStringFacet.StringEntry(theValue, 0, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
entries.put(theValue, stringEntry);
}
stringEntry.count++;
valueAggregator.stringEntry = stringEntry;
valueValues.forEachValueInDoc(docId, valueAggregator);
}
@Override
public void onMissing(int docId) {
missing++;
}
public static class ValueAggregator implements DoubleValues.ValueInDocProc {
InternalTermsStatsStringFacet.StringEntry stringEntry;
@ -203,12 +200,13 @@ public class TermsStatsStringFacetExecutor extends FacetExecutor {
}
@Override
public void onValue(int docId, HashedBytesRef value) {
InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(value);
public void onValue(int docId, BytesRef value, int hashCode, BytesValues values) {
spare.reset(value, hashCode);
InternalTermsStatsStringFacet.StringEntry stringEntry = entries.get(spare);
if (stringEntry == null) {
value = keyValues.makeSafe(value);
stringEntry = new InternalTermsStatsStringFacet.StringEntry(value, 1, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
entries.put(value, stringEntry);
HashedBytesRef theValue = new HashedBytesRef(makesSafe(value, values), hashCode);
stringEntry = new InternalTermsStatsStringFacet.StringEntry(theValue, 1, 0, 0, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY);
entries.put(theValue, stringEntry);
} else {
stringEntry.count++;
}

81
src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java
View File

@ -78,87 +78,6 @@ public abstract class AbstractFieldDataTests {
ifdService.clear();
}
public static class BytesValuesVerifierProc implements BytesValues.ValueInDocProc {
private static final BytesRef MISSING = new BytesRef();
private final int docId;
private final List<BytesRef> expected = new ArrayList<BytesRef>();
private int idx;
BytesValuesVerifierProc(int docId) {
this.docId = docId;
}
public BytesValuesVerifierProc addExpected(String value) {
expected.add(new BytesRef(value));
return this;
}
public BytesValuesVerifierProc addExpected(BytesRef value) {
expected.add(value);
return this;
}
public BytesValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, BytesRef value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class HashedBytesValuesVerifierProc implements HashedBytesValues.ValueInDocProc {
private static final HashedBytesRef MISSING = new HashedBytesRef();
private final int docId;
private final List<HashedBytesRef> expected = new ArrayList<HashedBytesRef>();
private int idx;
HashedBytesValuesVerifierProc(int docId) {
this.docId = docId;
}
public HashedBytesValuesVerifierProc addExpected(String value) {
expected.add(new HashedBytesRef(value));
return this;
}
public HashedBytesValuesVerifierProc addExpected(BytesRef value) {
expected.add(new HashedBytesRef(value));
return this;
}
public HashedBytesValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, HashedBytesRef value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class StringValuesVerifierProc implements StringValues.ValueInDocProc {

159
src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java
View File

@ -112,46 +112,26 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
BytesValues hashedBytesValues = fieldData.getBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one())));
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
@ -214,6 +194,11 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
}
private HashedBytesRef convert(BytesValues values, int doc) {
BytesRef ref = new BytesRef();
return new HashedBytesRef(ref, values.getValueHashed(doc, ref));
}
protected void fillSingleValueWithMissing() throws Exception {
Document d = new Document();
@ -262,17 +247,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(0));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
@ -281,32 +255,24 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
bytesValuesIter = bytesValues.getIter(1);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
BytesValues hashedBytesValues = fieldData.getBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), nullValue());
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef())));
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIter(1);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
@ -397,19 +363,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(2));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
@ -417,31 +370,23 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
BytesValues hashedBytesValues = fieldData.getBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(one())));
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
@ -550,18 +495,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(2));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(0));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
@ -572,34 +505,26 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
bytesValuesIter = bytesValues.getIter(1);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
BytesValues hashedBytesValues = fieldData.getBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), nullValue());
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(convert(hashedBytesValues, 0), equalTo(new HashedBytesRef(two())));
assertThat(convert(hashedBytesValues, 1), equalTo(new HashedBytesRef(new BytesRef())));
assertThat(convert(hashedBytesValues, 2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
assertThat(new HashedBytesRef(hashedBytesValuesIter.next(), hashedBytesValuesIter.hash()), equalTo(new HashedBytesRef(four())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIter(1);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
@ -670,16 +595,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef()));
assertThat(bytesRef, equalTo(new BytesRef()));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(0));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(0));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(0));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
@ -689,11 +604,7 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
bytesValuesIter = bytesValues.getIter(2);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addMissing());
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addMissing());
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
BytesValues hashedBytesValues = fieldData.getBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(false));
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
@ -703,7 +614,7 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
assertThat(hashedBytesValues.getValue(1), nullValue());
assertThat(hashedBytesValues.getValue(2), nullValue());
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
BytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIter(1);
@ -712,10 +623,6 @@ public abstract class StringFieldDataTests extends AbstractFieldDataTests {
hashedBytesValuesIter = hashedBytesValues.getIter(2);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addMissing());
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addMissing());
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(false));