honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

initial commit

This commit is contained in:
Shay Banon 2013-01-18 12:00:49 +01:00
parent 7cfdd9ef59
commit 7397007e05
52 changed files with 10294 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/main/java/org/elasticsearch/common/lucene/HashedBytesRef.java Normal file
View File

@ -0,0 +1,76 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.common.lucene;
import org.apache.lucene.util.BytesRef;
/**
* A wrapped to {@link BytesRef} that also caches the hashCode for it.
*/
public class HashedBytesRef {
public BytesRef bytes;
public int hash;
public HashedBytesRef() {
}
public HashedBytesRef(String bytes) {
this(new BytesRef(bytes));
}
public HashedBytesRef(BytesRef bytes) {
this(bytes, bytes.hashCode());
}
public HashedBytesRef(BytesRef bytes, int hash) {
this.bytes = bytes;
this.hash = hash;
}
public HashedBytesRef resetHashCode() {
this.hash = bytes.hashCode();
return this;
}
public HashedBytesRef reset(BytesRef bytes, int hash) {
this.bytes = bytes;
this.hash = hash;
return this;
}
@Override
public int hashCode() {
return hash;
}
@Override
public boolean equals(Object other) {
if (other instanceof HashedBytesRef) {
return bytes.equals(((HashedBytesRef) other).bytes);
}
return false;
}
@Override
public String toString() {
return bytes.toString();
}
}

32
src/main/java/org/elasticsearch/index/fielddata/AbstractIndexFieldData.java Normal file
View File

@ -0,0 +1,32 @@
package org.elasticsearch.index.fielddata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.settings.IndexSettings;
/**
*/
public abstract class AbstractIndexFieldData<FD extends AtomicFieldData> extends AbstractIndexComponent implements IndexFieldData<FD> {
private final String fieldName;
protected final FieldDataType fieldDataType;
protected final IndexFieldDataCache cache;
public AbstractIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
super(index, indexSettings);
this.fieldName = fieldName;
this.fieldDataType = fieldDataType;
this.cache = cache;
}
@Override
public String getFieldName() {
return this.fieldName;
}
@Override
public void clear() {
cache.clear(index, fieldName);
}
}

66
src/main/java/org/elasticsearch/index/fielddata/AtomicFieldData.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
/**
* The thread safe {@link org.apache.lucene.index.AtomicReader} level cache of the data.
*/
public interface AtomicFieldData<Script extends ScriptDocValues> {
/**
* Does *one* of the docs contain multi values?
*/
boolean isMultiValued();
/**
* Are the values ordered? (in ascending manner).
*/
boolean isValuesOrdered();
/**
* The number of docs in this field data.
*/
int getNumDocs();
/**
* Size (in bytes) of memory used by this field data.
*/
long getMemorySizeInBytes();
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
BytesValues getBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
HashedBytesValues getHashedBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as strings.
*/
StringValues getStringValues();
/**
* Returns a "scripting" based values.
*/
Script getScriptValues();
}

37
src/main/java/org/elasticsearch/index/fielddata/AtomicNumericFieldData.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
/**
*/
public interface AtomicNumericFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
ByteValues getByteValues();
ShortValues getShortValues();
IntValues getIntValues();
LongValues getLongValues();
FloatValues getFloatValues();
DoubleValues getDoubleValues();
}

40
src/main/java/org/elasticsearch/index/fielddata/AtomicOrdinalFieldData.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
/**
*/
public interface AtomicOrdinalFieldData<Script extends ScriptDocValues> extends AtomicFieldData<Script> {
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
OrdinalsBytesValues getBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as bytes.
*/
OrdinalsHashedBytesValues getHashedBytesValues();
/**
* Use a non thread safe (lightweight) view of the values as strings.
*/
OrdinalsStringValues getStringValues();
}

198
src/main/java/org/elasticsearch/index/fielddata/ByteValues.java Normal file
View File

@ -0,0 +1,198 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.ByteArrayRef;
import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public interface ByteValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
byte getValue(int docId);
byte getValueMissing(int docId, byte missingValue);
ByteArrayRef getValues(int docId);
Iter getIter(int docId);
void forEachValueInDoc(int docId, ValueInDocProc proc);
static interface ValueInDocProc {
void onValue(int docId, byte value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
byte next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public byte next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public byte value;
public boolean done;
public Single reset(byte value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public byte next() {
assert !done;
done = true;
return value;
}
}
}
public static class LongBased implements ByteValues {
private final LongValues values;
private final ByteArrayRef arrayScratch = new ByteArrayRef(new byte[1], 1);
private final ValueIter iter = new ValueIter();
private final Proc proc = new Proc();
public LongBased(LongValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public byte getValue(int docId) {
return (byte) values.getValue(docId);
}
@Override
public byte getValueMissing(int docId, byte missingValue) {
return (byte) values.getValueMissing(docId, missingValue);
}
@Override
public ByteArrayRef getValues(int docId) {
LongArrayRef arrayRef = values.getValues(docId);
int size = arrayRef.size();
if (size == 0) {
return ByteArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = arrayRef.start; i < arrayRef.end; i++) {
arrayScratch.values[arrayScratch.end++] = (byte) arrayRef.values[i];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
static class ValueIter implements Iter {
private LongValues.Iter iter;
public ValueIter reset(LongValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public byte next() {
return (byte) iter.next();
}
}
static class Proc implements LongValues.ValueInDocProc {
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, long value) {
proc.onValue(docId, (byte) value);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
}

305
src/main/java/org/elasticsearch/index/fielddata/BytesValues.java Normal file
View File

@ -0,0 +1,305 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public interface BytesValues {
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
*/
BytesRef getValue(int docId);
/**
* Returns the bytes value for the docId, with the provided "ret" which will be filled with the
* result which will also be returned. If there is no value for this docId, the length will be 0.
* Note, the bytes are not "safe".
*/
BytesRef getValueScratch(int docId, BytesRef ret);
/**
* Returns a bytes value for a docId. The content is guaranteed not to be shared.
*/
BytesRef getValueSafe(int docId);
/**
* Returns an array wrapping all the bytes values for a doc. The content is guaranteed not to be shared.
*/
BytesRefArrayRef getValues(int docId);
/**
* Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
*/
Iter getIter(int docId);
/**
* Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
*/
Iter getIterSafe(int docId);
/**
* Go over all the possible values in their BytesRef format for a specific doc.
*/
void forEachValueInDoc(int docId, ValueInDocProc proc);
/**
* Go over all the possible values in their BytesRef format for a specific doc.
*/
void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
public static interface ValueInDocProc {
void onValue(int docId, BytesRef value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
BytesRef next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public BytesRef next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public BytesRef value;
public boolean done;
public Single reset(BytesRef value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public BytesRef next() {
assert !done;
done = true;
return value;
}
}
}
public static class StringBased implements BytesValues {
private final StringValues values;
protected final BytesRef scratch = new BytesRef();
private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
private final ValueIter valueIter = new ValueIter();
private final SafeValueIter safeValueIter = new SafeValueIter();
private final Proc proc = new Proc();
private final SafeProc safeProc = new SafeProc();
public StringBased(StringValues values) {
this.values = values;
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public BytesRef getValue(int docId) {
String value = values.getValue(docId);
if (value == null) return null;
scratch.copyChars(value);
return scratch;
}
@Override
public BytesRef getValueScratch(int docId, BytesRef ret) {
String value = values.getValue(docId);
if (value == null) {
ret.length = 0;
return ret;
}
ret.copyChars(value);
return ret;
}
@Override
public BytesRef getValueSafe(int docId) {
String value = values.getValue(docId);
if (value == null) return null;
return new BytesRef(value);
}
@Override
public BytesRefArrayRef getValues(int docId) {
StringArrayRef arrayRef = values.getValues(docId);
int size = arrayRef.size();
if (size == 0) {
return BytesRefArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = arrayRef.start; i < arrayRef.end; i++) {
String value = arrayRef.values[i];
arrayScratch.values[arrayScratch.end++] = value == null ? null : new BytesRef(value);
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return valueIter.reset(values.getIter(docId));
}
@Override
public Iter getIterSafe(int docId) {
return safeValueIter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
@Override
public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.safeProc.reset(proc));
}
static class ValueIter implements Iter {
private final BytesRef scratch = new BytesRef();
private StringValues.Iter iter;
public ValueIter reset(StringValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public BytesRef next() {
scratch.copyChars(iter.next());
return scratch;
}
}
static class SafeValueIter implements Iter {
private StringValues.Iter iter;
public SafeValueIter reset(StringValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public BytesRef next() {
return new BytesRef(iter.next());
}
}
static class Proc implements StringValues.ValueInDocProc {
private final BytesRef scratch = new BytesRef();
private BytesValues.ValueInDocProc proc;
public Proc reset(BytesValues.ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, String value) {
scratch.copyChars(value);
proc.onValue(docId, scratch);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
static class SafeProc implements StringValues.ValueInDocProc {
private BytesValues.ValueInDocProc proc;
public SafeProc reset(BytesValues.ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, String value) {
proc.onValue(docId, new BytesRef(value));
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
}

100
src/main/java/org/elasticsearch/index/fielddata/DoubleValues.java Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
/**
*/
public interface DoubleValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
double getValue(int docId);
double getValueMissing(int docId, double missingValue);
DoubleArrayRef getValues(int docId);
Iter getIter(int docId);
void forEachValueInDoc(int docId, ValueInDocProc proc);
static interface ValueInDocProc {
void onValue(int docId, double value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
double next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public double next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public double value;
public boolean done;
public Single reset(double value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public double next() {
assert !done;
done = true;
return value;
}
}
}
}

56
src/main/java/org/elasticsearch/index/fielddata/FieldDataType.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.Nullable;
/**
*/
public class FieldDataType {
private final String type;
@Nullable
private final String format;
private final ImmutableMap<String, String> options;
public FieldDataType(String type) {
this(type, null, ImmutableMap.<String, String>of());
}
public FieldDataType(String type, @Nullable String format, ImmutableMap<String, String> options) {
this.type = type;
this.format = format;
this.options = options;
}
public String getType() {
return this.type;
}
@Nullable
public String getFormat() {
return this.format;
}
public ImmutableMap<String, String> getOptions() {
return this.options;
}
}

198
src/main/java/org/elasticsearch/index/fielddata/FloatValues.java Normal file
View File

@ -0,0 +1,198 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
import org.elasticsearch.index.fielddata.util.FloatArrayRef;
/**
*/
public interface FloatValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
float getValue(int docId);
float getValueMissing(int docId, float missingValue);
FloatArrayRef getValues(int docId);
Iter getIter(int docId);
void forEachValueInDoc(int docId, ValueInDocProc proc);
static interface ValueInDocProc {
void onValue(int docId, float value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
float next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public float next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public float value;
public boolean done;
public Single reset(float value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public float next() {
assert !done;
done = true;
return value;
}
}
}
public static class DoubleBased implements FloatValues {
private final DoubleValues values;
private final FloatArrayRef arrayScratch = new FloatArrayRef(new float[1], 1);
private final ValueIter iter = new ValueIter();
private final Proc proc = new Proc();
public DoubleBased(DoubleValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public float getValue(int docId) {
return (float) values.getValue(docId);
}
@Override
public float getValueMissing(int docId, float missingValue) {
return (float) values.getValueMissing(docId, missingValue);
}
@Override
public FloatArrayRef getValues(int docId) {
DoubleArrayRef arrayRef = values.getValues(docId);
int size = arrayRef.size();
if (size == 0) {
return FloatArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = arrayRef.start; i < arrayRef.end; i++) {
arrayScratch.values[arrayScratch.end++] = (float) arrayRef.values[i];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
static class ValueIter implements Iter {
private DoubleValues.Iter iter;
public ValueIter reset(DoubleValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public float next() {
return (float) iter.next();
}
}
static class Proc implements DoubleValues.ValueInDocProc {
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, double value) {
proc.onValue(docId, (float) value);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
}

411
src/main/java/org/elasticsearch/index/fielddata/HashedBytesValues.java Normal file
View File

@ -0,0 +1,411 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.common.lucene.HashedBytesRef;
/**
*/
public interface HashedBytesValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
/**
* Returns a bytes value for a docId. Note, the content of it might be shared across invocation.
*/
HashedBytesRef getValue(int docId);
/**
* Returns a bytes value for a docId. The content is guaranteed not to be shared.
*/
HashedBytesRef getValueSafe(int docId);
/**
* Returns a bytes value iterator for a docId. Note, the content of it might be shared across invocation.
*/
Iter getIter(int docId);
/**
* Returns a bytes value iterator for a docId. The content is guaranteed not to be shared.
*/
Iter getIterSafe(int docId);
/**
* Go over all the possible values in their BytesRef format for a specific doc.
*/
void forEachValueInDoc(int docId, ValueInDocProc proc);
/**
* Go over all the possible values in their BytesRef format for a specific doc.
*/
void forEachSafeValueInDoc(int docId, ValueInDocProc proc);
public static interface ValueInDocProc {
void onValue(int docId, HashedBytesRef value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
HashedBytesRef next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public HashedBytesRef next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public HashedBytesRef value;
public boolean done;
public Single reset(HashedBytesRef value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public HashedBytesRef next() {
assert !done;
done = true;
return value;
}
}
}
/**
* A {@link BytesValues} based implementation.
*/
static class BytesBased implements HashedBytesValues {
private final BytesValues values;
protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private final ValueIter valueIter = new ValueIter();
private final SafeValueIter safeValueIter = new SafeValueIter();
private final Proc proc = new Proc();
private final SafeProc safeProc = new SafeProc();
public BytesBased(BytesValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public HashedBytesRef getValue(int docId) {
BytesRef value = values.getValue(docId);
if (value == null) return null;
scratch.bytes = value;
return scratch.resetHashCode();
}
@Override
public HashedBytesRef getValueSafe(int docId) {
return new HashedBytesRef(values.getValueSafe(docId));
}
@Override
public Iter getIter(int docId) {
return valueIter.reset(values.getIter(docId));
}
@Override
public Iter getIterSafe(int docId) {
return safeValueIter.reset(values.getIterSafe(docId));
}
@Override
public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
@Override
public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.safeProc.reset(proc));
}
static class ValueIter implements Iter {
private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private BytesValues.Iter iter;
public ValueIter reset(BytesValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public HashedBytesRef next() {
scratch.bytes = iter.next();
return scratch.resetHashCode();
}
}
static class SafeValueIter implements Iter {
private BytesValues.Iter iter;
public SafeValueIter reset(BytesValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public HashedBytesRef next() {
return new HashedBytesRef(iter.next());
}
}
static class Proc implements BytesValues.ValueInDocProc {
private final HashedBytesRef scratch = new HashedBytesRef();
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, BytesRef value) {
scratch.bytes = value;
proc.onValue(docId, scratch.resetHashCode());
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
static class SafeProc implements BytesValues.ValueInDocProc {
private ValueInDocProc proc;
public SafeProc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, BytesRef value) {
proc.onValue(docId, new HashedBytesRef(value));
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
static class StringBased implements HashedBytesValues {
private final StringValues values;
protected final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private final ValueIter valueIter = new ValueIter();
private final SafeValueIter safeValueIter = new SafeValueIter();
private final Proc proc = new Proc();
private final SafeProc safeProc = new SafeProc();
public StringBased(StringValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public HashedBytesRef getValue(int docId) {
String value = values.getValue(docId);
if (value == null) return null;
scratch.bytes.copyChars(value);
return scratch.resetHashCode();
}
@Override
public HashedBytesRef getValueSafe(int docId) {
String value = values.getValue(docId);
if (value == null) return null;
return new HashedBytesRef(new BytesRef(values.getValue(docId)));
}
@Override
public Iter getIter(int docId) {
return valueIter.reset(values.getIter(docId));
}
@Override
public Iter getIterSafe(int docId) {
return safeValueIter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, final ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
@Override
public void forEachSafeValueInDoc(int docId, final ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.safeProc.reset(proc));
}
static class ValueIter implements Iter {
private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private StringValues.Iter iter;
public ValueIter reset(StringValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public HashedBytesRef next() {
scratch.bytes.copyChars(iter.next());
return scratch.resetHashCode();
}
}
static class SafeValueIter implements Iter {
private StringValues.Iter iter;
public SafeValueIter reset(StringValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public HashedBytesRef next() {
return new HashedBytesRef(new BytesRef(iter.next()));
}
}
static class Proc implements StringValues.ValueInDocProc {
private final HashedBytesRef scratch = new HashedBytesRef(new BytesRef());
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, String value) {
scratch.bytes.copyChars(value);
proc.onValue(docId, scratch);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
static class SafeProc implements StringValues.ValueInDocProc {
private ValueInDocProc proc;
public SafeProc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, String value) {
proc.onValue(docId, new HashedBytesRef(new BytesRef(value)));
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
}

77
src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparatorSource;
import org.apache.lucene.search.SortField;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexComponent;
import org.elasticsearch.index.settings.IndexSettings;
/**
*/
public interface IndexFieldData<FD extends AtomicFieldData> extends IndexComponent {
/**
* The field name.
*/
String getFieldName();
/**
* Are the values ordered? (in ascending manner).
*/
boolean valuesOrdered();
/**
* Loads the atomic field data for the reader, possibly cached.
*/
FD load(AtomicReaderContext context);
/**
* Loads directly the atomic field data for the reader, ignoring any caching involved.
*/
FD loadDirect(AtomicReaderContext context) throws Exception;
/**
* Comparator used for sorting.
*/
XFieldComparatorSource comparatorSource(@Nullable Object missingValue);
/**
* Clears any resources associated with this field data.
*/
void clear();
// we need this extended source we we have custom comparators to reuse our field data
// in this case, we need to reduce type that will be used when search results are reduced
// on another node (we don't have the custom source them...)
public abstract class XFieldComparatorSource extends FieldComparatorSource {
public abstract SortField.Type reducedType();
}
interface Builder {
IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache);
}
}

91
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataCache.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SegmentReader;
import org.elasticsearch.index.Index;
import java.util.concurrent.Callable;
/**
* A simple field data cache abstraction.
*/
public interface IndexFieldDataCache {
<FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(AtomicReaderContext context, IFD indexFieldData) throws Exception;
void clear(Index index);
void clear(Index index, String fieldName);
/**
* The resident field data cache is a *per field* cache that keeps all the values in memory.
*/
static abstract class FieldBased implements IndexFieldDataCache, SegmentReader.CoreClosedListener {
private final Cache<Object, AtomicFieldData> cache;
protected FieldBased(Cache<Object, AtomicFieldData> cache) {
this.cache = cache;
}
@Override
public void onClose(SegmentReader owner) {
cache.invalidate(owner.getCoreCacheKey());
}
@Override
public <FD extends AtomicFieldData, IFD extends IndexFieldData<FD>> FD load(final AtomicReaderContext context, final IFD indexFieldData) throws Exception {
//noinspection unchecked
return (FD) cache.get(context.reader().getCoreCacheKey(), new Callable<AtomicFieldData>() {
@Override
public AtomicFieldData call() throws Exception {
return indexFieldData.loadDirect(context);
}
});
}
@Override
public void clear(Index index) {
cache.invalidateAll();
}
@Override
public void clear(Index index, String fieldName) {
cache.invalidateAll();
}
}
static class Resident extends FieldBased {
public Resident() {
super(CacheBuilder.newBuilder().<Object, AtomicFieldData>build());
}
}
static class Soft extends FieldBased {
public Soft() {
super(CacheBuilder.newBuilder().softValues().<Object, AtomicFieldData>build());
}
}
}

124
src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java Normal file
View File

@ -0,0 +1,124 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.plain.ConcreteBytesRefIndexFieldData;
import org.elasticsearch.index.fielddata.plain.DoubleArrayIndexFieldData;
import org.elasticsearch.index.settings.IndexSettings;
import java.util.concurrent.ConcurrentMap;
/**
*/
public class IndexFieldDataService extends AbstractIndexComponent {
private final static ImmutableMap<String, IndexFieldData.Builder> buildersByType;
private final static ImmutableMap<Tuple<String, String>, IndexFieldData.Builder> buildersByTypeAndFormat;
static {
buildersByType = MapBuilder.<String, IndexFieldData.Builder>newMapBuilder()
.put("string", new ConcreteBytesRefIndexFieldData.Builder())
.put("double", new DoubleArrayIndexFieldData.Builder())
.immutableMap();
buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder().
put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
.put(Tuple.tuple("double", "array"), new DoubleArrayIndexFieldData.Builder())
.immutableMap();
}
private final ConcurrentMap<String, IndexFieldData> loadedFieldData = ConcurrentCollections.newConcurrentMap();
public IndexFieldDataService(Index index) {
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS);
}
@Inject
public IndexFieldDataService(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
}
public void clear() {
synchronized (loadedFieldData) {
for (IndexFieldData fieldData : loadedFieldData.values()) {
fieldData.clear();
}
loadedFieldData.clear();
}
}
public void clearField(String fieldName) {
synchronized (loadedFieldData) {
IndexFieldData fieldData = loadedFieldData.remove(fieldName);
if (fieldData != null) {
fieldData.clear();
}
}
}
public <IFD extends IndexFieldData> IFD getForField(String fieldName, FieldDataType type) {
IndexFieldData fieldData = loadedFieldData.get(type.getType());
if (fieldData == null) {
synchronized (loadedFieldData) {
fieldData = loadedFieldData.get(type.getType());
if (fieldData == null) {
IndexFieldData.Builder builder = null;
if (type.getFormat() != null) {
builder = buildersByTypeAndFormat.get(Tuple.tuple(type.getType(), type.getFormat()));
}
if (builder == null) {
builder = buildersByType.get(type.getType());
}
if (builder == null) {
throw new ElasticSearchIllegalArgumentException("failed to find field data builder for field " + fieldName + ", and type " + type);
}
IndexFieldDataCache cache;
if (type.getOptions().containsKey("cache")) {
String cacheType = type.getOptions().get("cache");
if ("resident".equals(cacheType)) {
cache = new IndexFieldDataCache.Resident();
} else if ("soft".equals(cacheType)) {
cache = new IndexFieldDataCache.Soft();
} else {
throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]");
}
} else {
cache = new IndexFieldDataCache.Resident();
}
fieldData = builder.build(index, indexSettings, fieldName, type, cache);
loadedFieldData.put(fieldName, fieldData);
}
}
}
return (IFD) fieldData;
}
}

48
src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
/**
*/
public interface IndexNumericFieldData<FD extends AtomicNumericFieldData> extends IndexFieldData<FD> {
static enum NumericType {
BYTE,
SHORT,
INT,
LONG,
FLOAT,
DOUBLE
}
NumericType getNumericType();
/**
* Loads the atomic field data for the reader, possibly cached.
*/
FD load(AtomicReaderContext context);
/**
* Loads directly the atomic field data for the reader, ignoring any caching involved.
*/
FD loadDirect(AtomicReaderContext context) throws Exception;
}

37
src/main/java/org/elasticsearch/index/fielddata/IndexOrdinalFieldData.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.index.AtomicReaderContext;
/**
*/
public interface IndexOrdinalFieldData<FD extends AtomicOrdinalFieldData> extends IndexFieldData<FD> {
/**
* Loads the atomic field data for the reader, possibly cached.
*/
FD load(AtomicReaderContext context);
/**
* Loads directly the atomic field data for the reader, ignoring any caching involved.
*/
FD loadDirect(AtomicReaderContext context) throws Exception;
}

198
src/main/java/org/elasticsearch/index/fielddata/IntValues.java Normal file
View File

@ -0,0 +1,198 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public interface IntValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
int getValue(int docId);
int getValueMissing(int docId, int missingValue);
IntArrayRef getValues(int docId);
Iter getIter(int docId);
void forEachValueInDoc(int docId, ValueInDocProc proc);
static interface ValueInDocProc {
void onValue(int docId, int value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
int next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public int next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public int value;
public boolean done;
public Single reset(int value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public int next() {
assert !done;
done = true;
return value;
}
}
}
public static class LongBased implements IntValues {
private final LongValues values;
private final IntArrayRef arrayScratch = new IntArrayRef(new int[1], 1);
private final ValueIter iter = new ValueIter();
private final Proc proc = new Proc();
public LongBased(LongValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public int getValue(int docId) {
return (int) values.getValue(docId);
}
@Override
public int getValueMissing(int docId, int missingValue) {
return (int) values.getValueMissing(docId, missingValue);
}
@Override
public IntArrayRef getValues(int docId) {
LongArrayRef arrayRef = values.getValues(docId);
int size = arrayRef.size();
if (size == 0) {
return IntArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = arrayRef.start; i < arrayRef.end; i++) {
arrayScratch.values[arrayScratch.end++] = (int) arrayRef.values[i];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
static class ValueIter implements Iter {
private LongValues.Iter iter;
public ValueIter reset(LongValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public int next() {
return (int) iter.next();
}
}
static class Proc implements LongValues.ValueInDocProc {
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, long value) {
proc.onValue(docId, (int) value);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
}

100
src/main/java/org/elasticsearch/index/fielddata/LongValues.java Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.LongArrayRef;
/**
*/
public interface LongValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
long getValue(int docId);
long getValueMissing(int docId, long missingValue);
LongArrayRef getValues(int docId);
Iter getIter(int docId);
void forEachValueInDoc(int docId, ValueInDocProc proc);
static interface ValueInDocProc {
void onValue(int docId, long value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
long next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public long next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public long value;
public boolean done;
public Single reset(long value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public long next() {
assert !done;
done = true;
return value;
}
}
}
}

73
src/main/java/org/elasticsearch/index/fielddata/OrdinalsBytesValues.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public interface OrdinalsBytesValues extends BytesValues {
Ordinals.Docs ordinals();
BytesRef getValueByOrd(int ord);
/**
* Returns the bytes value for the docId, with the provided "ret" which will be filled with the
* result which will also be returned. If there is no value for this docId, the length will be 0.
* Note, the bytes are not "safe".
*/
BytesRef getValueScratchByOrd(int ord, BytesRef ret);
BytesRef getSafeValueByOrd(int ord);
public static class StringBased extends BytesValues.StringBased implements OrdinalsBytesValues {
private final OrdinalsStringValues values;
public StringBased(OrdinalsStringValues values) {
super(values);
this.values = values;
}
@Override
public Ordinals.Docs ordinals() {
return values.ordinals();
}
@Override
public BytesRef getValueByOrd(int ord) {
scratch.copyChars(values.getValueByOrd(ord));
return scratch;
}
@Override
public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
ret.copyChars(values.getValueByOrd(ord));
return ret;
}
@Override
public BytesRef getSafeValueByOrd(int ord) {
return new BytesRef(values.getValueByOrd(ord));
}
}
}

87
src/main/java/org/elasticsearch/index/fielddata/OrdinalsHashedBytesValues.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public interface OrdinalsHashedBytesValues extends HashedBytesValues {
Ordinals.Docs ordinals();
HashedBytesRef getValueByOrd(int ord);
HashedBytesRef getSafeValueByOrd(int ord);
static class BytesBased extends HashedBytesValues.BytesBased implements OrdinalsHashedBytesValues {
private final OrdinalsBytesValues values;
public BytesBased(OrdinalsBytesValues values) {
super(values);
this.values = values;
}
@Override
public Ordinals.Docs ordinals() {
return values.ordinals();
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
scratch.bytes = values.getValueByOrd(ord);
return scratch.resetHashCode();
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(values.getSafeValueByOrd(ord));
}
}
static class StringBased extends HashedBytesValues.StringBased implements OrdinalsHashedBytesValues {
private final OrdinalsStringValues values;
public StringBased(OrdinalsStringValues values) {
super(values);
this.values = values;
}
@Override
public Ordinals.Docs ordinals() {
return values.ordinals();
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
scratch.bytes.copyChars(values.getValueByOrd(ord));
return scratch.resetHashCode();
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(new BytesRef(values.getValueByOrd(ord)));
}
}
}

31
src/main/java/org/elasticsearch/index/fielddata/OrdinalsStringValues.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
/**
*/
public interface OrdinalsStringValues extends StringValues {
Ordinals.Docs ordinals();
String getValueByOrd(int ord);
}

258
src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java Normal file
View File

@ -0,0 +1,258 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.util.*;
/**
* Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
* and a <code>getValues</code> that return the relevant type that then can be used in scripts.
*/
public interface ScriptDocValues {
void setNextDocId(int docId);
boolean isEmpty();
static class Strings implements ScriptDocValues {
private final StringValues values;
private int docId;
public Strings(StringValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public String getValue() {
return values.getValue(docId);
}
public StringArrayRef getValues() {
return values.getValues(docId);
}
}
static class Bytes implements ScriptDocValues {
private final BytesValues values;
private int docId;
public Bytes(BytesValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public BytesRef getValue() {
return values.getValue(docId);
}
public BytesRefArrayRef getValues() {
return values.getValues(docId);
}
}
static class NumericByte implements ScriptDocValues {
private final ByteValues values;
private int docId;
public NumericByte(ByteValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public byte getValue() {
return values.getValue(docId);
}
public ByteArrayRef getValues() {
return values.getValues(docId);
}
}
static class NumericShort implements ScriptDocValues {
private final ShortValues values;
private int docId;
public NumericShort(ShortValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public short getValue() {
return values.getValue(docId);
}
public ShortArrayRef getValues() {
return values.getValues(docId);
}
}
static class NumericInteger implements ScriptDocValues {
private final IntValues values;
private int docId;
public NumericInteger(IntValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public int getValue() {
return values.getValue(docId);
}
public IntArrayRef getValues() {
return values.getValues(docId);
}
}
static class NumericLong implements ScriptDocValues {
private final LongValues values;
private int docId;
public NumericLong(LongValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public long getValue() {
return values.getValue(docId);
}
public LongArrayRef getValues() {
return values.getValues(docId);
}
}
static class NumericFloat implements ScriptDocValues {
private final FloatValues values;
private int docId;
public NumericFloat(FloatValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public float getValue() {
return values.getValue(docId);
}
public FloatArrayRef getValues() {
return values.getValues(docId);
}
}
static class NumericDouble implements ScriptDocValues {
private final DoubleValues values;
private int docId;
public NumericDouble(DoubleValues values) {
this.values = values;
}
@Override
public void setNextDocId(int docId) {
this.docId = docId;
}
@Override
public boolean isEmpty() {
return !values.hasValue(docId);
}
public double getValue() {
return values.getValue(docId);
}
public DoubleArrayRef getValues() {
return values.getValues(docId);
}
}
}

198
src/main/java/org/elasticsearch/index/fielddata/ShortValues.java Normal file
View File

@ -0,0 +1,198 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.LongArrayRef;
import org.elasticsearch.index.fielddata.util.ShortArrayRef;
/**
*/
public interface ShortValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
short getValue(int docId);
short getValueMissing(int docId, short missingValue);
ShortArrayRef getValues(int docId);
Iter getIter(int docId);
void forEachValueInDoc(int docId, ValueInDocProc proc);
static interface ValueInDocProc {
void onValue(int docId, short value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
short next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public short next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public short value;
public boolean done;
public Single reset(short value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public short next() {
assert !done;
done = true;
return value;
}
}
}
public static class LongBased implements ShortValues {
private final LongValues values;
private final ShortArrayRef arrayScratch = new ShortArrayRef(new short[1], 1);
private final ValueIter iter = new ValueIter();
private final Proc proc = new Proc();
public LongBased(LongValues values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return values.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return values.hasValue(docId);
}
@Override
public short getValue(int docId) {
return (short) values.getValue(docId);
}
@Override
public short getValueMissing(int docId, short missingValue) {
return (short) values.getValueMissing(docId, missingValue);
}
@Override
public ShortArrayRef getValues(int docId) {
LongArrayRef arrayRef = values.getValues(docId);
int size = arrayRef.size();
if (size == 0) {
return ShortArrayRef.EMPTY;
}
arrayScratch.reset(size);
for (int i = arrayRef.start; i < arrayRef.end; i++) {
arrayScratch.values[arrayScratch.end++] = (short) arrayRef.values[i];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(values.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
values.forEachValueInDoc(docId, this.proc.reset(proc));
}
static class ValueIter implements Iter {
private LongValues.Iter iter;
public ValueIter reset(LongValues.Iter iter) {
this.iter = iter;
return this;
}
@Override
public boolean hasNext() {
return iter.hasNext();
}
@Override
public short next() {
return (short) iter.next();
}
}
static class Proc implements LongValues.ValueInDocProc {
private ValueInDocProc proc;
public Proc reset(ValueInDocProc proc) {
this.proc = proc;
return this;
}
@Override
public void onValue(int docId, long value) {
proc.onValue(docId, (short) value);
}
@Override
public void onMissing(int docId) {
proc.onMissing(docId);
}
}
}
}

102
src/main/java/org/elasticsearch/index/fielddata/StringValues.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata;
import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public interface StringValues {
/**
* Is one of the documents in this field data values is multi valued?
*/
boolean isMultiValued();
/**
* Is there a value for this doc?
*/
boolean hasValue(int docId);
String getValue(int docId);
StringArrayRef getValues(int docId);
Iter getIter(int docId);
/**
* Go over all the possible values.
*/
void forEachValueInDoc(int docId, ValueInDocProc proc);
public static interface ValueInDocProc {
void onValue(int docId, String value);
void onMissing(int docId);
}
static interface Iter {
boolean hasNext();
String next();
static class Empty implements Iter {
public static final Empty INSTANCE = new Empty();
@Override
public boolean hasNext() {
return false;
}
@Override
public String next() {
throw new ElasticSearchIllegalStateException();
}
}
static class Single implements Iter {
public String value;
public boolean done;
public Single reset(String value) {
this.value = value;
this.done = false;
return this;
}
@Override
public boolean hasNext() {
return !done;
}
@Override
public String next() {
assert !done;
done = true;
return value;
}
}
}
}

52
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
import java.io.IOException;
/**
*/
public class BytesRefFieldComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexFieldData indexFieldData;
public BytesRefFieldComparatorSource(IndexFieldData indexFieldData) {
this.indexFieldData = indexFieldData;
}
@Override
public SortField.Type reducedType() {
return SortField.Type.STRING;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert fieldname.equals(indexFieldData.getFieldName());
if (indexFieldData.valuesOrdered() && indexFieldData instanceof IndexOrdinalFieldData) {
return new BytesRefOrdValComparator((IndexOrdinalFieldData) indexFieldData, numHits);
}
return new BytesRefValComparator(indexFieldData, numHits);
}
}

475
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefOrdValComparator.java Normal file
View File

@ -0,0 +1,475 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchIllegalArgumentException;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexOrdinalFieldData;
import org.elasticsearch.index.fielddata.OrdinalsBytesValues;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import java.io.IOException;
/**
* Sorts by field's natural Term sort order, using
* ordinals. This is functionally equivalent to {@link
* org.apache.lucene.search.FieldComparator.TermValComparator}, but it first resolves the string
* to their relative ordinal positions (using the index
* returned by {@link org.apache.lucene.search.FieldCache#getTermsIndex}), and
* does most comparisons using the ordinals. For medium
* to large results, this comparator will be much faster
* than {@link org.apache.lucene.search.FieldComparator.TermValComparator}. For very small
* result sets it may be slower.
*/
public final class BytesRefOrdValComparator extends FieldComparator<BytesRef> {
final IndexOrdinalFieldData indexFieldData;
/* Ords for each slot.
@lucene.internal */
final int[] ords;
/* Values for each slot.
@lucene.internal */
final BytesRef[] values;
/* Which reader last copied a value into the slot. When
we compare two slots, we just compare-by-ord if the
readerGen is the same; else we must compare the
values (slower).
@lucene.internal */
final int[] readerGen;
/* Gen of current reader we are on.
@lucene.internal */
int currentReaderGen = -1;
/* Current reader's doc ord/values.
@lucene.internal */
OrdinalsBytesValues termsIndex;
/* Bottom slot, or -1 if queue isn't full yet
@lucene.internal */
int bottomSlot = -1;
/* Bottom ord (same as ords[bottomSlot] once bottomSlot
is set). Cached for faster compares.
@lucene.internal */
int bottomOrd;
/* True if current bottom slot matches the current
reader.
@lucene.internal */
boolean bottomSameReader;
/* Bottom value (same as values[bottomSlot] once
bottomSlot is set). Cached for faster compares.
@lucene.internal */
BytesRef bottomValue;
final BytesRef tempBR = new BytesRef();
public BytesRefOrdValComparator(IndexOrdinalFieldData indexFieldData, int numHits) {
this.indexFieldData = indexFieldData;
ords = new int[numHits];
values = new BytesRef[numHits];
readerGen = new int[numHits];
}
@Override
public int compare(int slot1, int slot2) {
if (readerGen[slot1] == readerGen[slot2]) {
return ords[slot1] - ords[slot2];
}
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
throw new UnsupportedOperationException();
}
@Override
public void copy(int slot, int doc) {
throw new UnsupportedOperationException();
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
BytesRef docValue = termsIndex.getValue(doc);
if (docValue == null) {
if (value == null) {
return 0;
}
return -1;
} else if (value == null) {
return 1;
}
return docValue.compareTo(value);
}
/**
* Base class for specialized (per bit width of the
* ords) per-segment comparator. NOTE: this is messy;
* we do this only because hotspot can't reliably inline
* the underlying array access when looking up doc->ord
*
* @lucene.internal
*/
abstract class PerSegmentComparator extends FieldComparator<BytesRef> {
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
return BytesRefOrdValComparator.this.setNextReader(context);
}
@Override
public int compare(int slot1, int slot2) {
return BytesRefOrdValComparator.this.compare(slot1, slot2);
}
@Override
public void setBottom(final int bottom) {
BytesRefOrdValComparator.this.setBottom(bottom);
}
@Override
public BytesRef value(int slot) {
return BytesRefOrdValComparator.this.value(slot);
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
return BytesRefOrdValComparator.this.compareDocToValue(doc, value);
}
}
// Used per-segment when bit width of doc->ord is 8:
private final class ByteOrdComparator extends PerSegmentComparator {
private final byte[] readerOrds;
private final OrdinalsBytesValues termsIndex;
private final int docBase;
public ByteOrdComparator(byte[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
this.readerOrds = readerOrds;
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final int docOrd = (readerOrds[doc] & 0xFF);
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = readerOrds[doc] & 0xFF;
ords[slot] = ord;
if (ord == 0) {
values[slot] = null;
} else {
assert ord > 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.getValueScratchByOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
// Used per-segment when bit width of doc->ord is 16:
private final class ShortOrdComparator extends PerSegmentComparator {
private final short[] readerOrds;
private final OrdinalsBytesValues termsIndex;
private final int docBase;
public ShortOrdComparator(short[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
this.readerOrds = readerOrds;
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final int docOrd = (readerOrds[doc] & 0xFFFF);
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = readerOrds[doc] & 0xFFFF;
ords[slot] = ord;
if (ord == 0) {
values[slot] = null;
} else {
assert ord > 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.getValueScratchByOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
// Used per-segment when bit width of doc->ord is 32:
private final class IntOrdComparator extends PerSegmentComparator {
private final int[] readerOrds;
private final OrdinalsBytesValues termsIndex;
private final int docBase;
public IntOrdComparator(int[] readerOrds, OrdinalsBytesValues termsIndex, int docBase) {
this.readerOrds = readerOrds;
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final int docOrd = readerOrds[doc];
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = readerOrds[doc];
ords[slot] = ord;
if (ord == 0) {
values[slot] = null;
} else {
assert ord > 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.getValueScratchByOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
// Used per-segment when bit width is not a native array
// size (8, 16, 32):
final class AnyOrdComparator extends PerSegmentComparator {
private final IndexFieldData fieldData;
private final Ordinals.Docs readerOrds;
private final OrdinalsBytesValues termsIndex;
private final int docBase;
public AnyOrdComparator(IndexFieldData fieldData, OrdinalsBytesValues termsIndex, int docBase) {
this.fieldData = fieldData;
this.readerOrds = termsIndex.ordinals();
this.termsIndex = termsIndex;
this.docBase = docBase;
}
@Override
public int compareBottom(int doc) {
assert bottomSlot != -1;
final int docOrd = readerOrds.getOrd(doc);
if (bottomSameReader) {
// ord is precisely comparable, even in the equal case
return bottomOrd - docOrd;
} else if (bottomOrd >= docOrd) {
// the equals case always means bottom is > doc
// (because we set bottomOrd to the lower bound in
// setBottom):
return 1;
} else {
return -1;
}
}
@Override
public void copy(int slot, int doc) {
final int ord = readerOrds.getOrd(doc);
ords[slot] = ord;
if (ord == 0) {
values[slot] = null;
} else {
assert ord > 0;
if (values[slot] == null) {
values[slot] = new BytesRef();
}
termsIndex.getValueScratchByOrd(ord, values[slot]);
}
readerGen[slot] = currentReaderGen;
}
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
final int docBase = context.docBase;
termsIndex = indexFieldData.load(context).getBytesValues();
// TODO, we should support sorting on multi valued field, take the best ascending value out of all the values
if (termsIndex.isMultiValued()) {
throw new ElasticSearchIllegalArgumentException("can't sort on a multi valued field");
}
final Ordinals.Docs docToOrd = termsIndex.ordinals();
Object ordsStorage = docToOrd.ordinals().getBackingStorage();
FieldComparator<BytesRef> perSegComp = null;
if (ordsStorage instanceof byte[]) {
perSegComp = new ByteOrdComparator((byte[]) ordsStorage, termsIndex, docBase);
} else if (ordsStorage instanceof short[]) {
perSegComp = new ShortOrdComparator((short[]) ordsStorage, termsIndex, docBase);
} else if (ordsStorage instanceof int[]) {
perSegComp = new IntOrdComparator((int[]) ordsStorage, termsIndex, docBase);
}
// Don't specialize the long[] case since it's not
// possible, ie, worse case is MAX_INT-1 docs with
// every one having a unique value.
// TODO: ES - should we optimize for the PackedInts.Reader case as well?
if (perSegComp == null) {
perSegComp = new AnyOrdComparator(indexFieldData, termsIndex, docBase);
}
currentReaderGen++;
if (bottomSlot != -1) {
perSegComp.setBottom(bottomSlot);
}
return perSegComp;
}
@Override
public void setBottom(final int bottom) {
bottomSlot = bottom;
bottomValue = values[bottomSlot];
if (currentReaderGen == readerGen[bottomSlot]) {
bottomOrd = ords[bottomSlot];
bottomSameReader = true;
} else {
if (bottomValue == null) {
// 0 ord is null for all segments
assert ords[bottomSlot] == 0;
bottomOrd = 0;
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
} else {
final int index = binarySearch(termsIndex, bottomValue);
if (index < 0) {
bottomOrd = -index - 2;
bottomSameReader = false;
} else {
bottomOrd = index;
// exact value match
bottomSameReader = true;
readerGen[bottomSlot] = currentReaderGen;
ords[bottomSlot] = bottomOrd;
}
}
}
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
final protected static int binarySearch(OrdinalsBytesValues a, BytesRef key) {
return binarySearch(a, key, 1, a.ordinals().getNumDocs() - 1);
}
final protected static int binarySearch(OrdinalsBytesValues a, BytesRef key, int low, int high) {
while (low <= high) {
int mid = (low + high) >>> 1;
BytesRef midVal = a.getValueByOrd(mid);
int cmp;
if (midVal != null) {
cmp = midVal.compareTo(key);
} else {
cmp = -1;
}
if (cmp < 0)
low = mid + 1;
else if (cmp > 0)
high = mid - 1;
else
return mid;
}
return -(low + 1);
}
}

119
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/BytesRefValComparator.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.index.fielddata.BytesValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import java.io.IOException;
/**
* Sorts by field's natural Term sort order. All
* comparisons are done using BytesRef.compareTo, which is
* slow for medium to large result sets but possibly
* very fast for very small results sets.
*/
public final class BytesRefValComparator extends FieldComparator<BytesRef> {
private final IndexFieldData indexFieldData;
private BytesRef[] values;
private BytesValues docTerms;
private BytesRef bottom;
BytesRefValComparator(IndexFieldData indexFieldData, int numHits) {
values = new BytesRef[numHits];
this.indexFieldData = indexFieldData;
}
@Override
public int compare(int slot1, int slot2) {
final BytesRef val1 = values[slot1];
final BytesRef val2 = values[slot2];
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareBottom(int doc) {
BytesRef val2 = docTerms.getValue(doc);
if (bottom == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return bottom.compareTo(val2);
}
@Override
public void copy(int slot, int doc) {
if (values[slot] == null) {
values[slot] = new BytesRef();
}
docTerms.getValueScratch(doc, values[slot]);
}
@Override
public FieldComparator<BytesRef> setNextReader(AtomicReaderContext context) throws IOException {
docTerms = indexFieldData.load(context).getBytesValues();
return this;
}
@Override
public void setBottom(final int bottom) {
this.bottom = values[bottom];
}
@Override
public BytesRef value(int slot) {
return values[slot];
}
@Override
public int compareValues(BytesRef val1, BytesRef val2) {
if (val1 == null) {
if (val2 == null) {
return 0;
}
return -1;
} else if (val2 == null) {
return 1;
}
return val1.compareTo(val2);
}
@Override
public int compareDocToValue(int doc, BytesRef value) {
return docTerms.getValue(doc).compareTo(value);
}
}

105
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparator.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.FieldComparator;
import org.elasticsearch.index.fielddata.DoubleValues;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import java.io.IOException;
/**
*/
public class DoubleValuesComparator extends FieldComparator<Double> {
private final IndexNumericFieldData indexFieldData;
private final double missingValue;
protected final double[] values;
private double bottom;
private DoubleValues readerValues;
public DoubleValuesComparator(IndexNumericFieldData indexFieldData, double missingValue, int numHits) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
this.values = new double[numHits];
}
@Override
public int compare(int slot1, int slot2) {
final double v1 = values[slot1];
final double v2 = values[slot2];
if (v1 > v2) {
return 1;
} else if (v1 < v2) {
return -1;
} else {
return 0;
}
}
@Override
public void setBottom(int slot) {
this.bottom = values[slot];
}
@Override
public int compareBottom(int doc) throws IOException {
double v2 = readerValues.getValueMissing(doc, missingValue);
if (bottom > v2) {
return 1;
} else if (bottom < v2) {
return -1;
} else {
return 0;
}
}
@Override
public void copy(int slot, int doc) throws IOException {
values[slot] = readerValues.getValueMissing(doc, missingValue);
}
@Override
public FieldComparator<Double> setNextReader(AtomicReaderContext context) throws IOException {
this.readerValues = indexFieldData.load(context).getDoubleValues();
return this;
}
@Override
public Double value(int slot) {
return Double.valueOf(values[slot]);
}
@Override
public int compareDocToValue(int doc, Double valueObj) throws IOException {
final double value = valueObj.doubleValue();
double docValue = readerValues.getValueMissing(doc, missingValue);
if (docValue < value) {
return -1;
} else if (docValue > value) {
return 1;
} else {
return 0;
}
}
}

62
src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/DoubleValuesComparatorSource.java Normal file
View File

@ -0,0 +1,62 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.fieldcomparator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import java.io.IOException;
/**
*/
public class DoubleValuesComparatorSource extends IndexFieldData.XFieldComparatorSource {
private final IndexNumericFieldData indexFieldData;
private final Object missingValue;
public DoubleValuesComparatorSource(IndexNumericFieldData indexFieldData, @Nullable Object missingValue) {
this.indexFieldData = indexFieldData;
this.missingValue = missingValue;
}
@Override
public SortField.Type reducedType() {
return SortField.Type.DOUBLE;
}
@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
assert fieldname.equals(indexFieldData.getFieldName());
double dMissingValue;
if (missingValue == null || "_last".equals(missingValue)) {
dMissingValue = reversed ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
} else if ("_first".equals(missingValue)) {
dMissingValue = reversed ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY;
} else {
dMissingValue = missingValue instanceof Number ? ((Number) missingValue).doubleValue() : Double.parseDouble(missingValue.toString());
}
return new DoubleValuesComparator(indexFieldData, dMissingValue, numHits);
}
}

122
src/main/java/org/elasticsearch/index/fielddata/ordinals/DocIdOrdinals.java Normal file
View File

@ -0,0 +1,122 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
/**
* Ordinals that effectively are single valued and map "one to one" to the
* doc ids. Note, the docId is incremented by 1 to get the ordinal, since 0
* denotes an empty value.
*/
public class DocIdOrdinals implements Ordinals {
private final int numDocs;
/**
* Constructs a new doc id ordinals.
*/
public DocIdOrdinals(int numDocs) {
this.numDocs = numDocs;
}
@Override
public Object getBackingStorage() {
return null;
}
@Override
public long getMemorySizeInBytes() {
return RamUsage.NUM_BYTES_OBJECT_REF;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getNumDocs() {
return numDocs;
}
@Override
public int getNumOrds() {
return numDocs + 1;
}
@Override
public Ordinals.Docs ordinals() {
return new Docs(this);
}
public static class Docs implements Ordinals.Docs {
private final DocIdOrdinals parent;
private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
private final SingleValueIter iter = new SingleValueIter();
public Docs(DocIdOrdinals parent) {
this.parent = parent;
}
@Override
public Ordinals ordinals() {
return parent;
}
@Override
public int getNumDocs() {
return parent.getNumDocs();
}
@Override
public int getNumOrds() {
return parent.getNumOrds();
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getOrd(int docId) {
return docId + 1;
}
@Override
public IntArrayRef getOrds(int docId) {
intsScratch.values[0] = docId + 1;
return intsScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(docId + 1);
}
@Override
public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
proc.onOrdinal(docId, docId + 1);
}
}
}

112
src/main/java/org/elasticsearch/index/fielddata/ordinals/EmptyOrdinals.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
/**
*/
public class EmptyOrdinals implements Ordinals {
private final int numDocs;
public EmptyOrdinals(int numDocs) {
this.numDocs = numDocs;
}
@Override
public long getMemorySizeInBytes() {
return 0;
}
@Override
public Object getBackingStorage() {
return null;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getNumDocs() {
return this.numDocs;
}
@Override
public int getNumOrds() {
return numDocs + 1;
}
@Override
public Docs ordinals() {
return new Docs(this);
}
public static class Docs implements Ordinals.Docs {
private final EmptyOrdinals parent;
public Docs(EmptyOrdinals parent) {
this.parent = parent;
}
@Override
public Ordinals ordinals() {
return parent;
}
@Override
public int getNumDocs() {
return parent.getNumDocs();
}
@Override
public int getNumOrds() {
return parent.getNumOrds();
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getOrd(int docId) {
return 0;
}
@Override
public IntArrayRef getOrds(int docId) {
return IntArrayRef.EMPTY;
}
@Override
public Iter getIter(int docId) {
return EmptyIter.INSTANCE;
}
@Override
public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
proc.onOrdinal(docId, 0);
}
}
}

187
src/main/java/org/elasticsearch/index/fielddata/ordinals/MultiFlatArrayOrdinals.java Normal file
View File

@ -0,0 +1,187 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
/**
* "Flat" multi valued ordinals, the first level array size is as the maximum
* values a docId has. Ordinals are populated in order from the first flat array
* value to the next.
*/
public class MultiFlatArrayOrdinals implements Ordinals {
private ThreadLocal<IntArrayRef> intArrayRefCache = new ThreadLocal<IntArrayRef>() {
@Override
protected IntArrayRef initialValue() {
return new IntArrayRef(new int[ordinals.length]);
}
};
// ordinals with value 0 indicates no value
private final int[][] ordinals;
private final int numDocs;
private final int numOrds;
private long size = -1;
public MultiFlatArrayOrdinals(int[][] ordinals, int numOrds) {
assert ordinals.length > 0;
this.ordinals = ordinals;
this.numDocs = ordinals[0].length;
this.numOrds = numOrds;
}
@Override
public Object getBackingStorage() {
return ordinals;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
long size = 0;
size += RamUsage.NUM_BYTES_ARRAY_HEADER; // for the top level array
for (int[] ordinal : ordinals) {
size += RamUsage.NUM_BYTES_INT * ordinal.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
}
this.size = size;
}
return size;
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public int getNumDocs() {
return numDocs;
}
@Override
public int getNumOrds() {
return numOrds;
}
@Override
public Docs ordinals() {
return new Docs(this, ordinals, intArrayRefCache.get());
}
public static class Docs implements Ordinals.Docs {
private final MultiFlatArrayOrdinals parent;
private final int[][] ordinals;
private final IterImpl iter;
private final IntArrayRef intsScratch;
public Docs(MultiFlatArrayOrdinals parent, int[][] ordinals, IntArrayRef intsScratch) {
this.parent = parent;
this.ordinals = ordinals;
this.iter = new IterImpl(ordinals);
this.intsScratch = intsScratch;
}
@Override
public Ordinals ordinals() {
return this.parent;
}
@Override
public int getNumDocs() {
return parent.getNumDocs();
}
@Override
public int getNumOrds() {
return parent.getNumOrds();
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public int getOrd(int docId) {
return ordinals[0][docId];
}
@Override
public IntArrayRef getOrds(int docId) {
intsScratch.end = 0;
int i;
for (i = 0; i < ordinals.length; i++) {
int ordinal = ordinals[i][docId];
if (ordinal == 0) {
if (i == 0) return IntArrayRef.EMPTY;
break;
}
intsScratch.values[i] = ordinal;
}
intsScratch.end = i;
return intsScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(docId);
}
@Override
public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
for (int i = 0; i < ordinals.length; i++) {
int ordinal = ordinals[i][docId];
if (ordinal == 0) {
if (i == 0) proc.onOrdinal(docId, 0);
return;
}
proc.onOrdinal(docId, ordinal);
}
}
public static class IterImpl implements Docs.Iter {
private final int[][] ordinals;
private int docId;
private int i;
public IterImpl(int[][] ordinals) {
this.ordinals = ordinals;
}
public IterImpl reset(int docId) {
this.docId = docId;
this.i = 0;
return this;
}
@Override
public int next() {
if (i >= ordinals.length) return 0;
return ordinals[i++][docId];
}
}
}
}

153
src/main/java/org/elasticsearch/index/fielddata/ordinals/Ordinals.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
/**
* A thread safe ordinals abstraction. Ordinals can only be positive integers.
*/
public interface Ordinals {
/**
* Returns the backing storage for this ordinals.
*/
Object getBackingStorage();
/**
* The memory size this ordinals take.
*/
long getMemorySizeInBytes();
/**
* Is one of the docs maps to more than one ordinal?
*/
boolean isMultiValued();
/**
* The number of docs in this ordinals.
*/
int getNumDocs();
/**
* The number of ordinals.
*/
int getNumOrds();
/**
* Returns a lightweight (non thread safe) view iterator of the ordinals.
*/
Docs ordinals();
/**
* A non thread safe ordinals abstraction, yet very lightweight to create. The idea
* is that this gets created for each "iteration" over ordinals.
* <p/>
* <p>A value of 0 ordinal when iterating indicated "no" value.</p>
*/
interface Docs {
/**
* Returns the original ordinals used to generate this Docs "itereator".
*/
Ordinals ordinals();
/**
* The number of docs in this ordinals.
*/
int getNumDocs();
/**
* The number of ordinals.
*/
int getNumOrds();
/**
* Is one of the docs maps to more than one ordinal?
*/
boolean isMultiValued();
/**
* The ordinal that maps to the relevant docId. If it has no value, returns
* <tt>0</tt>.
*/
int getOrd(int docId);
/**
* Returns an array of ordinals matching the docIds, with 0 length one
* for a doc with no ordinals.
*/
IntArrayRef getOrds(int docId);
/**
* Returns an iterator of the ordinals that match the docId, with an
* empty iterator for a doc with no ordinals.
*/
Iter getIter(int docId);
/**
* Iterates over the ordinals associated with a docId. If there are no values,
* a callback with a value 0 will be done.
*/
void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc);
public static interface OrdinalInDocProc {
void onOrdinal(int docId, int ordinal);
}
/**
* An iterator over ordinals values.
*/
interface Iter {
/**
* Gets the next ordinal. Returning 0 if the iteration is exhausted.
*/
int next();
}
static class EmptyIter implements Iter {
public static EmptyIter INSTANCE = new EmptyIter();
@Override
public int next() {
return 0;
}
}
static class SingleValueIter implements Iter {
private int value;
public SingleValueIter reset(int value) {
this.value = value;
return this;
}
@Override
public int next() {
int actual = value;
value = 0;
return actual;
}
}
}
}

129
src/main/java/org/elasticsearch/index/fielddata/ordinals/SingleArrayOrdinals.java Normal file
View File

@ -0,0 +1,129 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
/**
*/
public class SingleArrayOrdinals implements Ordinals {
// ordinals with value 0 indicates no value
private final int[] ordinals;
private final int numOrds;
private long size = -1;
public SingleArrayOrdinals(int[] ordinals, int numOrds) {
this.ordinals = ordinals;
this.numOrds = numOrds;
}
@Override
public Object getBackingStorage() {
return ordinals;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
size = RamUsage.NUM_BYTES_INT * ordinals.length + RamUsage.NUM_BYTES_ARRAY_HEADER;
}
return size;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getNumDocs() {
return ordinals.length;
}
@Override
public int getNumOrds() {
return numOrds;
}
@Override
public Docs ordinals() {
return new Docs(this, ordinals);
}
public static class Docs implements Ordinals.Docs {
private final SingleArrayOrdinals parent;
private final int[] ordinals;
private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
private final SingleValueIter iter = new SingleValueIter();
public Docs(SingleArrayOrdinals parent, int[] ordinals) {
this.parent = parent;
this.ordinals = ordinals;
}
@Override
public Ordinals ordinals() {
return parent;
}
@Override
public int getNumDocs() {
return parent.getNumDocs();
}
@Override
public int getNumOrds() {
return parent.getNumOrds();
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getOrd(int docId) {
return ordinals[docId];
}
@Override
public IntArrayRef getOrds(int docId) {
int ordinal = ordinals[docId];
if (ordinal == 0) return IntArrayRef.EMPTY;
intsScratch.values[0] = docId;
return intsScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals[docId]);
}
@Override
public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
proc.onOrdinal(docId, ordinals[docId]);
}
}
}

133
src/main/java/org/elasticsearch/index/fielddata/ordinals/SinglePackedOrdinals.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.ordinals;
import org.apache.lucene.util.packed.PackedInts;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
/**
*/
public class SinglePackedOrdinals implements Ordinals {
// ordinals with value 0 indicates no value
private final PackedInts.Reader reader;
private final int numOrds;
private long size = -1;
public SinglePackedOrdinals(PackedInts.Reader reader, int numOrds) {
this.reader = reader;
this.numOrds = numOrds;
}
@Override
public Object getBackingStorage() {
if (reader.hasArray()) {
return reader.getArray();
}
return reader;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
size = RamUsage.NUM_BYTES_OBJECT_REF + reader.ramBytesUsed();
}
return size;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getNumDocs() {
return reader.size();
}
@Override
public int getNumOrds() {
return numOrds;
}
@Override
public Docs ordinals() {
return new Docs(this, reader);
}
public static class Docs implements Ordinals.Docs {
private final SinglePackedOrdinals parent;
private final PackedInts.Reader reader;
private final IntArrayRef intsScratch = new IntArrayRef(new int[1]);
private final SingleValueIter iter = new SingleValueIter();
public Docs(SinglePackedOrdinals parent, PackedInts.Reader reader) {
this.parent = parent;
this.reader = reader;
}
@Override
public Ordinals ordinals() {
return parent;
}
@Override
public int getNumDocs() {
return parent.getNumDocs();
}
@Override
public int getNumOrds() {
return parent.getNumOrds();
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public int getOrd(int docId) {
return (int) reader.get(docId);
}
@Override
public IntArrayRef getOrds(int docId) {
int ordinal = (int) reader.get(docId);
if (ordinal == 0) return IntArrayRef.EMPTY;
intsScratch.values[0] = docId;
return intsScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset((int) reader.get(docId));
}
@Override
public void forEachOrdinalInDoc(int docId, OrdinalInDocProc proc) {
proc.onOrdinal(docId, (int) reader.get(docId));
}
}
}

673
src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefAtomicFieldData.java Normal file
View File

@ -0,0 +1,673 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public class ConcreteBytesRefAtomicFieldData implements AtomicOrdinalFieldData<ScriptDocValues.Strings> {
// 0 ordinal in values means no value (its null)
private final BytesRef[] values;
private final Ordinals ordinals;
private int[] hashes;
private long size = -1;
public ConcreteBytesRefAtomicFieldData(BytesRef[] values, Ordinals ordinals) {
this.values = values;
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public int getNumDocs() {
return ordinals.getNumDocs();
}
@Override
public boolean isValuesOrdered() {
return true;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
long size = RamUsage.NUM_BYTES_ARRAY_HEADER;
for (BytesRef value : values) {
if (value != null) {
size += RamUsage.NUM_BYTES_OBJECT_REF + RamUsage.NUM_BYTES_OBJECT_HEADER +
RamUsage.NUM_BYTES_ARRAY_HEADER + (value.length + (2 * RamUsage.NUM_BYTES_INT));
}
}
size += ordinals.getMemorySizeInBytes();
this.size = size;
}
return size;
}
@Override
public OrdinalsBytesValues getBytesValues() {
return ordinals.isMultiValued() ? new BytesValues.Multi(values, ordinals.ordinals()) : new BytesValues.Single(values, ordinals.ordinals());
}
@Override
public OrdinalsHashedBytesValues getHashedBytesValues() {
if (hashes == null) {
int[] hashes = new int[values.length];
for (int i = 0; i < values.length; i++) {
BytesRef value = values[i];
hashes[i] = value == null ? 0 : value.hashCode();
}
this.hashes = hashes;
}
return ordinals.isMultiValued() ? new HashedBytesValues.Multi(values, hashes, ordinals.ordinals()) : new HashedBytesValues.Single(values, hashes, ordinals.ordinals());
}
@Override
public OrdinalsStringValues getStringValues() {
return ordinals.isMultiValued() ? new StringValues.Multi(values, ordinals.ordinals()) : new StringValues.Single(values, ordinals.ordinals());
}
@Override
public ScriptDocValues.Strings getScriptValues() {
return new ScriptDocValues.Strings(getStringValues());
}
static abstract class BytesValues implements org.elasticsearch.index.fielddata.OrdinalsBytesValues {
protected final BytesRef[] values;
protected final Ordinals.Docs ordinals;
BytesValues(BytesRef[] values, Ordinals.Docs ordinals) {
this.values = values;
this.ordinals = ordinals;
}
@Override
public Ordinals.Docs ordinals() {
return this.ordinals;
}
@Override
public BytesRef getValueByOrd(int ord) {
return values[ord];
}
@Override
public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
BytesRef value = values[ord];
if (value == null) {
ret.length = 0;
} else {
ret.bytes = value.bytes;
ret.offset = value.offset;
ret.length = value.offset;
}
return ret;
}
@Override
public BytesRef getSafeValueByOrd(int ord) {
return values[ord];
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public BytesRef getValue(int docId) {
return values[ordinals.getOrd(docId)];
}
@Override
public BytesRef getValueScratch(int docId, BytesRef ret) {
BytesRef value = values[ordinals.getOrd(docId)];
if (value == null) {
ret.length = 0;
} else {
ret.bytes = value.bytes;
ret.offset = value.offset;
ret.length = value.length;
}
return ret;
}
@Override
public BytesRef getValueSafe(int docId) {
return values[ordinals.getOrd(docId)];
}
@Override
public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
forEachValueInDoc(docId, proc);
}
static class Single extends BytesValues {
private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[1], 1);
private final Iter.Single iter = new Iter.Single();
Single(BytesRef[] values, Ordinals.Docs ordinals) {
super(values, ordinals);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public BytesRefArrayRef getValues(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return BytesRefArrayRef.EMPTY;
arrayScratch.values[0] = values[ord];
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(values[ord]);
}
@Override
public Iter getIterSafe(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(values[ord]);
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
} else {
proc.onValue(docId, values[ord]);
}
}
}
static class Multi extends BytesValues {
private final BytesRefArrayRef arrayScratch = new BytesRefArrayRef(new BytesRef[10], 0);
private final ValuesIter iter;
Multi(BytesRef[] values, Ordinals.Docs ordinals) {
super(values, ordinals);
this.iter = new ValuesIter(values);
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public BytesRefArrayRef getValues(int docId) {
IntArrayRef ords = ordinals.getOrds(docId);
int size = ords.size();
if (size == 0) return BytesRefArrayRef.EMPTY;
arrayScratch.reset(size);
for (int i = ords.start; i < ords.end; i++) {
arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public Iter getIterSafe(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, values[ord]);
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements Iter {
private final BytesRef[] values;
private Ordinals.Docs.Iter ordsIter;
private int ord;
ValuesIter(BytesRef[] values) {
this.values = values;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public BytesRef next() {
BytesRef value = values[ord];
ord = ordsIter.next();
return value;
}
}
}
}
static abstract class HashedBytesValues implements org.elasticsearch.index.fielddata.OrdinalsHashedBytesValues {
protected final BytesRef[] values;
protected final int[] hashes;
protected final Ordinals.Docs ordinals;
protected final HashedBytesRef scratch = new HashedBytesRef();
HashedBytesValues(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
this.values = values;
this.hashes = hashes;
this.ordinals = ordinals;
}
@Override
public Ordinals.Docs ordinals() {
return this.ordinals;
}
@Override
public HashedBytesRef getValueByOrd(int ord) {
return scratch.reset(values[ord], hashes[ord]);
}
@Override
public HashedBytesRef getSafeValueByOrd(int ord) {
return new HashedBytesRef(values[ord], hashes[ord]);
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public HashedBytesRef getValue(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return null;
return scratch.reset(values[ord], hashes[ord]);
}
@Override
public HashedBytesRef getValueSafe(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return null;
return new HashedBytesRef(values[ord], hashes[ord]);
}
static class Single extends HashedBytesValues {
private final Iter.Single iter = new Iter.Single();
Single(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
super(values, hashes, ordinals);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(scratch.reset(values[ord], hashes[ord]));
}
@Override
public Iter getIterSafe(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(new HashedBytesRef(values[ord], hashes[ord]));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
} else {
proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
}
}
@Override
public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
} else {
proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
}
}
}
static class Multi extends HashedBytesValues {
private final ValuesIter iter;
private final SafeValuesIter safeIter;
Multi(BytesRef[] values, int[] hashes, Ordinals.Docs ordinals) {
super(values, hashes, ordinals);
this.iter = new ValuesIter(values, hashes);
this.safeIter = new SafeValuesIter(values, hashes);
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public Iter getIterSafe(int docId) {
return safeIter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, scratch.reset(values[ord], hashes[ord]));
} while ((ord = iter.next()) != 0);
}
@Override
public void forEachSafeValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, new HashedBytesRef(values[ord], hashes[ord]));
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements Iter {
private final BytesRef[] values;
private final int[] hashes;
private Ordinals.Docs.Iter ordsIter;
private int ord;
private final HashedBytesRef scratch = new HashedBytesRef();
ValuesIter(BytesRef[] values, int[] hashes) {
this.values = values;
this.hashes = hashes;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public HashedBytesRef next() {
HashedBytesRef value = scratch.reset(values[ord], hashes[ord]);
ord = ordsIter.next();
return value;
}
}
static class SafeValuesIter implements Iter {
private final BytesRef[] values;
private final int[] hashes;
private Ordinals.Docs.Iter ordsIter;
private int ord;
SafeValuesIter(BytesRef[] values, int[] hashes) {
this.values = values;
this.hashes = hashes;
}
public SafeValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public HashedBytesRef next() {
HashedBytesRef value = new HashedBytesRef(values[ord], hashes[ord]);
ord = ordsIter.next();
return value;
}
}
}
}
static abstract class StringValues implements OrdinalsStringValues {
protected final BytesRef[] values;
protected final Ordinals.Docs ordinals;
protected StringValues(BytesRef[] values, Ordinals.Docs ordinals) {
this.values = values;
this.ordinals = ordinals;
}
@Override
public Ordinals.Docs ordinals() {
return ordinals;
}
@Override
public String getValueByOrd(int ord) {
BytesRef value = values[ord];
if (value == null) return null;
return value.utf8ToString();
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public String getValue(int docId) {
BytesRef value = values[ordinals.getOrd(docId)];
if (value == null) return null;
return value.utf8ToString();
}
static class Single extends StringValues {
private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
private final Iter.Single iter = new Iter.Single();
Single(BytesRef[] values, Ordinals.Docs ordinals) {
super(values, ordinals);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public StringArrayRef getValues(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return StringArrayRef.EMPTY;
BytesRef value = values[ord];
arrayScratch.values[0] = value == null ? null : value.utf8ToString();
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) return Iter.Empty.INSTANCE;
return iter.reset(values[ord].utf8ToString());
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
proc.onMissing(docId);
return;
}
proc.onValue(docId, values[ord].utf8ToString());
}
}
static class Multi extends StringValues {
private final StringArrayRef arrayScratch = new StringArrayRef(new String[10], 0);
private final ValuesIter iter;
Multi(BytesRef[] values, Ordinals.Docs ordinals) {
super(values, ordinals);
iter = new ValuesIter(values);
}
@Override
public boolean isMultiValued() {
return true;
}
@Override
public StringArrayRef getValues(int docId) {
IntArrayRef ords = ordinals.getOrds(docId);
int size = ords.size();
if (size == 0) return StringArrayRef.EMPTY;
arrayScratch.reset(size);
for (int i = ords.start; i < ords.end; i++) {
BytesRef value = values[ords.values[i]];
arrayScratch.values[arrayScratch.end++] = value == null ? null : value.utf8ToString();
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
BytesRef value = values[ord];
proc.onValue(docId, value == null ? null : value.utf8ToString());
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements StringValues.Iter {
private final BytesRef[] values;
private Ordinals.Docs.Iter ordsIter;
private int ord;
ValuesIter(BytesRef[] values) {
this.values = values;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public String next() {
BytesRef value = values[ord];
ord = ordsIter.next();
return value == null ? null : value.utf8ToString();
}
}
}
}
}

155
src/main/java/org/elasticsearch/index/fielddata/plain/ConcreteBytesRefIndexFieldData.java Normal file
View File

@ -0,0 +1,155 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.index.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
import org.elasticsearch.index.fielddata.ordinals.SingleArrayOrdinals;
import org.elasticsearch.index.settings.IndexSettings;
import java.util.ArrayList;
/**
*/
public class ConcreteBytesRefIndexFieldData extends AbstractIndexFieldData<ConcreteBytesRefAtomicFieldData> implements IndexOrdinalFieldData<ConcreteBytesRefAtomicFieldData> {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
return new ConcreteBytesRefIndexFieldData(index, indexSettings, fieldName, type, cache);
}
}
public ConcreteBytesRefIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
super(index, indexSettings, fieldName, fieldDataType, cache);
}
@Override
public boolean valuesOrdered() {
return true;
}
@Override
public ConcreteBytesRefAtomicFieldData load(AtomicReaderContext context) {
try {
return cache.load(context, this);
} catch (Throwable e) {
if (e instanceof ElasticSearchException) {
throw (ElasticSearchException) e;
} else {
throw new ElasticSearchException(e.getMessage(), e);
}
}
}
@Override
public ConcreteBytesRefAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldName());
if (terms == null) {
return new ConcreteBytesRefAtomicFieldData(new BytesRef[1], new EmptyOrdinals(reader.maxDoc()));
}
long size = terms.size();
if (size == -1) {
size = 1024;
}
final ArrayList<BytesRef> values = new ArrayList<BytesRef>((int) size);
ArrayList<int[]> ordinals = new ArrayList<int[]>();
int[] idx = new int[reader.maxDoc()];
ordinals.add(new int[reader.maxDoc()]);
values.add(null); // first "t" indicates null value
int termOrd = 1; // current term number
TermsEnum termsEnum = terms.iterator(null);
try {
DocsEnum docsEnum = null;
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
values.add(BytesRef.deepCopyOf(term));
docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
int[] ordinal;
if (idx[docId] >= ordinals.size()) {
ordinal = new int[reader.maxDoc()];
ordinals.add(ordinal);
} else {
ordinal = ordinals.get(idx[docId]);
}
ordinal[docId] = termOrd;
idx[docId]++;
}
termOrd++;
}
} catch (RuntimeException e) {
if (e.getClass().getName().endsWith("StopFillCacheException")) {
// all is well, in case numeric parsers are used.
} else {
throw e;
}
}
if (ordinals.size() == 1) {
// optimizing to DocIdOrdinals which will use less memory removes the "order" notion of ordinals
// we can potentially do this, but only for things like facet processes
// that don't require order
// int[] nativeOrdinals = ordinals.get(0);
// boolean allHaveValue = true;
// for (int nativeOrdinal : nativeOrdinals) {
// if (nativeOrdinal == 0) {
// allHaveValue = false;
// break;
// }
// }
// if (allHaveValue) {
// BytesRef[] convertedValues = new BytesRef[values.size()];
// for (int i = 1; i < nativeOrdinals.length; i++) {
// convertedValues[i] = values.get(nativeOrdinals[i]);
// }
// return new ConcreteBytesRefAtomicFieldData(convertedValues, DocIdOrdinals.INSTANCE);
// } else {
// return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new SingleArrayOrdinals(nativeOrdinals));
// }
return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new SingleArrayOrdinals(ordinals.get(0), termOrd));
} else {
int[][] nativeOrdinals = new int[ordinals.size()][];
for (int i = 0; i < nativeOrdinals.length; i++) {
nativeOrdinals[i] = ordinals.get(i);
}
return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
}
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
// TODO support "missingValue" for sortMissingValue options here...
return new BytesRefFieldComparatorSource(this);
}
}

913
src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayAtomicFieldData.java Normal file
View File

@ -0,0 +1,913 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.common.RamUsage;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
import org.elasticsearch.index.fielddata.util.DoubleArrayRef;
import org.elasticsearch.index.fielddata.util.IntArrayRef;
import org.elasticsearch.index.fielddata.util.LongArrayRef;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
/**
*/
public abstract class DoubleArrayAtomicFieldData implements AtomicNumericFieldData {
protected final double[] values;
private final int numDocs;
protected long size = -1;
public DoubleArrayAtomicFieldData(double[] values, int numDocs) {
this.values = values;
this.numDocs = numDocs;
}
@Override
public int getNumDocs() {
return numDocs;
}
public static class WithOrdinals extends DoubleArrayAtomicFieldData {
private final Ordinals ordinals;
public WithOrdinals(double[] values, int numDocs, Ordinals ordinals) {
super(values, numDocs);
this.ordinals = ordinals;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public boolean isValuesOrdered() {
return true;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
size = RamUsage.NUM_BYTES_INT/*size*/ + RamUsage.NUM_BYTES_INT/*numDocs*/ + +RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + ordinals.getMemorySizeInBytes();
}
return size;
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values, ordinals.ordinals());
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
}
@Override
public ByteValues getByteValues() {
return new ByteValues.LongBased(getLongValues());
}
@Override
public ShortValues getShortValues() {
return new ShortValues.LongBased(getLongValues());
}
@Override
public IntValues getIntValues() {
return new IntValues.LongBased(getLongValues());
}
@Override
public LongValues getLongValues() {
return new LongValues(values, ordinals.ordinals());
}
@Override
public FloatValues getFloatValues() {
return new FloatValues.DoubleBased(getDoubleValues());
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, ordinals.ordinals());
}
static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
private final double[] values;
private final Ordinals.Docs ordinals;
private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
private final ValuesIter valuesIter;
StringValues(double[] values, Ordinals.Docs ordinals) {
this.values = values;
this.ordinals = ordinals;
this.valuesIter = new ValuesIter(values);
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public String getValue(int docId) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
return null;
}
return Double.toString(values[ord]);
}
@Override
public StringArrayRef getValues(int docId) {
IntArrayRef ords = ordinals.getOrds(docId);
int size = ords.size();
if (size == 0) return StringArrayRef.EMPTY;
arrayScratch.reset(size);
for (int i = ords.start; i < ords.end; i++) {
arrayScratch.values[arrayScratch.end++] = Double.toString(values[ords.values[i]]);
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return valuesIter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, Double.toString(values[ord]));
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements Iter {
private final double[] values;
private Ordinals.Docs.Iter ordsIter;
private int ord;
ValuesIter(double[] values) {
this.values = values;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public String next() {
String value = Double.toString(values[ord]);
ord = ordsIter.next();
return value;
}
}
}
static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
private final double[] values;
private final Ordinals.Docs ordinals;
private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
private final ValuesIter iter;
LongValues(double[] values, Ordinals.Docs ordinals) {
this.values = values;
this.ordinals = ordinals;
this.iter = new ValuesIter(values);
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public long getValue(int docId) {
return (long) values[ordinals.getOrd(docId)];
}
@Override
public long getValueMissing(int docId, long missingValue) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
return missingValue;
} else {
return (long) values[ord];
}
}
@Override
public LongArrayRef getValues(int docId) {
IntArrayRef ords = ordinals.getOrds(docId);
int size = ords.size();
if (size == 0) return LongArrayRef.EMPTY;
arrayScratch.reset(size);
for (int i = ords.start; i < ords.end; i++) {
arrayScratch.values[arrayScratch.end++] = (long) values[ords.values[i]];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, (long) values[ord]);
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements LongValues.Iter {
private final double[] values;
private Ordinals.Docs.Iter ordsIter;
private int ord;
ValuesIter(double[] values) {
this.values = values;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public long next() {
double value = values[ord];
ord = ordsIter.next();
return (long) value;
}
}
}
static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
private final double[] values;
private final Ordinals.Docs ordinals;
private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
private final ValuesIter iter;
DoubleValues(double[] values, Ordinals.Docs ordinals) {
this.values = values;
this.ordinals = ordinals;
this.iter = new ValuesIter(values);
}
@Override
public boolean isMultiValued() {
return ordinals.isMultiValued();
}
@Override
public boolean hasValue(int docId) {
return ordinals.getOrd(docId) != 0;
}
@Override
public double getValue(int docId) {
return values[ordinals.getOrd(docId)];
}
@Override
public double getValueMissing(int docId, double missingValue) {
int ord = ordinals.getOrd(docId);
if (ord == 0) {
return missingValue;
} else {
return values[ord];
}
}
@Override
public DoubleArrayRef getValues(int docId) {
IntArrayRef ords = ordinals.getOrds(docId);
int size = ords.size();
if (size == 0) return DoubleArrayRef.EMPTY;
arrayScratch.reset(size);
for (int i = ords.start; i < ords.end; i++) {
arrayScratch.values[arrayScratch.end++] = values[ords.values[i]];
}
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(ordinals.getIter(docId));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
Ordinals.Docs.Iter iter = ordinals.getIter(docId);
int ord = iter.next();
if (ord == 0) {
proc.onMissing(docId);
return;
}
do {
proc.onValue(docId, values[ord]);
} while ((ord = iter.next()) != 0);
}
static class ValuesIter implements DoubleValues.Iter {
private final double[] values;
private Ordinals.Docs.Iter ordsIter;
private int ord;
ValuesIter(double[] values) {
this.values = values;
}
public ValuesIter reset(Ordinals.Docs.Iter ordsIter) {
this.ordsIter = ordsIter;
this.ord = ordsIter.next();
return this;
}
@Override
public boolean hasNext() {
return ord != 0;
}
@Override
public double next() {
double value = values[ord];
ord = ordsIter.next();
return value;
}
}
}
}
/**
* A single valued case, where not all values are "set", so we have a FixedBitSet that
* indicates which values have an actual value.
*/
public static class SingleFixedSet extends DoubleArrayAtomicFieldData {
private final FixedBitSet set;
public SingleFixedSet(double[] values, int numDocs, FixedBitSet set) {
super(values, numDocs);
this.set = set;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean isValuesOrdered() {
return false;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE) + (set.getBits().length * RamUsage.NUM_BYTES_LONG);
}
return size;
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values, set);
}
@Override
public ByteValues getByteValues() {
return new ByteValues.LongBased(getLongValues());
}
@Override
public ShortValues getShortValues() {
return new ShortValues.LongBased(getLongValues());
}
@Override
public IntValues getIntValues() {
return new IntValues.LongBased(getLongValues());
}
@Override
public LongValues getLongValues() {
return new LongValues(values, set);
}
@Override
public FloatValues getFloatValues() {
return new FloatValues.DoubleBased(getDoubleValues());
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values, set);
}
static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
private final double[] values;
private final FixedBitSet set;
private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
private final Iter.Single iter = new Iter.Single();
StringValues(double[] values, FixedBitSet set) {
this.values = values;
this.set = set;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return set.get(docId);
}
@Override
public String getValue(int docId) {
if (set.get(docId)) {
return Double.toString(values[docId]);
} else {
return null;
}
}
@Override
public StringArrayRef getValues(int docId) {
if (set.get(docId)) {
arrayScratch.values[0] = Double.toString(values[docId]);
return arrayScratch;
} else {
return StringArrayRef.EMPTY;
}
}
@Override
public Iter getIter(int docId) {
if (set.get(docId)) {
return iter.reset(Double.toString(values[docId]));
} else {
return Iter.Empty.INSTANCE;
}
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
if (set.get(docId)) {
proc.onValue(docId, Double.toString(values[docId]));
} else {
proc.onMissing(docId);
}
}
}
static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
private final double[] values;
private final FixedBitSet set;
private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
private final Iter.Single iter = new Iter.Single();
LongValues(double[] values, FixedBitSet set) {
this.values = values;
this.set = set;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return set.get(docId);
}
@Override
public long getValue(int docId) {
return (long) values[docId];
}
@Override
public long getValueMissing(int docId, long missingValue) {
if (set.get(docId)) {
return (long) values[docId];
} else {
return missingValue;
}
}
@Override
public LongArrayRef getValues(int docId) {
if (set.get(docId)) {
arrayScratch.values[0] = (long) values[docId];
return arrayScratch;
} else {
return LongArrayRef.EMPTY;
}
}
@Override
public Iter getIter(int docId) {
if (set.get(docId)) {
return iter.reset((long) values[docId]);
} else {
return Iter.Empty.INSTANCE;
}
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
if (set.get(docId)) {
proc.onValue(docId, (long) values[docId]);
}
}
}
static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
private final double[] values;
private final FixedBitSet set;
private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
private final Iter.Single iter = new Iter.Single();
DoubleValues(double[] values, FixedBitSet set) {
this.values = values;
this.set = set;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return set.get(docId);
}
@Override
public double getValue(int docId) {
return values[docId];
}
@Override
public double getValueMissing(int docId, double missingValue) {
if (set.get(docId)) {
return values[docId];
} else {
return missingValue;
}
}
@Override
public DoubleArrayRef getValues(int docId) {
if (set.get(docId)) {
arrayScratch.values[0] = values[docId];
return arrayScratch;
} else {
return DoubleArrayRef.EMPTY;
}
}
@Override
public Iter getIter(int docId) {
if (set.get(docId)) {
return iter.reset(values[docId]);
} else {
return Iter.Empty.INSTANCE;
}
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
if (set.get(docId)) {
proc.onValue(docId, values[docId]);
}
}
}
}
/**
* Assumes all the values are "set", and docId is used as the index to the value array.
*/
public static class Single extends DoubleArrayAtomicFieldData {
/**
* Note, here, we assume that there is no offset by 1 from docId, so position 0
* is the value for docId 0.
*/
public Single(double[] values, int numDocs) {
super(values, numDocs);
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean isValuesOrdered() {
return false;
}
@Override
public long getMemorySizeInBytes() {
if (size == -1) {
size = RamUsage.NUM_BYTES_ARRAY_HEADER + (values.length * RamUsage.NUM_BYTES_DOUBLE);
}
return size;
}
@Override
public ScriptDocValues getScriptValues() {
return new ScriptDocValues.NumericDouble(getDoubleValues());
}
@Override
public BytesValues getBytesValues() {
return new BytesValues.StringBased(getStringValues());
}
@Override
public HashedBytesValues getHashedBytesValues() {
return new HashedBytesValues.StringBased(getStringValues());
}
@Override
public StringValues getStringValues() {
return new StringValues(values);
}
@Override
public ByteValues getByteValues() {
return new ByteValues.LongBased(getLongValues());
}
@Override
public ShortValues getShortValues() {
return new ShortValues.LongBased(getLongValues());
}
@Override
public IntValues getIntValues() {
return new IntValues.LongBased(getLongValues());
}
@Override
public LongValues getLongValues() {
return new LongValues(values);
}
@Override
public FloatValues getFloatValues() {
return new FloatValues.DoubleBased(getDoubleValues());
}
@Override
public DoubleValues getDoubleValues() {
return new DoubleValues(values);
}
static class StringValues implements org.elasticsearch.index.fielddata.StringValues {
private final double[] values;
private final StringArrayRef arrayScratch = new StringArrayRef(new String[1], 1);
private final Iter.Single iter = new Iter.Single();
StringValues(double[] values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return true;
}
@Override
public String getValue(int docId) {
return Double.toString(values[docId]);
}
@Override
public StringArrayRef getValues(int docId) {
arrayScratch.values[0] = Double.toString(values[docId]);
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(Double.toString(values[docId]));
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
proc.onValue(docId, Double.toString(values[docId]));
}
}
static class LongValues implements org.elasticsearch.index.fielddata.LongValues {
private final double[] values;
private final LongArrayRef arrayScratch = new LongArrayRef(new long[1], 1);
private final Iter.Single iter = new Iter.Single();
LongValues(double[] values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return true;
}
@Override
public long getValue(int docId) {
return (long) values[docId];
}
@Override
public long getValueMissing(int docId, long missingValue) {
return (long) values[docId];
}
@Override
public LongArrayRef getValues(int docId) {
arrayScratch.values[0] = (long) values[docId];
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset((long) values[docId]);
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
proc.onValue(docId, (long) values[docId]);
}
}
static class DoubleValues implements org.elasticsearch.index.fielddata.DoubleValues {
private final double[] values;
private final DoubleArrayRef arrayScratch = new DoubleArrayRef(new double[1], 1);
private final Iter.Single iter = new Iter.Single();
DoubleValues(double[] values) {
this.values = values;
}
@Override
public boolean isMultiValued() {
return false;
}
@Override
public boolean hasValue(int docId) {
return true;
}
@Override
public double getValue(int docId) {
return values[docId];
}
@Override
public double getValueMissing(int docId, double missingValue) {
return values[docId];
}
@Override
public DoubleArrayRef getValues(int docId) {
arrayScratch.values[0] = values[docId];
return arrayScratch;
}
@Override
public Iter getIter(int docId) {
return iter.reset(values[docId]);
}
@Override
public void forEachValueInDoc(int docId, ValueInDocProc proc) {
proc.onValue(docId, values[docId]);
}
}
}
}

156
src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java Normal file
View File

@ -0,0 +1,156 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.plain;
import gnu.trove.list.array.TDoubleArrayList;
import org.apache.lucene.index.*;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.elasticsearch.ElasticSearchException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource;
import org.elasticsearch.index.fielddata.ordinals.MultiFlatArrayOrdinals;
import org.elasticsearch.index.settings.IndexSettings;
import java.util.ArrayList;
/**
*/
public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArrayAtomicFieldData> implements IndexNumericFieldData<DoubleArrayAtomicFieldData> {
public static class Builder implements IndexFieldData.Builder {
@Override
public IndexFieldData build(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType type, IndexFieldDataCache cache) {
return new DoubleArrayIndexFieldData(index, indexSettings, fieldName, type, cache);
}
}
public DoubleArrayIndexFieldData(Index index, @IndexSettings Settings indexSettings, String fieldName, FieldDataType fieldDataType, IndexFieldDataCache cache) {
super(index, indexSettings, fieldName, fieldDataType, cache);
}
@Override
public NumericType getNumericType() {
return NumericType.DOUBLE;
}
@Override
public boolean valuesOrdered() {
// because we might have single values? we can dynamically update a flag to reflect that
// based on the atomic field data loaded
return false;
}
@Override
public DoubleArrayAtomicFieldData load(AtomicReaderContext context) {
try {
return cache.load(context, this);
} catch (Throwable e) {
if (e instanceof ElasticSearchException) {
throw (ElasticSearchException) e;
} else {
throw new ElasticSearchException(e.getMessage(), e);
}
}
}
@Override
public DoubleArrayAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
AtomicReader reader = context.reader();
Terms terms = reader.terms(getFieldName());
if (terms == null) {
return new DoubleArrayAtomicFieldData.Single(new double[0], 0);
}
// TODO: how can we guess the number of terms? numerics end up creating more terms per value...
final TDoubleArrayList values = new TDoubleArrayList();
ArrayList<int[]> ordinals = new ArrayList<int[]>();
int[] idx = new int[reader.maxDoc()];
ordinals.add(new int[reader.maxDoc()]);
values.add(0); // first "t" indicates null value
int termOrd = 1; // current term number
TermsEnum termsEnum = terms.iterator(null);
try {
DocsEnum docsEnum = null;
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) {
values.add(FieldCache.NUMERIC_UTILS_DOUBLE_PARSER.parseDouble(term));
docsEnum = termsEnum.docs(reader.getLiveDocs(), docsEnum, 0);
for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
int[] ordinal;
if (idx[docId] >= ordinals.size()) {
ordinal = new int[reader.maxDoc()];
ordinals.add(ordinal);
} else {
ordinal = ordinals.get(idx[docId]);
}
ordinal[docId] = termOrd;
idx[docId]++;
}
termOrd++;
}
} catch (RuntimeException e) {
if (e.getClass().getName().endsWith("StopFillCacheException")) {
// all is well, in case numeric parsers are used.
} else {
throw e;
}
}
if (ordinals.size() == 1) {
int[] nativeOrdinals = ordinals.get(0);
FixedBitSet set = new FixedBitSet(reader.maxDoc());
double[] sValues = new double[reader.maxDoc()];
boolean allHaveValue = true;
for (int i = 0; i < nativeOrdinals.length; i++) {
int nativeOrdinal = nativeOrdinals[i];
if (nativeOrdinal == 0) {
allHaveValue = false;
} else {
set.set(i);
sValues[i] = values.get(nativeOrdinal);
}
}
if (allHaveValue) {
return new DoubleArrayAtomicFieldData.Single(sValues, reader.maxDoc());
} else {
return new DoubleArrayAtomicFieldData.SingleFixedSet(sValues, reader.maxDoc(), set);
}
} else {
int[][] nativeOrdinals = new int[ordinals.size()][];
for (int i = 0; i < nativeOrdinals.length; i++) {
nativeOrdinals[i] = ordinals.get(i);
}
return new DoubleArrayAtomicFieldData.WithOrdinals(values.toArray(new double[values.size()]), reader.maxDoc(), new MultiFlatArrayOrdinals(nativeOrdinals, termOrd));
}
}
@Override
public XFieldComparatorSource comparatorSource(@Nullable Object missingValue) {
return new DoubleValuesComparatorSource(this, missingValue);
}
}

172
src/main/java/org/elasticsearch/index/fielddata/util/ByteArrayRef.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import com.google.common.primitives.Bytes;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class ByteArrayRef extends AbstractList<Byte> implements RandomAccess {
public static final ByteArrayRef EMPTY = new ByteArrayRef(new byte[0]);
public byte[] values;
public int start;
public int end;
public ByteArrayRef(byte[] values) {
this(values, 0, values.length);
}
public ByteArrayRef(byte[] values, int length) {
this(values, 0, length);
}
public ByteArrayRef(byte[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new byte[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public Byte get(int index) {
assert index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
// Overridden to prevent a ton of boxing
return (target instanceof Byte)
&& indexOf(values, (Byte) target, start, end) != -1;
}
@Override
public int indexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Byte) {
int i = indexOf(values, (Byte) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Byte) {
int i = lastIndexOf(values, (Byte) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public Byte set(int index, Byte element) {
assert index < size();
byte oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof ByteArrayRef) {
ByteArrayRef that = (ByteArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (values[start + i] != that.values[that.start + i]) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + Bytes.hashCode(values[i]);
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
private static int indexOf(byte[] array, byte target, int start, int end) {
for (int i = start; i < end; i++) {
if (array[i] == target) {
return i;
}
}
return -1;
}
private static int lastIndexOf(byte[] array, byte target, int start, int end) {
for (int i = end - 1; i >= start; i--) {
if (array[i] == target) {
return i;
}
}
return -1;
}
}

59
src/main/java/org/elasticsearch/index/fielddata/util/BytesRefArrayRef.java Normal file
View File

@ -0,0 +1,59 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
/**
*/
public class BytesRefArrayRef {
public static final BytesRefArrayRef EMPTY = new BytesRefArrayRef(new BytesRef[0]);
public BytesRef[] values;
public int start;
public int end;
public BytesRefArrayRef(BytesRef[] values) {
this(values, 0, values.length);
}
public BytesRefArrayRef(BytesRef[] values, int length) {
this(values, 0, length);
}
public BytesRefArrayRef(BytesRef[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new BytesRef[ArrayUtil.oversize(newLength, 32)];
}
}
public int size() {
return end - start;
}
}

172
src/main/java/org/elasticsearch/index/fielddata/util/DoubleArrayRef.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import com.google.common.primitives.Doubles;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class DoubleArrayRef extends AbstractList<Double> implements RandomAccess {
public static final DoubleArrayRef EMPTY = new DoubleArrayRef(new double[0]);
public double[] values;
public int start;
public int end;
public DoubleArrayRef(double[] values) {
this(values, 0, values.length);
}
public DoubleArrayRef(double[] values, int length) {
this(values, 0, length);
}
public DoubleArrayRef(double[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new double[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public Double get(int index) {
assert index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
// Overridden to prevent a ton of boxing
return (target instanceof Double)
&& indexOf(values, (Double) target, start, end) != -1;
}
@Override
public int indexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Double) {
int i = indexOf(values, (Double) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Double) {
int i = lastIndexOf(values, (Double) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public Double set(int index, Double element) {
assert index < size();
double oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof DoubleArrayRef) {
DoubleArrayRef that = (DoubleArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (values[start + i] != that.values[that.start + i]) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + Doubles.hashCode(values[i]);
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
private static int indexOf(double[] array, double target, int start, int end) {
for (int i = start; i < end; i++) {
if (array[i] == target) {
return i;
}
}
return -1;
}
private static int lastIndexOf(double[] array, double target, int start, int end) {
for (int i = end - 1; i >= start; i--) {
if (array[i] == target) {
return i;
}
}
return -1;
}
}

172
src/main/java/org/elasticsearch/index/fielddata/util/FloatArrayRef.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import com.google.common.primitives.Floats;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class FloatArrayRef extends AbstractList<Float> implements RandomAccess {
public static final FloatArrayRef EMPTY = new FloatArrayRef(new float[0]);
public float[] values;
public int start;
public int end;
public FloatArrayRef(float[] values) {
this(values, 0, values.length);
}
public FloatArrayRef(float[] values, int length) {
this(values, 0, length);
}
public FloatArrayRef(float[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new float[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public Float get(int index) {
assert index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
// Overridden to prevent a ton of boxing
return (target instanceof Float)
&& indexOf(values, (Float) target, start, end) != -1;
}
@Override
public int indexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Float) {
int i = indexOf(values, (Float) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Float) {
int i = lastIndexOf(values, (Float) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public Float set(int index, Float element) {
assert index < size();
float oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof FloatArrayRef) {
FloatArrayRef that = (FloatArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (values[start + i] != that.values[that.start + i]) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + Floats.hashCode(values[i]);
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
private static int indexOf(float[] array, float target, int start, int end) {
for (int i = start; i < end; i++) {
if (array[i] == target) {
return i;
}
}
return -1;
}
private static int lastIndexOf(float[] array, float target, int start, int end) {
for (int i = end - 1; i >= start; i--) {
if (array[i] == target) {
return i;
}
}
return -1;
}
}

172
src/main/java/org/elasticsearch/index/fielddata/util/IntArrayRef.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import com.google.common.primitives.Ints;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class IntArrayRef extends AbstractList<Integer> implements RandomAccess {
public static final IntArrayRef EMPTY = new IntArrayRef(new int[0]);
public int[] values;
public int start;
public int end;
public IntArrayRef(int[] values) {
this(values, 0, values.length);
}
public IntArrayRef(int[] values, int length) {
this(values, 0, length);
}
public IntArrayRef(int[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new int[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public Integer get(int index) {
assert index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
// Overridden to prevent a ton of boxing
return (target instanceof Integer)
&& indexOf(values, (Integer) target, start, end) != -1;
}
@Override
public int indexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Integer) {
int i = indexOf(values, (Integer) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Integer) {
int i = lastIndexOf(values, (Integer) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public Integer set(int index, Integer element) {
assert index < size();
int oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof IntArrayRef) {
IntArrayRef that = (IntArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (values[start + i] != that.values[that.start + i]) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + Ints.hashCode(values[i]);
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
private static int indexOf(int[] array, int target, int start, int end) {
for (int i = start; i < end; i++) {
if (array[i] == target) {
return i;
}
}
return -1;
}
private static int lastIndexOf(int[] array, int target, int start, int end) {
for (int i = end - 1; i >= start; i--) {
if (array[i] == target) {
return i;
}
}
return -1;
}
}

172
src/main/java/org/elasticsearch/index/fielddata/util/LongArrayRef.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import com.google.common.primitives.Longs;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class LongArrayRef extends AbstractList<Long> implements RandomAccess {
public static final LongArrayRef EMPTY = new LongArrayRef(new long[0]);
public long[] values;
public int start;
public int end;
public LongArrayRef(long[] values) {
this(values, 0, values.length);
}
public LongArrayRef(long[] values, int length) {
this(values, 0, length);
}
public LongArrayRef(long[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new long[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public Long get(int index) {
assert index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
// Overridden to prevent a ton of boxing
return (target instanceof Long)
&& indexOf(values, (Long) target, start, end) != -1;
}
@Override
public int indexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Long) {
int i = indexOf(values, (Long) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Long) {
int i = lastIndexOf(values, (Long) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public Long set(int index, Long element) {
assert index < size();
long oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof LongArrayRef) {
LongArrayRef that = (LongArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (values[start + i] != that.values[that.start + i]) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + Longs.hashCode(values[i]);
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
private static int indexOf(long[] array, long target, int start, int end) {
for (int i = start; i < end; i++) {
if (array[i] == target) {
return i;
}
}
return -1;
}
private static int lastIndexOf(long[] array, long target, int start, int end) {
for (int i = end - 1; i >= start; i--) {
if (array[i] == target) {
return i;
}
}
return -1;
}
}

172
src/main/java/org/elasticsearch/index/fielddata/util/ShortArrayRef.java Normal file
View File

@ -0,0 +1,172 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import com.google.common.primitives.Shorts;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class ShortArrayRef extends AbstractList<Short> implements RandomAccess {
public static final ShortArrayRef EMPTY = new ShortArrayRef(new short[0]);
public short[] values;
public int start;
public int end;
public ShortArrayRef(short[] values) {
this(values, 0, values.length);
}
public ShortArrayRef(short[] values, int length) {
this(values, 0, length);
}
public ShortArrayRef(short[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new short[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public Short get(int index) {
assert index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
// Overridden to prevent a ton of boxing
return (target instanceof Short)
&& indexOf(values, (Short) target, start, end) != -1;
}
@Override
public int indexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Short) {
int i = indexOf(values, (Short) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
// Overridden to prevent a ton of boxing
if (target instanceof Short) {
int i = lastIndexOf(values, (Short) target, start, end);
if (i >= 0) {
return i - start;
}
}
return -1;
}
@Override
public Short set(int index, Short element) {
assert index < size();
short oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof ShortArrayRef) {
ShortArrayRef that = (ShortArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (values[start + i] != that.values[that.start + i]) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + Shorts.hashCode(values[i]);
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
private static int indexOf(short[] array, short target, int start, int end) {
for (int i = start; i < end; i++) {
if (array[i] == target) {
return i;
}
}
return -1;
}
private static int lastIndexOf(short[] array, short target, int start, int end) {
for (int i = end - 1; i >= start; i--) {
if (array[i] == target) {
return i;
}
}
return -1;
}
}

149
src/main/java/org/elasticsearch/index/fielddata/util/StringArrayRef.java Normal file
View File

@ -0,0 +1,149 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.fielddata.util;
import org.apache.lucene.util.ArrayUtil;
import java.util.AbstractList;
import java.util.RandomAccess;
/**
*/
public class StringArrayRef extends AbstractList<String> implements RandomAccess {
public static final StringArrayRef EMPTY = new StringArrayRef(new String[0]);
public String[] values;
public int start;
public int end;
public StringArrayRef(String[] values) {
this(values, 0, values.length);
}
public StringArrayRef(String[] values, int length) {
this(values, 0, length);
}
public StringArrayRef(String[] values, int start, int end) {
this.values = values;
this.start = start;
this.end = end;
}
public void reset(int newLength) {
assert start == 0; // NOTE: senseless if offset != 0
end = 0;
if (values.length < newLength) {
values = new String[ArrayUtil.oversize(newLength, 32)];
}
}
@Override
public int size() {
return end - start;
}
@Override
public boolean isEmpty() {
return size() != 0;
}
@Override
public String get(int index) {
assert index >= 0 && index < size();
return values[start + index];
}
@Override
public boolean contains(Object target) {
String sTarget = target.toString();
for (int i = start; i < end; i++) {
if (values[i].equals(sTarget)) return true;
}
return false;
}
@Override
public int indexOf(Object target) {
String sTarget = target.toString();
for (int i = start; i < end; i++) {
if (values[i].equals(sTarget)) return (i - start);
}
return -1;
}
@Override
public int lastIndexOf(Object target) {
String sTarget = target.toString();
for (int i = end - 1; i >= start; i--) {
if (values[i].equals(sTarget)) return (i - start);
}
return -1;
}
@Override
public String set(int index, String element) {
assert index >= 0 && index < size();
String oldValue = values[start + index];
values[start + index] = element;
return oldValue;
}
@Override
public boolean equals(Object object) {
if (object == this) {
return true;
}
if (object instanceof StringArrayRef) {
StringArrayRef that = (StringArrayRef) object;
int size = size();
if (that.size() != size) {
return false;
}
for (int i = 0; i < size; i++) {
if (!values[start + i].equals(that.values[that.start + i])) {
return false;
}
}
return true;
}
return super.equals(object);
}
@Override
public int hashCode() {
int result = 1;
for (int i = start; i < end; i++) {
result = 31 * result + values[i].hashCode();
}
return result;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder(size() * 10);
builder.append('[').append(values[start]);
for (int i = start + 1; i < end; i++) {
builder.append(", ").append(values[i]);
}
return builder.append(']').toString();
}
}

413
src/test/java/org/elasticsearch/test/unit/index/fielddata/AbstractFieldDataTests.java Normal file
View File

@ -0,0 +1,413 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.fielddata;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.index.*;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.fielddata.*;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.List;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.sameInstance;
/**
*/
@Test
public abstract class AbstractFieldDataTests {
protected IndexFieldDataService ifdService;
protected IndexWriter writer;
protected AtomicReaderContext readerContext;
protected abstract FieldDataType getFieldDataType();
public <IFD extends IndexFieldData> IFD getForField(String fieldName) {
return ifdService.getForField(fieldName, getFieldDataType());
}
@BeforeMethod
public void setup() throws Exception {
ifdService = new IndexFieldDataService(new Index("test"));
writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(Lucene.VERSION, new StandardAnalyzer(Lucene.VERSION)));
}
protected AtomicReaderContext refreshReader() throws Exception {
if (readerContext != null) {
readerContext.reader().close();
}
AtomicReader reader = new SlowCompositeReaderWrapper(DirectoryReader.open(writer, true));
readerContext = reader.getContext();
return readerContext;
}
@AfterMethod
public void tearDown() throws Exception {
if (readerContext != null) {
readerContext.reader().close();
}
writer.close();
ifdService.clear();
}
public static class BytesValuesVerifierProc implements BytesValues.ValueInDocProc {
private static final BytesRef MISSING = new BytesRef();
private final int docId;
private final List<BytesRef> expected = new ArrayList<BytesRef>();
private int idx;
BytesValuesVerifierProc(int docId) {
this.docId = docId;
}
public BytesValuesVerifierProc addExpected(String value) {
expected.add(new BytesRef(value));
return this;
}
public BytesValuesVerifierProc addExpected(BytesRef value) {
expected.add(value);
return this;
}
public BytesValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, BytesRef value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class HashedBytesValuesVerifierProc implements HashedBytesValues.ValueInDocProc {
private static final HashedBytesRef MISSING = new HashedBytesRef();
private final int docId;
private final List<HashedBytesRef> expected = new ArrayList<HashedBytesRef>();
private int idx;
HashedBytesValuesVerifierProc(int docId) {
this.docId = docId;
}
public HashedBytesValuesVerifierProc addExpected(String value) {
expected.add(new HashedBytesRef(value));
return this;
}
public HashedBytesValuesVerifierProc addExpected(BytesRef value) {
expected.add(new HashedBytesRef(value));
return this;
}
public HashedBytesValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, HashedBytesRef value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class StringValuesVerifierProc implements StringValues.ValueInDocProc {
private static final String MISSING = new String();
private final int docId;
private final List<String> expected = new ArrayList<String>();
private int idx;
StringValuesVerifierProc(int docId) {
this.docId = docId;
}
public StringValuesVerifierProc addExpected(String value) {
expected.add(value);
return this;
}
public StringValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, String value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class LongValuesVerifierProc implements LongValues.ValueInDocProc {
private static final Long MISSING = new Long(0);
private final int docId;
private final List<Long> expected = new ArrayList<Long>();
private int idx;
LongValuesVerifierProc(int docId) {
this.docId = docId;
}
public LongValuesVerifierProc addExpected(long value) {
expected.add(value);
return this;
}
public LongValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, long value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class DoubleValuesVerifierProc implements DoubleValues.ValueInDocProc {
private static final Double MISSING = new Double(0);
private final int docId;
private final List<Double> expected = new ArrayList<Double>();
private int idx;
DoubleValuesVerifierProc(int docId) {
this.docId = docId;
}
public DoubleValuesVerifierProc addExpected(double value) {
expected.add(value);
return this;
}
public DoubleValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, double value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class ByteValuesVerifierProc implements ByteValues.ValueInDocProc {
private static final Byte MISSING = new Byte((byte) 0);
private final int docId;
private final List<Byte> expected = new ArrayList<Byte>();
private int idx;
ByteValuesVerifierProc(int docId) {
this.docId = docId;
}
public ByteValuesVerifierProc addExpected(byte value) {
expected.add(value);
return this;
}
public ByteValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, byte value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class ShortValuesVerifierProc implements ShortValues.ValueInDocProc {
private static final Short MISSING = new Short((byte) 0);
private final int docId;
private final List<Short> expected = new ArrayList<Short>();
private int idx;
ShortValuesVerifierProc(int docId) {
this.docId = docId;
}
public ShortValuesVerifierProc addExpected(short value) {
expected.add(value);
return this;
}
public ShortValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, short value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class IntValuesVerifierProc implements IntValues.ValueInDocProc {
private static final Integer MISSING = new Integer(0);
private final int docId;
private final List<Integer> expected = new ArrayList<Integer>();
private int idx;
IntValuesVerifierProc(int docId) {
this.docId = docId;
}
public IntValuesVerifierProc addExpected(int value) {
expected.add(value);
return this;
}
public IntValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, int value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
public static class FloatValuesVerifierProc implements FloatValues.ValueInDocProc {
private static final Float MISSING = new Float(0);
private final int docId;
private final List<Float> expected = new ArrayList<Float>();
private int idx;
FloatValuesVerifierProc(int docId) {
this.docId = docId;
}
public FloatValuesVerifierProc addExpected(float value) {
expected.add(value);
return this;
}
public FloatValuesVerifierProc addMissing() {
expected.add(MISSING);
return this;
}
@Override
public void onValue(int docId, float value) {
assertThat(docId, equalTo(this.docId));
assertThat(value, equalTo(expected.get(idx++)));
}
@Override
public void onMissing(int docId) {
assertThat(docId, equalTo(this.docId));
assertThat(MISSING, sameInstance(expected.get(idx++)));
}
}
}

35
src/test/java/org/elasticsearch/test/unit/index/fielddata/ConcreteBytesStringFieldDataTests.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.fielddata;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.testng.annotations.Test;
/**
*/
@Test
public class ConcreteBytesStringFieldDataTests extends StringFieldDataTests {
@Override
protected FieldDataType getFieldDataType() {
return new FieldDataType("string", "concrete_bytes", ImmutableMap.<String, String>of());
}
}

32
src/test/java/org/elasticsearch/test/unit/index/fielddata/DoubleFieldDataTests.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.fielddata;
import org.elasticsearch.index.fielddata.FieldDataType;
/**
*/
public class DoubleFieldDataTests extends NumericFieldDataTests {
@Override
protected FieldDataType getFieldDataType() {
return new FieldDataType("double");
}
}

1344
src/test/java/org/elasticsearch/test/unit/index/fielddata/NumericFieldDataTests.java Normal file
View File

File diff suppressed because it is too large Load Diff

741
src/test/java/org/elasticsearch/test/unit/index/fielddata/StringFieldDataTests.java Normal file
View File

@ -0,0 +1,741 @@
/*
* Licensed to ElasticSearch and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. ElasticSearch licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.test.unit.index.fielddata;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.search.*;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.lucene.HashedBytesRef;
import org.elasticsearch.index.fielddata.*;
import org.elasticsearch.index.fielddata.util.BytesRefArrayRef;
import org.elasticsearch.index.fielddata.util.StringArrayRef;
import org.testng.annotations.Test;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.nullValue;
/**
*/
@Test
public abstract class StringFieldDataTests extends AbstractFieldDataTests {
protected String one() {
return "1";
}
protected String two() {
return "2";
}
protected String three() {
return "3";
}
protected String four() {
return "4";
}
protected void fillSingleValueAllSet() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "1", Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
writer.addDocument(d);
}
@Test
public void testSingleValueAllSet() throws Exception {
fillSingleValueAllSet();
IndexFieldData indexFieldData = getForField("value");
AtomicReaderContext readerContext = refreshReader();
AtomicFieldData fieldData = indexFieldData.load(readerContext);
assertThat(fieldData.getNumDocs(), equalTo(3));
BytesValues bytesValues = fieldData.getBytesValues();
assertThat(bytesValues.isMultiValued(), equalTo(false));
assertThat(bytesValues.hasValue(0), equalTo(true));
assertThat(bytesValues.hasValue(1), equalTo(true));
assertThat(bytesValues.hasValue(2), equalTo(true));
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
BytesRef bytesRef = new BytesRef();
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
assertThat(bytesRef, equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
assertThat(bytesRef, equalTo(new BytesRef(one())));
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIterSafe(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
assertThat(stringValues.hasValue(1), equalTo(true));
assertThat(stringValues.hasValue(2), equalTo(true));
assertThat(stringValues.getValue(0), equalTo(two()));
assertThat(stringValues.getValue(1), equalTo(one()));
assertThat(stringValues.getValue(2), equalTo(three()));
StringArrayRef stringArrayRef;
stringArrayRef = stringValues.getValues(0);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
stringArrayRef = stringValues.getValues(1);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));
stringArrayRef = stringValues.getValues(2);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
StringValues.Iter stringValuesIter = stringValues.getIter(0);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(two()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(1);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(one()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(2);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(three()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
IndexSearcher searcher = new IndexSearcher(readerContext.reader());
TopFieldDocs topDocs;
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null))));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(1));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(2));
topDocs = searcher.search(new MatchAllDocsQuery(), 10,
new Sort(new SortField("value", indexFieldData.comparatorSource(null), true)));
assertThat(topDocs.totalHits, equalTo(3));
assertThat(topDocs.scoreDocs[0].doc, equalTo(2));
assertThat(topDocs.scoreDocs[1].doc, equalTo(0));
assertThat(topDocs.scoreDocs[2].doc, equalTo(1));
}
protected void fillSingleValueWithMissing() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
//d.add(new StringField("value", one(), Field.Store.NO)); // MISSING....
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
writer.addDocument(d);
}
@Test
public void testSingleValueWithMissing() throws Exception {
fillSingleValueWithMissing();
IndexFieldData indexFieldData = getForField("value");
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData.getNumDocs(), equalTo(3));
BytesValues bytesValues = fieldData
.getBytesValues();
assertThat(bytesValues.isMultiValued(), equalTo(false));
assertThat(bytesValues.hasValue(0), equalTo(true));
assertThat(bytesValues.hasValue(1), equalTo(false));
assertThat(bytesValues.hasValue(2), equalTo(true));
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValue(1), nullValue());
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueSafe(1), nullValue());
assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
BytesRef bytesRef = new BytesRef();
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
assertThat(bytesRef, equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
assertThat(bytesRef, equalTo(new BytesRef()));
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(0));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIter(1);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIterSafe(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIterSafe(1);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()));
bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), nullValue());
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValueSafe(1), nullValue());
assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIter(1);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()));
hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
assertThat(stringValues.hasValue(1), equalTo(false));
assertThat(stringValues.hasValue(2), equalTo(true));
assertThat(stringValues.getValue(0), equalTo(two()));
assertThat(stringValues.getValue(1), nullValue());
assertThat(stringValues.getValue(2), equalTo(three()));
StringArrayRef stringArrayRef;
stringArrayRef = stringValues.getValues(0);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
stringArrayRef = stringValues.getValues(1);
assertThat(stringArrayRef.size(), equalTo(0));
stringArrayRef = stringValues.getValues(2);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
StringValues.Iter stringValuesIter = stringValues.getIter(0);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(two()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(1);
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(2);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(three()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()));
stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
// TODO properly support missing....
}
protected void fillMultiValueAllSet() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
d.add(new StringField("value", "4", Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
d.add(new StringField("value", "1", Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
writer.addDocument(d);
}
@Test
public void testMultiValueAllSet() throws Exception {
fillMultiValueAllSet();
IndexFieldData indexFieldData = getForField("value");
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData.getNumDocs(), equalTo(3));
BytesValues bytesValues = fieldData.getBytesValues();
assertThat(bytesValues.isMultiValued(), equalTo(true));
assertThat(bytesValues.hasValue(0), equalTo(true));
assertThat(bytesValues.hasValue(1), equalTo(true));
assertThat(bytesValues.hasValue(2), equalTo(true));
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValue(1), equalTo(new BytesRef(one())));
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueSafe(1), equalTo(new BytesRef(one())));
assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
BytesRef bytesRef = new BytesRef();
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
assertThat(bytesRef, equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef(one())));
assertThat(bytesRef, equalTo(new BytesRef(one())));
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(2));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(one())));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIterSafe(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addExpected(one()));
bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(true));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), equalTo(new HashedBytesRef(one())));
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValueSafe(1), equalTo(new HashedBytesRef(one())));
assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addExpected(one()));
hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
assertThat(stringValues.hasValue(1), equalTo(true));
assertThat(stringValues.hasValue(2), equalTo(true));
assertThat(stringValues.getValue(0), equalTo(two()));
assertThat(stringValues.getValue(1), equalTo(one()));
assertThat(stringValues.getValue(2), equalTo(three()));
StringArrayRef stringArrayRef;
stringArrayRef = stringValues.getValues(0);
assertThat(stringArrayRef.size(), equalTo(2));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));
stringArrayRef = stringValues.getValues(1);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(one()));
stringArrayRef = stringValues.getValues(2);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
StringValues.Iter stringValuesIter = stringValues.getIter(0);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(two()));
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(four()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(1);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(one()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(2);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(three()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addExpected(one()));
stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
}
protected void fillMultiValueWithMissing() throws Exception {
Document d = new Document();
d.add(new StringField("_id", "1", Field.Store.NO));
d.add(new StringField("value", "2", Field.Store.NO));
d.add(new StringField("value", "4", Field.Store.NO));
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "2", Field.Store.NO));
//d.add(new StringField("value", one(), Field.Store.NO)); // MISSING
writer.addDocument(d);
d = new Document();
d.add(new StringField("_id", "3", Field.Store.NO));
d.add(new StringField("value", "3", Field.Store.NO));
writer.addDocument(d);
}
@Test
public void testMultiValueWithMissing() throws Exception {
fillMultiValueWithMissing();
IndexFieldData indexFieldData = getForField("value");
AtomicFieldData fieldData = indexFieldData.load(refreshReader());
assertThat(fieldData.getNumDocs(), equalTo(3));
BytesValues bytesValues = fieldData.getBytesValues();
assertThat(bytesValues.isMultiValued(), equalTo(true));
assertThat(bytesValues.hasValue(0), equalTo(true));
assertThat(bytesValues.hasValue(1), equalTo(false));
assertThat(bytesValues.hasValue(2), equalTo(true));
assertThat(bytesValues.getValue(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValue(1), nullValue());
assertThat(bytesValues.getValue(2), equalTo(new BytesRef(three())));
assertThat(bytesValues.getValueSafe(0), equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueSafe(1), nullValue());
assertThat(bytesValues.getValueSafe(2), equalTo(new BytesRef(three())));
BytesRef bytesRef = new BytesRef();
assertThat(bytesValues.getValueScratch(0, bytesRef), equalTo(new BytesRef(two())));
assertThat(bytesRef, equalTo(new BytesRef(two())));
assertThat(bytesValues.getValueScratch(1, bytesRef), equalTo(new BytesRef()));
assertThat(bytesRef, equalTo(new BytesRef()));
assertThat(bytesValues.getValueScratch(2, bytesRef), equalTo(new BytesRef(three())));
assertThat(bytesRef, equalTo(new BytesRef(three())));
BytesRefArrayRef bytesRefArrayRef = bytesValues.getValues(0);
assertThat(bytesRefArrayRef.size(), equalTo(2));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(two())));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start + 1], equalTo(new BytesRef(four())));
bytesRefArrayRef = bytesValues.getValues(1);
assertThat(bytesRefArrayRef.size(), equalTo(0));
bytesRefArrayRef = bytesValues.getValues(2);
assertThat(bytesRefArrayRef.size(), equalTo(1));
assertThat(bytesRefArrayRef.values[bytesRefArrayRef.start], equalTo(new BytesRef(three())));
BytesValues.Iter bytesValuesIter = bytesValues.getIter(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIter(1);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIterSafe(0);
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(two())));
assertThat(bytesValuesIter.hasNext(), equalTo(true));
assertThat(bytesValuesIter.next(), equalTo(new BytesRef(four())));
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValuesIter = bytesValues.getIterSafe(1);
assertThat(bytesValuesIter.hasNext(), equalTo(false));
bytesValues.forEachValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
bytesValues.forEachValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
bytesValues.forEachSafeValueInDoc(0, new BytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
bytesValues.forEachSafeValueInDoc(1, new BytesValuesVerifierProc(1).addMissing());
bytesValues.forEachSafeValueInDoc(2, new BytesValuesVerifierProc(2).addExpected(three()));
HashedBytesValues hashedBytesValues = fieldData.getHashedBytesValues();
assertThat(hashedBytesValues.hasValue(0), equalTo(true));
assertThat(hashedBytesValues.hasValue(1), equalTo(false));
assertThat(hashedBytesValues.hasValue(2), equalTo(true));
assertThat(hashedBytesValues.getValue(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValue(1), nullValue());
assertThat(hashedBytesValues.getValue(2), equalTo(new HashedBytesRef(three())));
assertThat(hashedBytesValues.getValueSafe(0), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValues.getValueSafe(1), nullValue());
assertThat(hashedBytesValues.getValueSafe(2), equalTo(new HashedBytesRef(three())));
HashedBytesValues.Iter hashedBytesValuesIter = hashedBytesValues.getIter(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIter(1);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIterSafe(0);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(two())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(true));
assertThat(hashedBytesValuesIter.next(), equalTo(new HashedBytesRef(four())));
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValuesIter = hashedBytesValues.getIterSafe(1);
assertThat(hashedBytesValuesIter.hasNext(), equalTo(false));
hashedBytesValues.forEachValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
hashedBytesValues.forEachValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
hashedBytesValues.forEachSafeValueInDoc(0, new HashedBytesValuesVerifierProc(0).addExpected(two()).addExpected(four()));
hashedBytesValues.forEachSafeValueInDoc(1, new HashedBytesValuesVerifierProc(1).addMissing());
hashedBytesValues.forEachSafeValueInDoc(2, new HashedBytesValuesVerifierProc(2).addExpected(three()));
StringValues stringValues = fieldData.getStringValues();
assertThat(stringValues.hasValue(0), equalTo(true));
assertThat(stringValues.hasValue(1), equalTo(false));
assertThat(stringValues.hasValue(2), equalTo(true));
assertThat(stringValues.getValue(0), equalTo(two()));
assertThat(stringValues.getValue(1), nullValue());
assertThat(stringValues.getValue(2), equalTo(three()));
StringArrayRef stringArrayRef;
stringArrayRef = stringValues.getValues(0);
assertThat(stringArrayRef.size(), equalTo(2));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(two()));
assertThat(stringArrayRef.values[stringArrayRef.start + 1], equalTo(four()));
stringArrayRef = stringValues.getValues(1);
assertThat(stringArrayRef.size(), equalTo(0));
stringArrayRef = stringValues.getValues(2);
assertThat(stringArrayRef.size(), equalTo(1));
assertThat(stringArrayRef.values[stringArrayRef.start], equalTo(three()));
StringValues.Iter stringValuesIter = stringValues.getIter(0);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(two()));
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(four()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(1);
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValuesIter = stringValues.getIter(2);
assertThat(stringValuesIter.hasNext(), equalTo(true));
assertThat(stringValuesIter.next(), equalTo(three()));
assertThat(stringValuesIter.hasNext(), equalTo(false));
stringValues.forEachValueInDoc(0, new StringValuesVerifierProc(0).addExpected(two()).addExpected(four()));
stringValues.forEachValueInDoc(1, new StringValuesVerifierProc(1).addMissing());
stringValues.forEachValueInDoc(2, new StringValuesVerifierProc(2).addExpected(three()));
}
}