remove concrete bytes for field data
no really need for it, specifically with the fact that we don't need to deepCopy on makeSafe for the (default) paged bytes
This commit is contained in:
Shay Banon
parent
455b5da52f
commit
da5dff9ee4
|
|
@ -58,7 +58,6 @@ public class IndexFieldDataService extends AbstractIndexComponent {
|
|||
.immutableMap();
|
||||
|
||||
buildersByTypeAndFormat = MapBuilder.<Tuple<String, String>, IndexFieldData.Builder>newMapBuilder()
|
||||
.put(Tuple.tuple("string", "concrete_bytes"), new ConcreteBytesRefIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", "paged_bytes"), new PagedBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("string", "fst"), new FSTBytesIndexFieldData.Builder())
|
||||
.put(Tuple.tuple("float", "array"), new FloatArrayIndexFieldData.Builder())
|
||||
|
|
|
|||
|
|
@ -1,296 +0,0 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.plain;
|
||||
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.elasticsearch.common.RamUsage;
|
||||
import org.elasticsearch.index.fielddata.AtomicFieldData;
|
||||
import org.elasticsearch.index.fielddata.ScriptDocValues;
|
||||
import org.elasticsearch.index.fielddata.ordinals.EmptyOrdinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals;
|
||||
import org.elasticsearch.index.fielddata.ordinals.Ordinals.Docs;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class ConcreteBytesRefAtomicFieldData implements AtomicFieldData.WithOrdinals<ScriptDocValues.Strings> {
|
||||
|
||||
public static ConcreteBytesRefAtomicFieldData empty(int numDocs) {
|
||||
return new Empty(numDocs);
|
||||
}
|
||||
|
||||
// 0 ordinal in values means no value (its null)
|
||||
private final BytesRef[] values;
|
||||
protected final Ordinals ordinals;
|
||||
|
||||
private volatile int[] hashes;
|
||||
private long size = -1;
|
||||
|
||||
public ConcreteBytesRefAtomicFieldData(BytesRef[] values, Ordinals ordinals) {
|
||||
this.values = values;
|
||||
this.ordinals = ordinals;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return ordinals.isMultiValued();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumDocs() {
|
||||
return ordinals.getNumDocs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isValuesOrdered() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
if (size == -1) {
|
||||
long size = RamUsage.NUM_BYTES_ARRAY_HEADER;
|
||||
for (BytesRef value : values) {
|
||||
if (value != null) {
|
||||
size += RamUsage.NUM_BYTES_OBJECT_REF + RamUsage.NUM_BYTES_OBJECT_HEADER +
|
||||
RamUsage.NUM_BYTES_ARRAY_HEADER + (value.length + (2 * RamUsage.NUM_BYTES_INT));
|
||||
}
|
||||
}
|
||||
size += ordinals.getMemorySizeInBytes();
|
||||
this.size = size;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return ordinals.isMultiValued() ? new BytesValues.Multi(values, ordinals.ordinals()) : new BytesValues.Single(values, ordinals.ordinals());
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues getHashedBytesValues() {
|
||||
if (hashes == null) {
|
||||
int[] hashes = new int[values.length];
|
||||
for (int i = 0; i < values.length; i++) {
|
||||
BytesRef value = values[i];
|
||||
hashes[i] = value == null ? 0 : value.hashCode();
|
||||
}
|
||||
this.hashes = hashes;
|
||||
}
|
||||
return ordinals.isMultiValued() ? new BytesValues.MultiHashed(values, ordinals.ordinals(), hashes) : new BytesValues.SingleHashed(values, ordinals.ordinals(), hashes);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public ScriptDocValues.Strings getScriptValues() {
|
||||
return new ScriptDocValues.Strings(getBytesValues());
|
||||
}
|
||||
|
||||
static abstract class BytesValues extends org.elasticsearch.index.fielddata.BytesValues.WithOrdinals {
|
||||
|
||||
protected final BytesRef[] values;
|
||||
|
||||
BytesValues(BytesRef[] values, Ordinals.Docs ordinals) {
|
||||
super(ordinals);
|
||||
this.values = values;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueByOrd(int ord) {
|
||||
return values[ord];
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getValueScratchByOrd(int ord, BytesRef ret) {
|
||||
BytesRef value = values[ord];
|
||||
if (value == null) {
|
||||
ret.length = 0;
|
||||
} else {
|
||||
ret.bytes = value.bytes;
|
||||
ret.offset = value.offset;
|
||||
ret.length = value.length;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef getSafeValueByOrd(int ord) {
|
||||
return values[ord];
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesRef makeSafe(BytesRef bytes) {
|
||||
// no need to do anything, its already concrete bytes...
|
||||
return bytes;
|
||||
}
|
||||
|
||||
static class Single extends BytesValues {
|
||||
|
||||
private final Iter.Single iter;
|
||||
|
||||
Single(BytesRef[] values, Ordinals.Docs ordinals) {
|
||||
super(values, ordinals);
|
||||
this.iter = newSingleIter();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iter getIter(int docId) {
|
||||
int ord = ordinals.getOrd(docId);
|
||||
if (ord == 0) return Iter.Empty.INSTANCE;
|
||||
return iter.reset(values[ord], ord);
|
||||
}
|
||||
}
|
||||
|
||||
static final class SingleHashed extends Single {
|
||||
private final int[] hashes;
|
||||
SingleHashed(BytesRef[] values, Docs ordinals, int[] hashes) {
|
||||
super(values, ordinals);
|
||||
this.hashes = hashes;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Iter.Single newSingleIter() {
|
||||
return new Iter.Single() {
|
||||
public int hash() {
|
||||
return hashes[ord];
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValueHashed(int docId, BytesRef ret) {
|
||||
final int ord = ordinals.getOrd(docId);
|
||||
getValueScratchByOrd(ord, ret);
|
||||
return hashes[ord];
|
||||
}
|
||||
}
|
||||
|
||||
static class Multi extends BytesValues {
|
||||
|
||||
private final Iter.Multi iter;
|
||||
|
||||
Multi(BytesRef[] values, Ordinals.Docs ordinals) {
|
||||
super(values, ordinals);
|
||||
assert ordinals.isMultiValued();
|
||||
this.iter = newMultiIter();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Iter.Multi newMultiIter() {
|
||||
return new Iter.Multi(this) {
|
||||
private BytesRef current = null;
|
||||
@Override
|
||||
public BytesRef next() {
|
||||
current = withOrds.getValueByOrd(innerOrd);
|
||||
ord = innerOrd;
|
||||
innerOrd = ordsIter.next();
|
||||
return current;
|
||||
}
|
||||
public int hash() {
|
||||
assert current != null;
|
||||
return current.hashCode();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public Iter getIter(int docId) {
|
||||
return iter.reset(ordinals.getIter(docId));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static final class MultiHashed extends Multi {
|
||||
private final int[] hashes;
|
||||
|
||||
MultiHashed(BytesRef[] values, Ordinals.Docs ordinals, int[] hashes) {
|
||||
super(values, ordinals);
|
||||
this.hashes = hashes;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Iter.Multi newMultiIter() {
|
||||
return new Iter.Multi(this) {
|
||||
private BytesRef current = null;
|
||||
@Override
|
||||
public BytesRef next() {
|
||||
current = withOrds.getValueByOrd(innerOrd);
|
||||
ord = innerOrd;
|
||||
innerOrd = ordsIter.next();
|
||||
return current;
|
||||
}
|
||||
|
||||
public int hash() {
|
||||
return hashes[ord];
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getValueHashed(int docId, BytesRef ret) {
|
||||
final int ord = ordinals.getOrd(docId);
|
||||
getValueScratchByOrd(ord, ret);
|
||||
return hashes[ord];
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static class Empty extends ConcreteBytesRefAtomicFieldData {
|
||||
|
||||
Empty(int numDocs) {
|
||||
super(null, new EmptyOrdinals(numDocs));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isMultiValued() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getNumDocs() {
|
||||
return ordinals.getNumDocs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isValuesOrdered() {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getMemorySizeInBytes() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public BytesValues.WithOrdinals getBytesValues() {
|
||||
return new BytesValues.WithOrdinals.Empty(ordinals.ordinals());
|
||||
}
|
||||
|
||||
@Override
|
||||
public ScriptDocValues.Strings getScriptValues() {
|
||||
return ScriptDocValues.EMPTY_STRINGS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.fielddata.plain;
|
||||
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.apache.lucene.index.AtomicReader;
|
||||
import org.apache.lucene.index.AtomicReaderContext;
|
||||
import org.apache.lucene.index.Terms;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefIterator;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.index.Index;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldData;
|
||||
import org.elasticsearch.index.fielddata.IndexFieldDataCache;
|
||||
import org.elasticsearch.index.fielddata.ordinals.OrdinalsBuilder;
|
||||
import org.elasticsearch.index.mapper.FieldMapper;
|
||||
import org.elasticsearch.index.settings.IndexSettings;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class ConcreteBytesRefIndexFieldData extends AbstractBytesIndexFieldData<ConcreteBytesRefAtomicFieldData> {
|
||||
|
||||
public static class Builder implements IndexFieldData.Builder {
|
||||
|
||||
@Override
|
||||
public IndexFieldData<ConcreteBytesRefAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
|
||||
return new ConcreteBytesRefIndexFieldData(index, indexSettings, fieldNames, type, cache);
|
||||
}
|
||||
}
|
||||
|
||||
public ConcreteBytesRefIndexFieldData(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
|
||||
super(index, indexSettings, fieldNames, fieldDataType, cache);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ConcreteBytesRefAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
|
||||
AtomicReader reader = context.reader();
|
||||
|
||||
Terms terms = reader.terms(getFieldNames().indexName());
|
||||
if (terms == null) {
|
||||
return ConcreteBytesRefAtomicFieldData.empty(reader.maxDoc());
|
||||
}
|
||||
|
||||
long size = terms.size();
|
||||
if (size == -1) {
|
||||
size = 1024;
|
||||
}
|
||||
final ArrayList<BytesRef> values = new ArrayList<BytesRef>((int) size);
|
||||
values.add(null); // first "t" indicates null value
|
||||
OrdinalsBuilder builder = new OrdinalsBuilder(terms, reader.maxDoc());
|
||||
try {
|
||||
BytesRefIterator iter = builder.buildFromTerms(filter(terms, reader), reader.getLiveDocs());
|
||||
BytesRef term;
|
||||
while ((term = iter.next()) != null) {
|
||||
values.add(BytesRef.deepCopyOf(term));
|
||||
}
|
||||
return new ConcreteBytesRefAtomicFieldData(values.toArray(new BytesRef[values.size()]), builder.build(fieldDataType.getSettings()));
|
||||
} finally {
|
||||
builder.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -69,13 +69,6 @@ public class ExtendedFacetsTests extends AbstractNodesTests {
|
|||
.addMapping("type1", jsonBuilder().startObject()
|
||||
.startObject("type1")
|
||||
.startObject("properties")
|
||||
.startObject("field1_concrete")
|
||||
.field("type", "string")
|
||||
.field("index", "not_analyzed")
|
||||
.startObject("fielddata")
|
||||
.field("format", "concrete_bytes")
|
||||
.endObject()
|
||||
.endObject()
|
||||
.startObject("field1_paged")
|
||||
.field("type", "string")
|
||||
.field("index", "not_analyzed")
|
||||
|
|
@ -143,7 +136,6 @@ public class ExtendedFacetsTests extends AbstractNodesTests {
|
|||
String queryVal = queryValues[random.nextInt(numOfQueryValues)];
|
||||
client.prepareIndex("test", "type1", Integer.toString(i))
|
||||
.setSource(jsonBuilder().startObject()
|
||||
.field("field1_concrete", field1Values)
|
||||
.field("field1_paged", field1Values)
|
||||
.field("field1_fst", field1Values)
|
||||
.field("field2", field2Val)
|
||||
|
|
@ -160,7 +152,7 @@ public class ExtendedFacetsTests extends AbstractNodesTests {
|
|||
}
|
||||
|
||||
client.admin().indices().prepareRefresh().execute().actionGet();
|
||||
String[] facetFields = new String[]{"field1_concrete", "field1_paged", "field1_fst"};
|
||||
String[] facetFields = new String[]{"field1_paged", "field1_fst"};
|
||||
TermsFacet.ComparatorType[] compTypes = TermsFacet.ComparatorType.values();
|
||||
for (String facetField : facetFields) {
|
||||
for (String queryVal : queryValToField1FacetEntries.keySet()) {
|
||||
|
|
|
|||
|
|
@ -1,35 +0,0 @@
|
|||
/*
|
||||
* Licensed to ElasticSearch and Shay Banon under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. ElasticSearch licenses this
|
||||
* file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.test.unit.index.fielddata;
|
||||
|
||||
import org.elasticsearch.common.settings.ImmutableSettings;
|
||||
import org.elasticsearch.index.fielddata.FieldDataType;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
/**
|
||||
*/
|
||||
@Test
|
||||
public class ConcreteBytesStringFieldDataTests extends StringFieldDataTests {
|
||||
|
||||
@Override
|
||||
protected FieldDataType getFieldDataType() {
|
||||
return new FieldDataType("string", ImmutableSettings.builder().put("format", "concrete_bytes"));
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue