Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Replace Map<String,Object> with IntObjectHashMap for DV producer (#13686) (#13810) 

Reverts "Replace Map<String,Object> with IntObjectHashMap for DV producer (#13686)

relates #13809
This commit is contained in:
Chris Hegarty 2024-09-20 11:06:43 +01:00 committed by GitHub
parent e4ac57746e
commit 7ef7122eba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 79 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -420,8 +420,6 @@ Optimizations
* GITHUB#13742: Reorder checks in LRUQueryCache#count (Shubham Chaudhary)
* GITHUB#13686: Replace Map<String,Object> with IntObjectHashMap for DV producer (Pan Guixin)
* GITHUB#13697: Add a bulk scorer to ToParentBlockJoinQuery, which delegates to the bulk scorer of the child query.
This should speed up query evaluation when the child query has a specialized bulk scorer, such as disjunctive queries.
(Mike Pellegrini)

33
lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.java
View File

@ -17,6 +17,8 @@
package org.apache.lucene.backward_codecs.lucene80;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.backward_codecs.packed.LegacyDirectMonotonicReader;
import org.apache.lucene.backward_codecs.packed.LegacyDirectReader;
import org.apache.lucene.backward_codecs.store.EndiannessReverserUtil;
@ -39,7 +41,6 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
@ -52,11 +53,11 @@ import org.apache.lucene.util.compress.LZ4;
/** reader for {@link Lucene80DocValuesFormat} */
final class Lucene80DocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<NumericEntry> numerics = new IntObjectHashMap<>();
private final IntObjectHashMap<BinaryEntry> binaries = new IntObjectHashMap<>();
private final IntObjectHashMap<SortedEntry> sorted = new IntObjectHashMap<>();
private final IntObjectHashMap<SortedSetEntry> sortedSets = new IntObjectHashMap<>();
private final IntObjectHashMap<SortedNumericEntry> sortedNumerics = new IntObjectHashMap<>();
private final Map<String, NumericEntry> numerics = new HashMap<>();
private final Map<String, BinaryEntry> binaries = new HashMap<>();
private final Map<String, SortedEntry> sorted = new HashMap<>();
private final Map<String, SortedSetEntry> sortedSets = new HashMap<>();
private final Map<String, SortedNumericEntry> sortedNumerics = new HashMap<>();
private final IndexInput data;
private final int maxDoc;
private int version = -1;
@ -138,7 +139,7 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
}
byte type = meta.readByte();
if (type == Lucene80DocValuesFormat.NUMERIC) {
numerics.put(info.number, readNumeric(meta));
numerics.put(info.name, readNumeric(meta));
} else if (type == Lucene80DocValuesFormat.BINARY) {
final boolean compressed;
if (version >= Lucene80DocValuesFormat.VERSION_CONFIGURABLE_COMPRESSION) {
@ -157,13 +158,13 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
} else {
compressed = version >= Lucene80DocValuesFormat.VERSION_BIN_COMPRESSED;
}
binaries.put(info.number, readBinary(meta, compressed));
binaries.put(info.name, readBinary(meta, compressed));
} else if (type == Lucene80DocValuesFormat.SORTED) {
sorted.put(info.number, readSorted(meta));
sorted.put(info.name, readSorted(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_SET) {
sortedSets.put(info.number, readSortedSet(meta));
sortedSets.put(info.name, readSortedSet(meta));
} else if (type == Lucene80DocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.number, readSortedNumeric(meta));
sortedNumerics.put(info.name, readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
@ -425,7 +426,7 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.number);
NumericEntry entry = numerics.get(field.name);
return getNumeric(entry);
}
@ -914,7 +915,7 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.number);
BinaryEntry entry = binaries.get(field.name);
if (entry.compressed) {
return getCompressedBinary(entry);
} else {
@ -972,7 +973,7 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedEntry entry = sorted.get(field.number);
SortedEntry entry = sorted.get(field.name);
return getSorted(entry);
}
@ -1406,7 +1407,7 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
SortedNumericEntry entry = sortedNumerics.get(field.number);
SortedNumericEntry entry = sortedNumerics.get(field.name);
if (entry.numValues == entry.numDocsWithField) {
return DocValues.singleton(getNumeric(entry));
}
@ -1542,7 +1543,7 @@ final class Lucene80DocValuesProducer extends DocValuesProducer {
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetEntry entry = sortedSets.get(field.number);
SortedSetEntry entry = sortedSets.get(field.name);
if (entry.singleValueEntry != null) {
return DocValues.singleton(getSorted(entry.singleValueEntry));
}

63
lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java
View File

@ -21,6 +21,8 @@ import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_IND
import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.index.BaseTermsEnum;
@ -41,7 +43,6 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
@ -58,12 +59,12 @@ import org.apache.lucene.util.packed.DirectReader;
/** reader for {@link Lucene90DocValuesFormat} */
final class Lucene90DocValuesProducer extends DocValuesProducer {
private final IntObjectHashMap<NumericEntry> numerics;
private final IntObjectHashMap<BinaryEntry> binaries;
private final IntObjectHashMap<SortedEntry> sorted;
private final IntObjectHashMap<SortedSetEntry> sortedSets;
private final IntObjectHashMap<SortedNumericEntry> sortedNumerics;
private final IntObjectHashMap<DocValuesSkipperEntry> skippers;
private final Map<String, NumericEntry> numerics;
private final Map<String, BinaryEntry> binaries;
private final Map<String, SortedEntry> sorted;
private final Map<String, SortedSetEntry> sortedSets;
private final Map<String, SortedNumericEntry> sortedNumerics;
private final Map<String, DocValuesSkipperEntry> skippers;
private final IndexInput data;
private final int maxDoc;
private int version = -1;
@ -80,12 +81,12 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
String metaName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
this.maxDoc = state.segmentInfo.maxDoc();
numerics = new IntObjectHashMap<>();
binaries = new IntObjectHashMap<>();
sorted = new IntObjectHashMap<>();
sortedSets = new IntObjectHashMap<>();
sortedNumerics = new IntObjectHashMap<>();
skippers = new IntObjectHashMap<>();
numerics = new HashMap<>();
binaries = new HashMap<>();
sorted = new HashMap<>();
sortedSets = new HashMap<>();
sortedNumerics = new HashMap<>();
skippers = new HashMap<>();
merging = false;
// read in the entries from the metadata file.
@ -148,12 +149,12 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
// Used for cloning
private Lucene90DocValuesProducer(
IntObjectHashMap<NumericEntry> numerics,
IntObjectHashMap<BinaryEntry> binaries,
IntObjectHashMap<SortedEntry> sorted,
IntObjectHashMap<SortedSetEntry> sortedSets,
IntObjectHashMap<SortedNumericEntry> sortedNumerics,
IntObjectHashMap<DocValuesSkipperEntry> skippers,
Map<String, NumericEntry> numerics,
Map<String, BinaryEntry> binaries,
Map<String, SortedEntry> sorted,
Map<String, SortedSetEntry> sortedSets,
Map<String, SortedNumericEntry> sortedNumerics,
Map<String, DocValuesSkipperEntry> skippers,
IndexInput data,
int maxDoc,
int version,
@ -193,18 +194,18 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
}
byte type = meta.readByte();
if (info.docValuesSkipIndexType() != DocValuesSkipIndexType.NONE) {
skippers.put(info.number, readDocValueSkipperMeta(meta));
skippers.put(info.name, readDocValueSkipperMeta(meta));
}
if (type == Lucene90DocValuesFormat.NUMERIC) {
numerics.put(info.number, readNumeric(meta));
numerics.put(info.name, readNumeric(meta));
} else if (type == Lucene90DocValuesFormat.BINARY) {
binaries.put(info.number, readBinary(meta));
binaries.put(info.name, readBinary(meta));
} else if (type == Lucene90DocValuesFormat.SORTED) {
sorted.put(info.number, readSorted(meta));
sorted.put(info.name, readSorted(meta));
} else if (type == Lucene90DocValuesFormat.SORTED_SET) {
sortedSets.put(info.number, readSortedSet(meta));
sortedSets.put(info.name, readSortedSet(meta));
} else if (type == Lucene90DocValuesFormat.SORTED_NUMERIC) {
sortedNumerics.put(info.number, readSortedNumeric(meta));
sortedNumerics.put(info.name, readSortedNumeric(meta));
} else {
throw new CorruptIndexException("invalid type: " + type, meta);
}
@ -429,7 +430,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.number);
NumericEntry entry = numerics.get(field.name);
return getNumeric(entry);
}
@ -785,7 +786,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryEntry entry = binaries.get(field.number);
BinaryEntry entry = binaries.get(field.name);
if (entry.docsWithFieldOffset == -2) {
return DocValues.emptyBinary();
@ -890,7 +891,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
SortedEntry entry = sorted.get(field.number);
SortedEntry entry = sorted.get(field.name);
return getSorted(entry);
}
@ -1367,7 +1368,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
SortedNumericEntry entry = sortedNumerics.get(field.number);
SortedNumericEntry entry = sortedNumerics.get(field.name);
return getSortedNumeric(entry);
}
@ -1512,7 +1513,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
SortedSetEntry entry = sortedSets.get(field.number);
SortedSetEntry entry = sortedSets.get(field.name);
if (entry.singleValueEntry != null) {
return DocValues.singleton(getSorted(entry.singleValueEntry));
}
@ -1786,7 +1787,7 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {
@Override
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
final DocValuesSkipperEntry entry = skippers.get(field.number);
final DocValuesSkipperEntry entry = skippers.get(field.name);
final IndexInput input = data.slice("doc value skipper", entry.offset, entry.length);
// Prefetch the first page of data. Following pages are expected to get prefetched through

25
lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
View File

@ -38,7 +38,6 @@ import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.util.IOUtils;
/**
@ -255,9 +254,9 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
}
private static class FieldsReader extends DocValuesProducer {
private class FieldsReader extends DocValuesProducer {
private final IntObjectHashMap<DocValuesProducer> fields = new IntObjectHashMap<>();
private final Map<String, DocValuesProducer> fields = new HashMap<>();
private final Map<String, DocValuesProducer> formats = new HashMap<>();
// clone for merge
@ -271,10 +270,10 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
}
// Then rebuild fields:
for (IntObjectHashMap.IntObjectCursor<DocValuesProducer> ent : other.fields) {
DocValuesProducer producer = oldToNew.get(ent.value);
for (Map.Entry<String, DocValuesProducer> ent : other.fields.entrySet()) {
DocValuesProducer producer = oldToNew.get(ent.getValue());
assert producer != null;
fields.put(ent.key, producer);
fields.put(ent.getKey(), producer);
}
}
@ -303,7 +302,7 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
segmentSuffix,
format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
}
fields.put(fi.number, formats.get(segmentSuffix));
fields.put(fieldName, formats.get(segmentSuffix));
}
}
}
@ -317,37 +316,37 @@ public abstract class PerFieldDocValuesFormat extends DocValuesFormat {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.number);
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getNumeric(field);
}
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.number);
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getBinary(field);
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.number);
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getSorted(field);
}
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.number);
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getSortedNumeric(field);
}
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.number);
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getSortedSet(field);
}
@Override
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
DocValuesProducer producer = fields.get(field.number);
DocValuesProducer producer = fields.get(field.name);
return producer == null ? null : producer.getSkipper(field);
}

21
lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java
View File

@ -18,10 +18,11 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.internal.hppc.LongArrayList;
import org.apache.lucene.store.Directory;
@ -31,7 +32,7 @@ import org.apache.lucene.store.Directory;
// producer?
class SegmentDocValuesProducer extends DocValuesProducer {
final IntObjectHashMap<DocValuesProducer> dvProducersByField = new IntObjectHashMap<>();
final Map<String, DocValuesProducer> dvProducersByField = new HashMap<>();
final Set<DocValuesProducer> dvProducers =
Collections.newSetFromMap(new IdentityHashMap<DocValuesProducer, Boolean>());
final LongArrayList dvGens = new LongArrayList();
@ -66,7 +67,7 @@ class SegmentDocValuesProducer extends DocValuesProducer {
dvGens.add(docValuesGen);
dvProducers.add(baseProducer);
}
dvProducersByField.put(fi.number, baseProducer);
dvProducersByField.put(fi.name, baseProducer);
} else {
assert !dvGens.contains(docValuesGen);
// otherwise, producer sees only the one fieldinfo it wrote
@ -75,7 +76,7 @@ class SegmentDocValuesProducer extends DocValuesProducer {
docValuesGen, si, dir, new FieldInfos(new FieldInfo[] {fi}));
dvGens.add(docValuesGen);
dvProducers.add(dvp);
dvProducersByField.put(fi.number, dvp);
dvProducersByField.put(fi.name, dvp);
}
}
} catch (Throwable t) {
@ -90,42 +91,42 @@ class SegmentDocValuesProducer extends DocValuesProducer {
@Override
public NumericDocValues getNumeric(FieldInfo field) throws IOException {
DocValuesProducer dvProducer = dvProducersByField.get(field.number);
DocValuesProducer dvProducer = dvProducersByField.get(field.name);
assert dvProducer != null;
return dvProducer.getNumeric(field);
}
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
DocValuesProducer dvProducer = dvProducersByField.get(field.number);
DocValuesProducer dvProducer = dvProducersByField.get(field.name);
assert dvProducer != null;
return dvProducer.getBinary(field);
}
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
DocValuesProducer dvProducer = dvProducersByField.get(field.number);
DocValuesProducer dvProducer = dvProducersByField.get(field.name);
assert dvProducer != null;
return dvProducer.getSorted(field);
}
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
DocValuesProducer dvProducer = dvProducersByField.get(field.number);
DocValuesProducer dvProducer = dvProducersByField.get(field.name);
assert dvProducer != null;
return dvProducer.getSortedNumeric(field);
}
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
DocValuesProducer dvProducer = dvProducersByField.get(field.number);
DocValuesProducer dvProducer = dvProducersByField.get(field.name);
assert dvProducer != null;
return dvProducer.getSortedSet(field);
}
@Override
public DocValuesSkipper getSkipper(FieldInfo field) throws IOException {
DocValuesProducer dvProducer = dvProducersByField.get(field.number);
DocValuesProducer dvProducer = dvProducersByField.get(field.name);
assert dvProducer != null;
return dvProducer.getSkipper(field);
}

13
lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
View File

@ -20,8 +20,10 @@ import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.FieldsProducer;
@ -32,7 +34,6 @@ import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.KnnCollector;
import org.apache.lucene.search.VectorScorer;
@ -389,7 +390,7 @@ final class SlowCompositeCodecReaderWrapper extends CodecReader {
private final CodecReader[] codecReaders;
private final DocValuesProducer[] producers;
private final int[] docStarts;
private final IntObjectHashMap<OrdinalMap> cachedOrdMaps = new IntObjectHashMap<>();
private final Map<String, OrdinalMap> cachedOrdMaps = new HashMap<>();
SlowCompositeDocValuesProducerWrapper(CodecReader[] codecReaders, int[] docStarts) {
this.codecReaders = codecReaders;
@ -428,14 +429,14 @@ final class SlowCompositeCodecReaderWrapper extends CodecReader {
public SortedDocValues getSorted(FieldInfo field) throws IOException {
OrdinalMap map = null;
synchronized (cachedOrdMaps) {
map = cachedOrdMaps.get(field.number);
map = cachedOrdMaps.get(field.name);
if (map == null) {
// uncached, or not a multi dv
SortedDocValues dv =
MultiDocValues.getSortedValues(new MultiReader(codecReaders), field.name);
if (dv instanceof MultiSortedDocValues) {
map = ((MultiSortedDocValues) dv).mapping;
cachedOrdMaps.put(field.number, map);
cachedOrdMaps.put(field.name, map);
}
return dv;
}
@ -464,14 +465,14 @@ final class SlowCompositeCodecReaderWrapper extends CodecReader {
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
OrdinalMap map = null;
synchronized (cachedOrdMaps) {
map = cachedOrdMaps.get(field.number);
map = cachedOrdMaps.get(field.name);
if (map == null) {
// uncached, or not a multi dv
SortedSetDocValues dv =
MultiDocValues.getSortedSetValues(new MultiReader(codecReaders), field.name);
if (dv instanceof MultiSortedSetDocValues) {
map = ((MultiSortedSetDocValues) dv).mapping;
cachedOrdMaps.put(field.number, map);
cachedOrdMaps.put(field.name, map);
}
return dv;
}