Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-6006: remove unnecessary FieldInfo.normType 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1632120 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2014-10-15 16:58:30 +00:00
parent 46bdcca89e
commit 78e445ce57
38 changed files with 766 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lucene/CHANGES.txt
View File

@ -160,6 +160,10 @@ API Changes
their jflex impl directly.
(Ryan Ernst)
* LUCENE-6006: Removed FieldInfo.normType since it's redundant: it
will be DocValuesType.NUMERIC if the field indexed and does not omit
norms, else null. (Robert Muir, Mike McCandless)
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary

113
lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java Normal file
View File

@ -0,0 +1,113 @@
package org.apache.lucene.codecs;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.util.Accountable;
/**
* Used only for backwards compatibility corner case, to provide
* re-animated norms when all fields are undead.
*
* @lucene.internal */
public class UndeadNormsProducer extends NormsProducer {
/** Used to bring undead norms back to life. */
public final static String LEGACY_UNDEAD_NORMS_KEY = UndeadNormsProducer.class.getSimpleName() + ".undeadnorms";
/** Use this instance */
public final static NormsProducer INSTANCE = new UndeadNormsProducer();
private UndeadNormsProducer() {
}
/* Returns true if all indexed fields have undead norms. */
public static boolean isUndeadArmy(FieldInfos fieldInfos) {
boolean everythingIsUndead = true;
for(FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.hasNorms()) {
String isUndead = fieldInfo.getAttribute(LEGACY_UNDEAD_NORMS_KEY);
if (isUndead != null) {
assert "true".equals(isUndead);
} else {
everythingIsUndead = false;
}
}
}
return everythingIsUndead;
}
/** Returns true if this field has undead norms. */
public static boolean isUndead(FieldInfo fieldInfo) {
String isUndead = fieldInfo.getAttribute(LEGACY_UNDEAD_NORMS_KEY);
if (isUndead != null) {
// Bring undead norms back to life; this is set in Lucene40FieldInfosFormat, to emulate pre-5.0 undead norms
assert "true".equals(isUndead);
return true;
} else {
return false;
}
}
/** Call this to note that the field with these attributes has undead norms. */
public static void setUndead(Map<String,String> attributes) {
attributes.put(LEGACY_UNDEAD_NORMS_KEY, "true");
}
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
return DocValues.emptyNumeric();
}
@Override
public void close() {
}
@Override
public long ramBytesUsed() {
return 0;
}
@Override
public Iterable<? extends Accountable> getChildResources() {
return Collections.emptyList();
}
@Override
public void checkIntegrity() throws IOException {
}
@Override
public NormsProducer getMergeInstance() throws IOException {
return this;
}
@Override
public String toString() {
return getClass().getSimpleName();
}
}

13
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
View File

@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@ -91,7 +92,7 @@ public class Lucene40FieldInfosFormat extends FieldInfosFormat {
byte val = input.readByte();
final LegacyDocValuesType oldValuesType = getDocValuesType((byte) (val & 0x0F));
final LegacyDocValuesType oldNormsType = getDocValuesType((byte) ((val >>> 4) & 0x0F));
final Map<String,String> attributes = input.readStringStringMap();;
final Map<String,String> attributes = input.readStringStringMap();
if (oldValuesType.mapping != null) {
attributes.put(LEGACY_DV_TYPE_KEY, oldValuesType.name());
}
@ -101,8 +102,12 @@ public class Lucene40FieldInfosFormat extends FieldInfosFormat {
}
attributes.put(LEGACY_NORM_TYPE_KEY, oldNormsType.name());
}
if (isIndexed && omitNorms == false && oldNormsType.mapping == null) {
// Undead norms! Lucene40NormsReader will check this and bring norms back from the dead:
UndeadNormsProducer.setUndead(attributes);
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, oldValuesType.mapping, oldNormsType.mapping, -1, Collections.unmodifiableMap(attributes));
omitNorms, storePayloads, indexOptions, oldValuesType.mapping, -1, Collections.unmodifiableMap(attributes));
}
CodecUtil.checkEOF(input);

7
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@ -46,6 +47,10 @@ public class Lucene40NormsFormat extends NormsFormat {
String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
"nrm",
Lucene40CompoundFormat.COMPOUND_FILE_EXTENSION);
return new Lucene40NormsReader(state, filename);
if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
return UndeadNormsProducer.INSTANCE;
} else {
return new Lucene40NormsReader(state, filename);
}
}
}

6
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
View File

@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@ -45,6 +47,10 @@ final class Lucene40NormsReader extends NormsProducer {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
if (UndeadNormsProducer.isUndead(field)) {
// Bring undead norms back to life; this is set in Lucene40FieldInfosFormat, to emulate pre-5.0 undead norms
return DocValues.emptyNumeric();
}
return impl.getNumeric(field);
}

15
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
View File

@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@ -41,7 +42,7 @@ import org.apache.lucene.util.IOUtils;
*/
@Deprecated
public class Lucene42FieldInfosFormat extends FieldInfosFormat {
/** Sole constructor. */
public Lucene42FieldInfosFormat() {
}
@ -86,8 +87,14 @@ public class Lucene42FieldInfosFormat extends FieldInfosFormat {
final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final Map<String,String> attributes = input.readStringStringMap();
if (isIndexed && omitNorms == false && normsType == null) {
// Undead norms! Lucene42NormsProducer will check this and bring norms back from the dead:
UndeadNormsProducer.setUndead(attributes);
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, normsType, -1, Collections.unmodifiableMap(attributes));
omitNorms, storePayloads, indexOptions, docValuesType, -1, Collections.unmodifiableMap(attributes));
}
CodecUtil.checkEOF(input);

7
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.packed.PackedInts;
@ -62,7 +63,11 @@ public class Lucene42NormsFormat extends NormsFormat {
@Override
public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
return UndeadNormsProducer.INSTANCE;
} else {
return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
}
}
static final String DATA_CODEC = "Lucene41NormsData";

6
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
View File

@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@ -45,6 +47,10 @@ final class Lucene42NormsProducer extends NormsProducer {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
if (UndeadNormsProducer.isUndead(field)) {
// Bring undead norms back to life; this is set in Lucene42FieldInfosFormat, to emulate pre-5.0 undead norms
return DocValues.emptyNumeric();
}
return impl.getNumeric(field);
}

17
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
View File

@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@ -42,7 +43,7 @@ import org.apache.lucene.store.IndexOutput;
*/
@Deprecated
public final class Lucene46FieldInfosFormat extends FieldInfosFormat {
/** Sole constructor. */
public Lucene46FieldInfosFormat() {
}
@ -88,8 +89,14 @@ public final class Lucene46FieldInfosFormat extends FieldInfosFormat {
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
if (isIndexed && omitNorms == false && normsType == null) {
// Undead norms! Lucene42NormsProducer will check this and bring norms back from the dead:
UndeadNormsProducer.setUndead(attributes);
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
}
if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
@ -148,7 +155,7 @@ public final class Lucene46FieldInfosFormat extends FieldInfosFormat {
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
final byte nrm = docValuesByte(fi.getNormType());
final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);

7
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
View File

@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@ -42,7 +43,11 @@ public class Lucene49NormsFormat extends NormsFormat {
@Override
public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
return UndeadNormsProducer.INSTANCE;
} else {
return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
}
}
static final String DATA_CODEC = "Lucene49NormsData";

8
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
View File

@ -25,7 +25,9 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
@ -40,8 +42,8 @@ import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.BlockPackedReader;
import org.apache.lucene.util.packed.PackedInts;
import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_CURRENT;
import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
/**
* Reader for 4.9 norms
@ -153,6 +155,10 @@ final class Lucene49NormsProducer extends NormsProducer {
@Override
public synchronized NumericDocValues getNorms(FieldInfo field) throws IOException {
if (UndeadNormsProducer.isUndead(field)) {
// Bring undead norms back to life; this is set in Lucene46FieldInfosFormat, to emulate pre-5.0 undead norms
return DocValues.emptyNumeric();
}
NumericDocValues instance = instances.get(field.name);
if (instance == null) {
instance = loadNorms(field);

25
lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html Normal file
View File

@ -0,0 +1,25 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
Common APIs for use by backwards compatibility codecs.
</body>
</html>

2
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
View File

@ -75,7 +75,7 @@ public final class Lucene40RWFieldInfosFormat extends Lucene40FieldInfosFormat {
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType(), fi.getAttribute(LEGACY_DV_TYPE_KEY));
final byte nrm = docValuesByte(fi.getNormType(), fi.getAttribute(LEGACY_NORM_TYPE_KEY));
final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null, fi.getAttribute(LEGACY_NORM_TYPE_KEY));
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);

2
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
View File

@ -64,7 +64,7 @@ public class TestLucene40FieldInfoFormat extends BaseFieldInfoFormatTestCase {
}
}
if (fi.getNormType() != null) {
if (fi.hasNorms()) {
fi.putAttribute(Lucene40FieldInfosFormat.LEGACY_NORM_TYPE_KEY, LegacyDocValuesType.FIXED_INTS_8.name());
}
}

107
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
View File

@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene40;
* limitations under the License.
*/
import java.io.InputStream;
import java.nio.file.Path;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.TestUtil;
/** Tests Lucene40's norms format */
public class TestLucene40NormsFormat extends BaseNormsFormatTestCase {
@ -28,4 +38,101 @@ public class TestLucene40NormsFormat extends BaseNormsFormatTestCase {
protected Codec getCodec() {
return codec;
}
/** Copy this back to /l/400/lucene/CreateUndeadNorms.java, then:
* - ant clean
* - pushd analysis/common; ant jar; popd
* - pushd core; ant jar; popd
* - javac -cp build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms.java
* - java -cp .:build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms
* - cd /tmp/undeadnorms ; zip index.40.undeadnorms.zip *
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
public class CreateUndeadNorms {
public static void main(String[] args) throws Exception {
File file = new File("/tmp/undeadnorms");
if (file.exists()) {
throw new RuntimeException("please remove /tmp/undeadnorms first");
}
Directory dir = FSDirectory.open(new File("/tmp/undeadnorms"));
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40)));
Document doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(new StringField("id", "1", Field.Store.NO));
Field content = new TextField("content", "some content", Field.Store.NO);
content.setTokenStream(new TokenStream() {
@Override
public boolean incrementToken() throws IOException {
throw new IOException("brains brains!");
}
});
doc.add(content);
try {
w.addDocument(doc);
throw new RuntimeException("didn't hit exception");
} catch (IOException ioe) {
// perfect
}
w.close();
dir.close();
}
}
*/
/**
* LUCENE-6006: Test undead norms.
* .....
* C C /
* /< /
* ___ __________/_#__=o
* /(- /(\_\________ \
* \ ) \ )_ \o \
* /|\ /|\ |' |
* | _|
* /o __\
* / ' |
* / / |
* /_/\______|
* ( _( <
* \ \ \
* \ \ |
* \____\____\
* ____\_\__\_\
* /` /` o\
* |___ |_______|
*
*/
public void testReadUndeadNorms() throws Exception {
InputStream resource = TestLucene40NormsFormat.class.getResourceAsStream("index.40.undeadnorms.zip");
assertNotNull(resource);
Path path = createTempDir("undeadnorms");
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
IndexReader r = DirectoryReader.open(dir);
NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
assertNotNull(undeadNorms);
assertEquals(2, r.maxDoc());
assertEquals(0, undeadNorms.get(0));
assertEquals(0, undeadNorms.get(1));
dir.close();
r.close();
}
}

BIN
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip Normal file
View File

Binary file not shown.

2
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
View File

@ -76,7 +76,7 @@ public final class Lucene42RWFieldInfosFormat extends Lucene42FieldInfosFormat {
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
final byte nrm = docValuesByte(fi.getNormType());
final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);

108
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
View File

@ -17,10 +17,18 @@ package org.apache.lucene.codecs.lucene42;
* limitations under the License.
*/
import java.io.InputStream;
import java.nio.file.Path;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
import org.junit.BeforeClass;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.TestUtil;
/** Tests Lucene42's norms format */
public class TestLucene42NormsFormat extends BaseNormsFormatTestCase {
@ -30,4 +38,100 @@ public class TestLucene42NormsFormat extends BaseNormsFormatTestCase {
protected Codec getCodec() {
return codec;
}
/** Copy this back to /l/421/lucene/CreateUndeadNorms.java, then:
* - ant clean
* - pushd analysis/common; ant jar; popd
* - pushd core; ant jar; popd
* - javac -cp build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms.java
* - java -cp .:build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms
* - cd /tmp/undeadnorms ; zip index.42.undeadnorms.zip *
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
public class CreateUndeadNorms {
public static void main(String[] args) throws Exception {
File file = new File("/tmp/undeadnorms");
if (file.exists()) {
throw new RuntimeException("please remove /tmp/undeadnorms first");
}
Directory dir = FSDirectory.open(file);
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_42, new WhitespaceAnalyzer(Version.LUCENE_42)));
Document doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(new StringField("id", "1", Field.Store.NO));
Field content = new TextField("content", "some content", Field.Store.NO);
content.setTokenStream(new TokenStream() {
@Override
public boolean incrementToken() throws IOException {
throw new IOException("brains brains!");
}
});
doc.add(content);
try {
w.addDocument(doc);
throw new RuntimeException("didn't hit exception");
} catch (IOException ioe) {
// perfect
}
w.close();
dir.close();
}
}
*/
/**
* LUCENE-6006: Test undead norms.
* .....
* C C /
* /< /
* ___ __________/_#__=o
* /(- /(\_\________ \
* \ ) \ )_ \o \
* /|\ /|\ |' |
* | _|
* /o __\
* / ' |
* / / |
* /_/\______|
* ( _( <
* \ \ \
* \ \ |
* \____\____\
* ____\_\__\_\
* /` /` o\
* |___ |_______|
*
*/
public void testReadUndeadNorms() throws Exception {
InputStream resource = TestLucene42NormsFormat.class.getResourceAsStream("index.42.undeadnorms.zip");
assertNotNull(resource);
Path path = createTempDir("undeadnorms");
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
IndexReader r = DirectoryReader.open(dir);
NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
assertNotNull(undeadNorms);
assertEquals(2, r.maxDoc());
assertEquals(0, undeadNorms.get(0));
assertEquals(0, undeadNorms.get(1));
dir.close();
r.close();
}
}

BIN
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip Normal file
View File

Binary file not shown.

129
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java Normal file
View File

@ -0,0 +1,129 @@
package org.apache.lucene.codecs.lucene46;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.InputStream;
import java.nio.file.Path;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
public class TestLucene46UndeadNorms extends LuceneTestCase {
/** Copy this back to /l/461/lucene/CreateUndeadNorms.java, then:
* - ant clean
* - pushd analysis/common; ant jar; popd
* - pushd core; ant jar; popd
* - javac -cp build/analysis/common/lucene-analyzers-common-4.6-SNAPSHOT.jar:build/core/lucene-core-4.6-SNAPSHOT.jar CreateUndeadNorms.java
* - java -cp .:build/analysis/common/lucene-analyzers-common-4.6-SNAPSHOT.jar:build/core/lucene-core-4.6-SNAPSHOT.jar CreateUndeadNorms
* - cd /tmp/undeadnorms ; zip index.46.undeadnorms.zip *
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
public class CreateUndeadNorms {
public static void main(String[] args) throws Exception {
File file = new File("/tmp/undeadnorms");
if (file.exists()) {
throw new RuntimeException("please remove /tmp/undeadnorms first");
}
Directory dir = FSDirectory.open(file);
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_46, new WhitespaceAnalyzer(Version.LUCENE_46)));
Document doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(new StringField("id", "1", Field.Store.NO));
Field content = new TextField("content", "some content", Field.Store.NO);
content.setTokenStream(new TokenStream() {
@Override
public boolean incrementToken() throws IOException {
throw new IOException("brains brains!");
}
});
doc.add(content);
try {
w.addDocument(doc);
throw new RuntimeException("didn't hit exception");
} catch (IOException ioe) {
// perfect
}
w.close();
dir.close();
}
}
*/
/**
* LUCENE-6006: Test undead norms.
* .....
* C C /
* /< /
* ___ __________/_#__=o
* /(- /(\_\________ \
* \ ) \ )_ \o \
* /|\ /|\ |' |
* | _|
* /o __\
* / ' |
* / / |
* /_/\______|
* ( _( <
* \ \ \
* \ \ |
* \____\____\
* ____\_\__\_\
* /` /` o\
* |___ |_______|
*
*/
public void testReadUndeadNorms() throws Exception {
InputStream resource = TestLucene46UndeadNorms.class.getResourceAsStream("index.46.undeadnorms.zip");
assertNotNull(resource);
Path path = createTempDir("undeadnorms");
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
IndexReader r = DirectoryReader.open(dir);
NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
assertNotNull(undeadNorms);
assertEquals(2, r.maxDoc());
assertEquals(0, undeadNorms.get(0));
assertEquals(0, undeadNorms.get(1));
dir.close();
r.close();
}
}

BIN
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip Normal file
View File

Binary file not shown.

107
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
View File

@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene49;
* limitations under the License.
*/
import java.io.InputStream;
import java.nio.file.Path;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.TestUtil;
/**
* Tests Lucene49NormsFormat
@ -30,4 +40,101 @@ public class TestLucene49NormsFormat extends BaseNormsFormatTestCase {
protected Codec getCodec() {
return codec;
}
/** Copy this back to /l/491/lucene/CreateUndeadNorms.java, then:
* - ant clean
* - pushd analysis/common; ant jar; popd
* - pushd core; ant jar; popd
* - javac -cp build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms.java
* - java -cp .:build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms
* - cd /tmp/undeadnorms ; zip index.49.undeadnorms.zip *
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
public class CreateUndeadNorms {
public static void main(String[] args) throws Exception {
File file = new File("/tmp/undeadnorms");
if (file.exists()) {
throw new RuntimeException("please remove /tmp/undeadnorms first");
}
Directory dir = FSDirectory.open(file);
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_4_9, new WhitespaceAnalyzer(Version.LUCENE_4_9)));
Document doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO));
w.addDocument(doc);
doc = new Document();
doc.add(new StringField("id", "1", Field.Store.NO));
Field content = new TextField("content", "some content", Field.Store.NO);
content.setTokenStream(new TokenStream() {
@Override
public boolean incrementToken() throws IOException {
throw new IOException("brains brains!");
}
});
doc.add(content);
try {
w.addDocument(doc);
throw new RuntimeException("didn't hit exception");
} catch (IOException ioe) {
// perfect
}
w.close();
dir.close();
}
}
*/
/**
* LUCENE-6006: Test undead norms.
* .....
* C C /
* /< /
* ___ __________/_#__=o
* /(- /(\_\________ \
* \ ) \ )_ \o \
* /|\ /|\ |' |
* | _|
* /o __\
* / ' |
* / / |
* /_/\______|
* ( _( <
* \ \ \
* \ \ |
* \____\____\
* ____\_\__\_\
* /` /` o\
* |___ |_______|
*
*/
public void testReadUndeadNorms() throws Exception {
InputStream resource = TestLucene49NormsFormat.class.getResourceAsStream("index.49.undeadnorms.zip");
assertNotNull(resource);
Path path = createTempDir("undeadnorms");
TestUtil.unzip(resource, path);
Directory dir = FSDirectory.open(path);
IndexReader r = DirectoryReader.open(dir);
NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
assertNotNull(undeadNorms);
assertEquals(2, r.maxDoc());
assertEquals(0, undeadNorms.get(0));
assertEquals(0, undeadNorms.get(1));
dir.close();
r.close();
}
}

BIN
lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip Normal file
View File

Binary file not shown.

3
lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -121,7 +120,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
private Path getIndexDir() {
String path = System.getProperty("tests.bwcdir");
assumeTrue("backcompat creation tests must be run with -Dtests,bwcdir=/path/to/write/indexes", path != null);
assumeTrue("backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes", path != null);
return Paths.get(path);
}

3
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
View File

@ -71,8 +71,7 @@ class SimpleTextDocValuesWriter extends DocValuesConsumer {
@Override
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
assert fieldSeen(field.name);
assert (field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC ||
field.getNormType() == FieldInfo.DocValuesType.NUMERIC);
assert field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC || field.hasNorms();
writeFieldEntry(field, FieldInfo.DocValuesType.NUMERIC);
// first pass to find min/max

14
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
View File

@ -59,7 +59,6 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
static final BytesRef STORETVOFF = new BytesRef(" term vector offsets ");
static final BytesRef PAYLOADS = new BytesRef(" payloads ");
static final BytesRef NORMS = new BytesRef(" norms ");
static final BytesRef NORMS_TYPE = new BytesRef(" norms type ");
static final BytesRef DOCVALUES = new BytesRef(" doc values ");
static final BytesRef DOCVALUES_GEN = new BytesRef(" doc values gen ");
static final BytesRef INDEXOPTIONS = new BytesRef(" index options ");
@ -115,11 +114,6 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
assert StringHelper.startsWith(scratch.get(), NORMS);
boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), NORMS_TYPE);
String nrmType = readString(NORMS_TYPE.length, scratch);
final DocValuesType normsType = docValuesType(nrmType);
SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DOCVALUES);
String dvType = readString(DOCVALUES.length, scratch);
@ -146,7 +140,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(atts));
omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts));
}
SimpleTextUtil.checkFooter(input);
@ -217,11 +211,7 @@ public class SimpleTextFieldInfosFormat extends FieldInfosFormat {
SimpleTextUtil.write(out, NORMS);
SimpleTextUtil.write(out, Boolean.toString(!fi.omitsNorms()), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, NORMS_TYPE);
SimpleTextUtil.write(out, getDocValuesType(fi.getNormType()), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, DOCVALUES);
SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
SimpleTextUtil.writeNewline(out);

13
lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
View File

@ -102,7 +102,7 @@ import org.apache.lucene.store.IndexOutput;
* @lucene.experimental
*/
public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
/** Sole constructor. */
public Lucene50FieldInfosFormat() {
}
@ -149,12 +149,11 @@ public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
// DV Types are packed in one byte
byte val = input.readByte();
final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
try {
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads,
indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@ -215,12 +214,10 @@ public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
output.writeVInt(fi.number);
output.writeByte(bits);
// pack the DV types in one byte
// pack the DV type and hasNorms in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
final byte nrm = docValuesByte(fi.getNormType());
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
assert (dv & (~0xF)) == 0;
output.writeByte(dv);
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}

8
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -1790,12 +1790,8 @@ public class CheckIndex implements Closeable {
}
private static void checkNorms(FieldInfo fi, LeafReader reader, PrintStream infoStream) throws IOException {
switch(fi.getNormType()) {
case NUMERIC:
checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
break;
default:
throw new AssertionError("wtf: " + fi.getNormType());
if (fi.hasNorms()) {
checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
}
}

27
lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
View File

@ -183,14 +183,10 @@ final class DefaultIndexingChain extends DocConsumer {
// we must check the final value of omitNorms for the fieldinfo: it could have
// changed for this field since the first time we added it.
if (fi.omitsNorms() == false) {
if (perField.norms != null) {
perField.norms.finish(state.segmentInfo.getDocCount());
perField.norms.flush(state, normsConsumer);
assert fi.getNormType() == DocValuesType.NUMERIC;
} else if (fi.isIndexed()) {
assert fi.getNormType() == null: "got " + fi.getNormType() + "; field=" + fi.name;
}
if (fi.omitsNorms() == false && fi.isIndexed()) {
assert perField.norms != null: "field=" + fi.name;
perField.norms.finish(state.segmentInfo.getDocCount());
perField.norms.flush(state, normsConsumer);
}
}
}
@ -535,6 +531,11 @@ final class DefaultIndexingChain extends DocConsumer {
void setInvertState() {
invertState = new FieldInvertState(fieldInfo.name);
termsHashPerField = termsHash.addField(invertState, fieldInfo);
if (fieldInfo.omitsNorms() == false) {
assert norms == null;
// Even if no documents actually succeed in setting a norm, we still write norms for this segment:
norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
}
}
@Override
@ -543,14 +544,8 @@ final class DefaultIndexingChain extends DocConsumer {
}
public void finish() throws IOException {
if (fieldInfo.omitsNorms() == false) {
if (norms == null) {
fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
}
if (invertState.length != 0) {
norms.addValue(docState.docID, similarity.computeNorm(invertState));
}
if (fieldInfo.omitsNorms() == false && invertState.length != 0) {
norms.addValue(docState.docID, similarity.computeNorm(invertState));
}
termsHashPerField.finish();

37
lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
View File

@ -40,15 +40,15 @@ public final class FieldInfo {
// True if any document indexed term vectors
private boolean storeTermVector;
private DocValuesType normType;
private boolean omitNorms; // omit norms associated with indexed fields
private IndexOptions indexOptions;
private boolean storePayloads; // whether this field stores payloads together with term positions
private Map<String,String> attributes;
private long dvGen;
/**
* Controls how much information is stored in the postings lists.
* @lucene.experimental
@ -120,12 +120,12 @@ public final class FieldInfo {
}
/**
* Sole Constructor.
* Sole constructor.
*
* @lucene.experimental
*/
public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector, boolean omitNorms,
boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normsType,
boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
long dvGen, Map<String,String> attributes) {
this.name = name;
this.indexed = indexed;
@ -136,13 +136,11 @@ public final class FieldInfo {
this.storePayloads = storePayloads;
this.omitNorms = omitNorms;
this.indexOptions = indexOptions;
this.normType = !omitNorms ? normsType : null;
} else { // for non-indexed fields, leave defaults
this.storeTermVector = false;
this.storePayloads = false;
this.omitNorms = false;
this.indexOptions = null;
this.normType = null;
}
this.dvGen = dvGen;
this.attributes = attributes;
@ -158,11 +156,6 @@ public final class FieldInfo {
if (indexOptions == null) {
throw new IllegalStateException("indexed field '" + name + "' must have index options");
}
if (omitNorms) {
if (normType != null) {
throw new IllegalStateException("indexed field '" + name + "' cannot both omit norms and have norms");
}
}
// Cannot store payloads unless positions are indexed:
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0 && storePayloads) {
throw new IllegalStateException("indexed field '" + name + "' cannot have payloads without positions");
@ -177,12 +170,8 @@ public final class FieldInfo {
if (omitNorms) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot omit norms");
}
if (normType != null) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot have norms");
}
if (indexOptions != null) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot have index options");
}
}
@ -206,7 +195,6 @@ public final class FieldInfo {
this.storePayloads |= storePayloads;
if (this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
this.normType = null;
}
if (this.indexOptions != indexOptions) {
if (this.indexOptions == null) {
@ -265,13 +253,6 @@ public final class FieldInfo {
return dvGen;
}
/**
* Returns {@link DocValuesType} of the norm. this may be null if the field has no norms.
*/
public DocValuesType getNormType() {
return normType;
}
void setStoreTermVectors() {
storeTermVector = true;
assert checkConsistency();
@ -284,14 +265,6 @@ public final class FieldInfo {
assert checkConsistency();
}
void setNormValueType(DocValuesType type) {
if (normType != null && normType != type) {
throw new IllegalArgumentException("cannot change Norm type from " + normType + " to " + type + " for field \"" + name + "\"");
}
normType = type;
assert checkConsistency();
}
/**
* Returns true if norms are explicitly omitted for this field
*/
@ -303,7 +276,7 @@ public final class FieldInfo {
* Returns true if this field actually has any norms.
*/
public boolean hasNorms() {
return normType != null;
return indexed && omitNorms == false;
}
/**

22
lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
View File

@ -276,10 +276,8 @@ public class FieldInfos implements Iterable<FieldInfo> {
}
/** NOTE: this method does not carry over termVector
* booleans nor docValuesType; the indexer chain
* (TermVectorsConsumerPerField, DocFieldProcessor) must
* set these fields when they succeed in consuming
* the document */
* the indexer chain must set these fields when they
* succeed in consuming the document */
public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
// TODO: really, indexer shouldn't even call this
// method (it's only called from DocFieldProcessor);
@ -288,12 +286,12 @@ public class FieldInfos implements Iterable<FieldInfo> {
// be updated by maybe FreqProxTermsWriterPerField:
return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
fieldType.omitNorms(), false,
fieldType.indexOptions(), fieldType.docValueType(), null);
fieldType.indexOptions(), fieldType.docValueType());
}
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normType) {
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
FieldInfo fi = fieldInfo(name);
if (fi == null) {
// This field wasn't yet added to this in-RAM
@ -302,7 +300,7 @@ public class FieldInfos implements Iterable<FieldInfo> {
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType, -1, null);
fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
assert !byName.containsKey(fi.name);
assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
@ -319,26 +317,22 @@ public class FieldInfos implements Iterable<FieldInfo> {
globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
}
}
if (!fi.omitsNorms() && normType != null) {
fi.setNormValueType(normType);
}
}
return fi;
}
public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
fi.getIndexOptions(), fi.getDocValuesType(), fi.getNormType());
fi.getIndexOptions(), fi.getDocValuesType());
}
public FieldInfo fieldInfo(String fieldName) {
return byName.get(fieldName);
}
final FieldInfos finish() {
FieldInfos finish() {
return new FieldInfos(byName.values().toArray(new FieldInfo[byName.size()]));
}
}

2
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -445,7 +445,7 @@ public class MemoryIndex {
if (!fieldInfos.containsKey(fieldName)) {
fieldInfos.put(fieldName,
new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, null, -1, null));
new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, -1, null));
}
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);

12
lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
View File

@ -21,22 +21,22 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
import org.apache.lucene.document.BinaryDocValuesField; // javadocs
import org.apache.lucene.document.DoubleField; // javadocs
import org.apache.lucene.document.FloatField; // javadocs
import org.apache.lucene.document.IntField; // javadocs
import org.apache.lucene.document.LongField; // javadocs
import org.apache.lucene.document.FloatField; // javadocs
import org.apache.lucene.document.DoubleField; // javadocs
import org.apache.lucene.document.BinaryDocValuesField; // javadocs
import org.apache.lucene.document.NumericDocValuesField; // javadocs
import org.apache.lucene.document.SortedDocValuesField; // javadocs
import org.apache.lucene.document.SortedSetDocValuesField; // javadocs
import org.apache.lucene.document.StringField; // javadocs
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterDirectoryReader;
import org.apache.lucene.index.FilterLeafReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@ -213,7 +213,7 @@ public class UninvertingReader extends FilterLeafReader {
}
}
filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), type, fi.getNormType(), -1, null));
fi.hasPayloads(), fi.getIndexOptions(), type, -1, null));
}
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
}

2
lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
View File

@ -93,7 +93,7 @@ public class AssertingNormsFormat extends NormsFormat {
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
assert field.getNormType() == FieldInfo.DocValuesType.NUMERIC;
assert field.hasNorms();
NumericDocValues values = in.getNorms(field);
assert values != null;
return new AssertingLeafReader.AssertingNumericDocValues(values, maxDoc);

6
lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
View File

@ -89,11 +89,6 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
fi.setStorePayloads();
}
}
if (fi.isIndexed() && !fi.omitsNorms()) {
if (random().nextBoolean()) {
fi.setNormValueType(DocValuesType.NUMERIC);
}
}
addAttributes(fi);
}
FieldInfos infos = builder.finish();
@ -165,7 +160,6 @@ public abstract class BaseFieldInfoFormatTestCase extends BaseIndexFileFormatTes
assertEquals(expected.name, actual.name);
assertEquals(expected.getDocValuesType(), actual.getDocValuesType());
assertEquals(expected.getIndexOptions(), actual.getIndexOptions());
assertEquals(expected.getNormType(), actual.getNormType());
assertEquals(expected.hasDocValues(), actual.hasDocValues());
assertEquals(expected.hasNorms(), actual.hasNorms());
assertEquals(expected.hasPayloads(), actual.hasPayloads());

59
lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
View File

@ -18,6 +18,8 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@ -128,7 +130,6 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
public void testAllZeros() throws Exception {
int iterations = atLeast(1);
final Random r = random();
for (int i = 0; i < iterations; i++) {
doTestNormsVersusStoredFields(new LongProducer() {
@Override
@ -264,4 +265,60 @@ public abstract class BaseNormsFormatTestCase extends BaseIndexFileFormatTestCas
}
// TODO: test thread safety (e.g. across different fields) explicitly here
/**
* LUCENE-6006: Tests undead norms.
* .....
* C C /
* /< /
* ___ __________/_#__=o
* /(- /(\_\________ \
* \ ) \ )_ \o \
* /|\ /|\ |' |
* | _|
* /o __\
* / ' |
* / / |
* /_/\______|
* ( _( <
* \ \ \
* \ \ |
* \____\____\
* ____\_\__\_\
* /` /` o\
* |___ |_______|
*
*/
public void testUndeadNorms() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), dir);
int numDocs = atLeast(1000);
List<Integer> toDelete = new ArrayList<>();
for(int i=0;i<numDocs;i++) {
Document doc = new Document();
doc.add(new StringField("id", ""+i, Field.Store.NO));
if (random().nextInt(5) == 1) {
toDelete.add(i);
doc.add(new TextField("content", "some content", Field.Store.NO));
}
w.addDocument(doc);
}
for(Integer id : toDelete) {
w.deleteDocuments(new Term("id", ""+id));
}
w.forceMerge(1);
IndexReader r = w.getReader();
// Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not
// be undead:
NumericDocValues norms = MultiDocValues.getNormValues(r, "content");
assertNotNull(norms);
for(int i=0;i<r.maxDoc();i++) {
assertEquals(0, norms.get(i));
}
r.close();
w.close();
dir.close();
}
}

4
lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
View File

@ -45,7 +45,6 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.Directory;
@ -373,7 +372,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
null, DocValuesType.NUMERIC, -1, null);
null, -1, null);
fieldUpto++;
SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@ -702,7 +701,6 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
doPayloads,
indexOptions,
null,
DocValuesType.NUMERIC,
-1,
null);
}

2
solr/core/src/java/org/apache/solr/search/Insanity.java
View File

@ -65,7 +65,7 @@ public class Insanity {
for (FieldInfo fi : in.getFieldInfos()) {
if (fi.name.equals(insaneField)) {
filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
fi.hasPayloads(), fi.getIndexOptions(), null, fi.getNormType(), -1, null));
fi.hasPayloads(), fi.getIndexOptions(), null, -1, null));
} else {
filteredInfos.add(fi);
}