Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

revisit payloads API in DocsAndPositionsEnum 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1372171 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-08-12 18:45:46 +00:00
parent 48e9e9f361
commit 0f3b19899e
38 changed files with 115 additions and 240 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lucene/CHANGES.txt
View File

@ -22,6 +22,11 @@ API Changes
had positions or offsets, since this can be configured on a
per-field-per-document basis. (Robert Muir)
* Removed DocsAndPositionsEnum.hasPayload() and simplified the
contract of getPayload(). It returns null if there is no payload,
otherwise returns the current payload. You can now call it multiple
times per position if you want. (Robert Muir)
Bug Fixes
* LUCENE-4297: BooleanScorer2 would multiply the coord() factor

8
lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java
View File

@ -126,6 +126,7 @@ public final class MappingMultiDocsAndPositionsEnum extends DocsAndPositionsEnum
BytesRef payload = current.getPayload();
if (mergeState.currentPayloadProcessor[upto] != null && payload != null) {
// to not violate the D&P api, we must give the processor a private copy
// TODO: reuse a BytesRef if there is a PPP
payload = BytesRef.deepCopyOf(payload);
mergeState.currentPayloadProcessor[upto].processPayload(payload);
if (payload.length == 0) {
@ -135,12 +136,5 @@ public final class MappingMultiDocsAndPositionsEnum extends DocsAndPositionsEnum
}
return payload;
}
@Override
public boolean hasPayload() {
// TODO: note this is actually bogus if there is a payloadProcessor,
// because it might remove it: but lets just remove this method completely
return current.hasPayload();
}
}

14
lucene/core/src/java/org/apache/lucene/codecs/PostingsConsumer.java
View File

@ -112,12 +112,7 @@ public abstract class PostingsConsumer {
totTF += freq;
for(int i=0;i<freq;i++) {
final int position = postingsEnum.nextPosition();
final BytesRef payload;
if (postingsEnum.hasPayload()) {
payload = postingsEnum.getPayload();
} else {
payload = null;
}
final BytesRef payload = postingsEnum.getPayload();
this.addPosition(position, payload, -1, -1);
}
this.finishDoc();
@ -137,12 +132,7 @@ public abstract class PostingsConsumer {
totTF += freq;
for(int i=0;i<freq;i++) {
final int position = postingsEnum.nextPosition();
final BytesRef payload;
if (postingsEnum.hasPayload()) {
payload = postingsEnum.getPayload();
} else {
payload = null;
}
final BytesRef payload = postingsEnum.getPayload();
this.addPosition(position, payload, postingsEnum.startOffset(), postingsEnum.endOffset());
}
this.finishDoc();

3
lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
View File

@ -271,8 +271,7 @@ public abstract class TermVectorsWriter implements Closeable {
final int startOffset = docsAndPositionsEnum.startOffset();
final int endOffset = docsAndPositionsEnum.endOffset();
BytesRef payload = docsAndPositionsEnum.hasPayload() ?
docsAndPositionsEnum.getPayload() : null;
BytesRef payload = docsAndPositionsEnum.getPayload();
if (payloadProcessor != null && payload != null) {
// to not violate the D&P api, we must give the processor a private copy

35
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
View File

@ -873,12 +873,7 @@ public class Lucene40PostingsReader extends PostingsReaderBase {
* payload was indexed. */
@Override
public BytesRef getPayload() throws IOException {
throw new IOException("No payloads exist for this field!");
}
@Override
public boolean hasPayload() {
return false;
return null;
}
}
@ -1152,28 +1147,26 @@ public class Lucene40PostingsReader extends PostingsReaderBase {
@Override
public BytesRef getPayload() throws IOException {
if (storePayloads) {
if (payloadLength <= 0) {
return null;
}
assert lazyProxPointer == -1;
assert posPendingCount < freq;
if (!payloadPending) {
throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
}
if (payloadLength > payload.bytes.length) {
payload.grow(payloadLength);
}
if (payloadPending) {
if (payloadLength > payload.bytes.length) {
payload.grow(payloadLength);
}
proxIn.readBytes(payload.bytes, 0, payloadLength);
payload.length = payloadLength;
payloadPending = false;
proxIn.readBytes(payload.bytes, 0, payloadLength);
payload.length = payloadLength;
payloadPending = false;
}
return payload;
} else {
throw new IOException("No payloads exist for this field!");
return null;
}
}
@Override
public boolean hasPayload() {
return payloadPending && payloadLength > 0;
}
}
}

12
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
View File

@ -671,7 +671,6 @@ public class Lucene40TermVectorsReader extends TermVectorsReader {
@Override
public BytesRef getPayload() {
// TODO: dumb that we have to duplicate hasPayload
if (payloadOffsets == null) {
return null;
} else {
@ -687,17 +686,6 @@ public class Lucene40TermVectorsReader extends TermVectorsReader {
}
}
@Override
public boolean hasPayload() {
if (payloadOffsets == null) {
return false;
} else {
int off = payloadOffsets[nextPos-1];
int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
return end - off > 0;
}
}
@Override
public int nextPosition() {
assert (positions != null && nextPos < positions.length) ||

41
lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
View File

@ -348,9 +348,8 @@ public class DirectPostingsFormat extends PostingsFormat {
scratch.add(docsAndPositionsEnum.endOffset());
}
if (hasPayloads) {
final BytesRef payload;
if (docsAndPositionsEnum.hasPayload()) {
payload = docsAndPositionsEnum.getPayload();
final BytesRef payload = docsAndPositionsEnum.getPayload();
if (payload != null) {
scratch.add(payload.length);
ros.writeBytes(payload.bytes, payload.offset, payload.length);
} else {
@ -421,9 +420,8 @@ public class DirectPostingsFormat extends PostingsFormat {
for(int pos=0;pos<freq;pos++) {
positions[upto][posUpto] = docsAndPositionsEnum.nextPosition();
if (hasPayloads) {
if (docsAndPositionsEnum.hasPayload()) {
BytesRef payload = docsAndPositionsEnum.getPayload();
assert payload != null;
BytesRef payload = docsAndPositionsEnum.getPayload();
if (payload != null) {
byte[] payloadBytes = new byte[payload.length];
System.arraycopy(payload.bytes, payload.offset, payloadBytes, 0, payload.length);
payloads[upto][pos] = payloadBytes;
@ -1806,18 +1804,12 @@ public class DirectPostingsFormat extends PostingsFormat {
return docID;
}
@Override
public boolean hasPayload() {
return payloadLength > 0;
}
@Override
public BytesRef getPayload() {
if (payloadLength > 0) {
payload.bytes = payloadBytes;
payload.offset = lastPayloadOffset;
payload.length = payloadLength;
payloadLength = 0;
return payload;
} else {
return null;
@ -2010,7 +2002,6 @@ public class DirectPostingsFormat extends PostingsFormat {
private int upto;
private int docID = -1;
private int posUpto;
private boolean gotPayload;
private int[] curPositions;
public HighFreqDocsAndPositionsEnum(Bits liveDocs, boolean hasOffsets) {
@ -2080,7 +2071,6 @@ public class DirectPostingsFormat extends PostingsFormat {
@Override
public int nextPosition() {
posUpto += posJump;
gotPayload = false;
return curPositions[posUpto];
}
@ -2214,21 +2204,22 @@ public class DirectPostingsFormat extends PostingsFormat {
}
}
@Override
public boolean hasPayload() {
return !gotPayload && payloads != null && payloads[upto][posUpto/(hasOffsets ? 3 : 1)] != null;
}
private final BytesRef payload = new BytesRef();
@Override
public BytesRef getPayload() {
final byte[] payloadBytes = payloads[upto][posUpto/(hasOffsets ? 3:1)];
payload.bytes = payloadBytes;
payload.length = payloadBytes.length;
payload.offset = 0;
gotPayload = true;
return payload;
if (payloads == null) {
return null;
} else {
final byte[] payloadBytes = payloads[upto][posUpto/(hasOffsets ? 3:1)];
if (payloadBytes == null) {
return null;
}
payload.bytes = payloadBytes;
payload.length = payloadBytes.length;
payload.offset = 0;
return payload;
}
}
}
}

14
lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
View File

@ -446,7 +446,6 @@ public class MemoryPostingsFormat extends PostingsFormat {
private int numDocs;
private int posPending;
private int payloadLength;
private boolean payloadRetrieved;
final boolean storeOffsets;
int offsetLength;
int startOffset;
@ -484,7 +483,6 @@ public class MemoryPostingsFormat extends PostingsFormat {
payloadLength = 0;
this.numDocs = numDocs;
posPending = 0;
payloadRetrieved = false;
startOffset = storeOffsets ? 0 : -1; // always return -1 if no offsets are stored
offsetLength = 0;
return this;
@ -577,10 +575,6 @@ public class MemoryPostingsFormat extends PostingsFormat {
payload.offset = in.getPosition();
in.skipBytes(payloadLength);
payload.length = payloadLength;
// Necessary, in case caller changed the
// payload.bytes from prior call:
payload.bytes = buffer;
payloadRetrieved = false;
}
//System.out.println(" pos=" + pos + " payload=" + payload + " fp=" + in.getPosition());
@ -599,13 +593,7 @@ public class MemoryPostingsFormat extends PostingsFormat {
@Override
public BytesRef getPayload() {
payloadRetrieved = true;
return payload;
}
@Override
public boolean hasPayload() {
return !payloadRetrieved && payload.length > 0;
return payload.length > 0 ? payload : null;
}
@Override

12
lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
View File

@ -532,19 +532,13 @@ public class PulsingPostingsReader extends PostingsReaderBase {
}
}
@Override
public boolean hasPayload() {
return storePayloads && !payloadRetrieved && payloadLength > 0;
}
@Override
public BytesRef getPayload() throws IOException {
//System.out.println("PR getPayload payloadLength=" + payloadLength + " this=" + this);
if (payloadRetrieved) {
throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
}
payloadRetrieved = true;
if (payloadLength > 0) {
return payload;
} else if (storePayloads && payloadLength > 0) {
payloadRetrieved = true;
if (payload == null) {
payload = new BytesRef(payloadLength);
} else {

12
lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
View File

@ -714,7 +714,11 @@ public class SepPostingsReader extends PostingsReaderBase {
@Override
public BytesRef getPayload() throws IOException {
if (!payloadPending) {
throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once.");
return null;
}
if (pendingPayloadBytes == 0) {
return payload;
}
assert pendingPayloadBytes >= payloadLength;
@ -731,15 +735,9 @@ public class SepPostingsReader extends PostingsReaderBase {
}
payloadIn.readBytes(payload.bytes, 0, payloadLength);
payloadPending = false;
payload.length = payloadLength;
pendingPayloadBytes = 0;
return payload;
}
@Override
public boolean hasPayload() {
return payloadPending && payloadLength > 0;
}
}
}

13
lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
View File

@ -472,18 +472,7 @@ class SimpleTextFieldsReader extends FieldsProducer {
@Override
public BytesRef getPayload() {
// Some tests rely on only being able to retrieve the
// payload once
try {
return payload;
} finally {
payload = null;
}
}
@Override
public boolean hasPayload() {
return payload != null;
return payload;
}
}

6
lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
View File

@ -524,15 +524,9 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader {
@Override
public BytesRef getPayload() {
// assert hasPayload(); // you should have called this
return payloads == null ? null : payloads[nextPos-1];
}
@Override
public boolean hasPayload() {
return payloads != null && payloads[nextPos-1] != null;
}
@Override
public int nextPosition() {
assert (positions != null && nextPos < positions.length) ||

18
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -830,11 +830,9 @@ public class CheckIndex {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
}
lastPos = pos;
if (postings.hasPayload()) {
BytesRef payload = postings.getPayload();
if (payload.length < 1) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length);
}
BytesRef payload = postings.getPayload();
if (payload != null && payload.length < 1) {
throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length);
}
if (hasOffsets) {
int startOffset = postings.startOffset();
@ -1526,10 +1524,10 @@ public class CheckIndex {
}
}
BytesRef payload = null;
if (postings.hasPayload()) {
BytesRef payload = postings.getPayload();
if (payload != null) {
assert vectorsHasPayload;
payload = postings.getPayload();
}
if (postingsHasPayload && vectorsHasPayload) {
@ -1538,13 +1536,13 @@ public class CheckIndex {
if (payload == null) {
// we have payloads, but not at this position.
// postings has payloads too, it should not have one at this position
if (postingsPostings.hasPayload()) {
if (postingsPostings.getPayload() != null) {
throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has no payload but postings does: " + postingsPostings.getPayload());
}
} else {
// we have payloads, and one at this position
// postings should also have one at this position, with the same bytes.
if (!postingsPostings.hasPayload()) {
if (postingsPostings.getPayload() == null) {
throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but postings does not.");
}
BytesRef postingsPayload = postingsPostings.getPayload();

9
lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
View File

@ -48,11 +48,8 @@ public abstract class DocsAndPositionsEnum extends DocsEnum {
public abstract int endOffset() throws IOException;
/** Returns the payload at this position, or null if no
* payload was indexed. Only call this once per
* position. You should not modify anything (neither
* members of the returned BytesRef nor bytes in the
* byte[]). */
* payload was indexed. You should not modify anything
* (neither members of the returned BytesRef nor bytes
* in the byte[]). */
public abstract BytesRef getPayload() throws IOException;
public abstract boolean hasPayload();
}

5
lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
View File

@ -307,11 +307,6 @@ public class FilterAtomicReader extends AtomicReader {
public BytesRef getPayload() throws IOException {
return in.getPayload();
}
@Override
public boolean hasPayload() {
return in.hasPayload();
}
@Override
public AttributeSource attributes() {

5
lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java
View File

@ -137,11 +137,6 @@ public final class MultiDocsAndPositionsEnum extends DocsAndPositionsEnum {
return current.endOffset();
}
@Override
public boolean hasPayload() {
return current.hasPayload();
}
@Override
public BytesRef getPayload() throws IOException {
return current.getPayload();

7
lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
View File

@ -548,12 +548,7 @@ class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum {
@Override
public BytesRef getPayload() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPayload() {
throw new UnsupportedOperationException();
return null;
}
@Override

4
lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
View File

@ -118,8 +118,8 @@ public class PayloadTermQuery extends SpanTermQuery {
}
protected void processPayload(Similarity similarity) throws IOException {
final DocsAndPositionsEnum postings = termSpans.getPostings();
if (postings.hasPayload()) {
if (termSpans.isPayloadAvailable()) {
final DocsAndPositionsEnum postings = termSpans.getPostings();
payload = postings.getPayload();
if (payload != null) {
payloadScore = function.currentScore(doc, term.field(),

4
lucene/core/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
View File

@ -126,7 +126,7 @@ public class NearSpansUnordered extends Spans {
// TODO: Remove warning after API has been finalized
@Override
public boolean isPayloadAvailable() {
public boolean isPayloadAvailable() throws IOException {
return spans.isPayloadAvailable();
}
@ -256,7 +256,7 @@ public class NearSpansUnordered extends Spans {
// TODO: Remove warning after API has been finalized
@Override
public boolean isPayloadAvailable() {
public boolean isPayloadAvailable() throws IOException {
SpansCell pointer = min();
while (pointer != null) {
if (pointer.isPayloadAvailable()) {

2
lucene/core/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
View File

@ -157,7 +157,7 @@ public class SpanNotQuery extends SpanQuery implements Cloneable {
// TODO: Remove warning after API has been finalized
@Override
public boolean isPayloadAvailable() {
public boolean isPayloadAvailable() throws IOException {
return includeSpans.isPayloadAvailable();
}

2
lucene/core/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
View File

@ -247,7 +247,7 @@ public class SpanOrQuery extends SpanQuery implements Cloneable {
}
@Override
public boolean isPayloadAvailable() {
public boolean isPayloadAvailable() throws IOException {
Spans top = top();
return top != null && top.isPayloadAvailable();
}

2
lucene/core/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
View File

@ -176,7 +176,7 @@ public abstract class SpanPositionCheckQuery extends SpanQuery implements Clonea
// TODO: Remove warning after API has been finalized
@Override
public boolean isPayloadAvailable() {
public boolean isPayloadAvailable() throws IOException {
return spans.isPayloadAvailable();
}

2
lucene/core/src/java/org/apache/lucene/search/spans/Spans.java
View File

@ -82,6 +82,6 @@ public abstract class Spans {
*
* @return true if there is a payload available at this position that can be loaded
*/
public abstract boolean isPayloadAvailable();
public abstract boolean isPayloadAvailable() throws IOException;
}

9
lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java
View File

@ -36,6 +36,7 @@ public class TermSpans extends Spans {
protected int freq;
protected int count;
protected int position;
protected boolean readPayload;
public TermSpans(DocsAndPositionsEnum postings, Term term) {
this.postings = postings;
@ -64,6 +65,7 @@ public class TermSpans extends Spans {
}
position = postings.nextPosition();
count++;
readPayload = false;
return true;
}
@ -78,7 +80,7 @@ public class TermSpans extends Spans {
count = 0;
position = postings.nextPosition();
count++;
readPayload = false;
return true;
}
@ -101,6 +103,7 @@ public class TermSpans extends Spans {
@Override
public Collection<byte[]> getPayload() throws IOException {
final BytesRef payload = postings.getPayload();
readPayload = true;
final byte[] bytes;
if (payload != null) {
bytes = new byte[payload.length];
@ -113,8 +116,8 @@ public class TermSpans extends Spans {
// TODO: Remove warning after API has been finalized
@Override
public boolean isPayloadAvailable() {
return postings.hasPayload();
public boolean isPayloadAvailable() throws IOException {
return readPayload == false && postings.getPayload() != null;
}
@Override

4
lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
View File

@ -436,14 +436,14 @@ public class TestCodecs extends LuceneTestCase {
final int pos = posEnum.nextPosition();
assertEquals(positions[i].pos, pos);
if (positions[i].payload != null) {
assertTrue(posEnum.hasPayload());
assertNotNull(posEnum.getPayload());
if (random().nextInt(3) < 2) {
// Verify the payload bytes
final BytesRef otherPayload = posEnum.getPayload();
assertTrue("expected=" + positions[i].payload.toString() + " got=" + otherPayload.toString(), positions[i].payload.equals(otherPayload));
}
} else {
assertFalse(posEnum.hasPayload());
assertNull(posEnum.getPayload());
}
}
}

6
lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
View File

@ -205,11 +205,11 @@ public class TestDocumentWriter extends LuceneTestCase {
int freq = termPositions.freq();
assertEquals(3, freq);
assertEquals(0, termPositions.nextPosition());
assertEquals(true, termPositions.hasPayload());
assertNotNull(termPositions.getPayload());
assertEquals(6, termPositions.nextPosition());
assertEquals(false, termPositions.hasPayload());
assertNull(termPositions.getPayload());
assertEquals(7, termPositions.nextPosition());
assertEquals(false, termPositions.hasPayload());
assertNull(termPositions.getPayload());
reader.close();
}

10
lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
View File

@ -412,12 +412,9 @@ public class TestDuelingCodecs extends LuceneTestCase {
assertEquals(info, freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
if (leftDocs.hasPayload()) {
assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
}
}
}
assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
@ -509,10 +506,7 @@ public class TestDuelingCodecs extends LuceneTestCase {
assertEquals(info, freq, rightDocs.freq());
for (int i = 0; i < freq; i++) {
assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
assertEquals(info, leftDocs.hasPayload(), rightDocs.hasPayload());
if (leftDocs.hasPayload()) {
assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
}
assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
}
}
}

10
lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java
View File

@ -205,8 +205,11 @@ public class TestLongPostings extends LuceneTestCase {
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
assertEquals(pos, postings.nextPosition());
if (random().nextBoolean() && postings.hasPayload()) {
if (random().nextBoolean()) {
postings.getPayload();
if (random().nextBoolean()) {
postings.getPayload(); // get it again
}
}
}
}
@ -247,8 +250,11 @@ public class TestLongPostings extends LuceneTestCase {
assertTrue(freq >=1 && freq <= 4);
for(int pos=0;pos<freq;pos++) {
assertEquals(pos, postings.nextPosition());
if (random().nextBoolean() && postings.hasPayload()) {
if (random().nextBoolean()) {
postings.getPayload();
if (random().nextBoolean()) {
postings.getPayload(); // get it again
}
}
}
}

4
lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
View File

@ -173,8 +173,8 @@ public class TestPayloadProcessorProvider extends LuceneTestCase {
DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text);
while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
tpe.nextPosition();
if (tpe.hasPayload()) {
BytesRef payload = tpe.getPayload();
BytesRef payload = tpe.getPayload();
if (payload != null) {
assertEquals(1, payload.length);
assertEquals(1, payload.bytes[0]);
++numPayloads;

19
lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
View File

@ -201,18 +201,10 @@ public class TestPayloads extends LuceneTestCase {
for (int i = 0; i < freq; i++) {
for (int j = 0; j < numTerms; j++) {
tps[j].nextPosition();
if (tps[j].hasPayload()) {
BytesRef br = tps[j].getPayload();
BytesRef br = tps[j].getPayload();
if (br != null) {
System.arraycopy(br.bytes, br.offset, verifyPayloadData, offset, br.length);
offset += br.length;
// Just to ensure all codecs can
// handle a caller that mucks with the
// returned payload:
if (rarely()) {
br.bytes = new byte[random().nextInt(5)];
}
br.length = 0;
br.offset = 0;
}
}
}
@ -268,11 +260,6 @@ public class TestPayloads extends LuceneTestCase {
tp.nextPosition();
assertEquals("Wrong payload length.", 3 * skipInterval - 2 * numDocs - 1, tp.getPayload().length);
/*
* Test multiple call of getPayload()
*/
assertFalse(tp.hasPayload());
reader.close();
// test long payload
@ -621,7 +608,6 @@ public class TestPayloads extends LuceneTestCase {
DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload"));
de.nextDoc();
de.nextPosition();
assertTrue(de.hasPayload());
assertEquals(new BytesRef("test"), de.getPayload());
writer.close();
reader.close();
@ -656,7 +642,6 @@ public class TestPayloads extends LuceneTestCase {
DocsAndPositionsEnum de = sr.termPositionsEnum(null, "field", new BytesRef("withPayload"));
de.nextDoc();
de.nextPosition();
assertTrue(de.hasPayload());
assertEquals(new BytesRef("test"), de.getPayload());
writer.close();
reader.close();

2
lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
View File

@ -74,7 +74,6 @@ public class TestPayloadsOnVectors extends LuceneTestCase {
DocsAndPositionsEnum de = termsEnum.docsAndPositions(null, null);
assertEquals(0, de.nextDoc());
assertEquals(0, de.nextPosition());
assertTrue(de.hasPayload());
assertEquals(new BytesRef("test"), de.getPayload());
writer.close();
reader.close();
@ -117,7 +116,6 @@ public class TestPayloadsOnVectors extends LuceneTestCase {
DocsAndPositionsEnum de = termsEnum.docsAndPositions(null, null);
assertEquals(0, de.nextDoc());
assertEquals(3, de.nextPosition());
assertTrue(de.hasPayload());
assertEquals(new BytesRef("test"), de.getPayload());
writer.close();
reader.close();

11
lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
View File

@ -731,20 +731,21 @@ public class TestPostingsFormat extends LuceneTestCase {
System.out.println(" now check payload length=" + (position.payload == null ? 0 : position.payload.length));
}
if (position.payload == null || position.payload.length == 0) {
assertFalse("should not have payload", docsAndPositionsEnum.hasPayload());
assertNull("should not have payload", docsAndPositionsEnum.getPayload());
} else {
assertTrue("should have payload but doesn't", docsAndPositionsEnum.hasPayload());
BytesRef payload = docsAndPositionsEnum.getPayload();
assertFalse("2nd call to hasPayload should be false", docsAndPositionsEnum.hasPayload());
assertNotNull("should have payload but doesn't", payload);
assertNotNull("payload should not be null", payload);
assertEquals("payload length is wrong", position.payload.length, payload.length);
for(int byteUpto=0;byteUpto<position.payload.length;byteUpto++) {
assertEquals("payload bytes are wrong",
position.payload[byteUpto],
payload.bytes[payload.offset+byteUpto]);
}
// make a deep copy
payload = BytesRef.deepCopyOf(payload);
assertEquals("2nd call to getPayload returns something different!", payload, docsAndPositionsEnum.getPayload());
}
} else {
if (VERBOSE) {

4
lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
View File

@ -170,7 +170,7 @@ public class TestPostingsOffsets extends LuceneTestCase {
assertTrue(storedNumbers.substring(start, end).equals(term));
if (withPayloads) {
// check that we have a payload and it starts with "pos"
assertTrue(dp.hasPayload());
assertNotNull(dp.getPayload());
BytesRef payload = dp.getPayload();
assertTrue(payload.utf8ToString().startsWith("pos:"));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!
@ -198,7 +198,7 @@ public class TestPostingsOffsets extends LuceneTestCase {
assertTrue(storedNumbers.substring(start, end).equals("hundred"));
if (withPayloads) {
// check that we have a payload and it starts with "pos"
assertTrue(dp.hasPayload());
assertNotNull(dp.getPayload());
BytesRef payload = dp.getPayload();
assertTrue(payload.utf8ToString().startsWith("pos:"));
} // note: withPayloads=false doesnt necessarily mean we dont have them from MockAnalyzer!

2
lucene/core/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java
View File

@ -160,7 +160,7 @@ public class MultiSpansWrapper extends Spans { // can't be package private due t
}
@Override
public boolean isPayloadAvailable() {
public boolean isPayloadAvailable() throws IOException {
if (current == null) {
return false;
}

11
lucene/facet/src/java/org/apache/lucene/facet/search/PayloadIterator.java
View File

@ -99,16 +99,13 @@ public class PayloadIterator {
// Prepare for payload extraction
tp.nextPosition();
// TODO: fix bug in SepCodec and then remove this check (the null check should be enough)
if (!tp.hasPayload()) {
return false;
}
BytesRef br = tp.getPayload();
if (br == null || br.length == 0) {
if (br == null) {
return false;
}
assert br.length > 0;
this.payloadLength = br.length;

5
lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -1015,11 +1015,6 @@ public class MemoryIndex {
return stride == 1 ? -1 : positions.get((posUpto - 1) * stride + 2);
}
@Override
public boolean hasPayload() {
return false;
}
@Override
public BytesRef getPayload() {
return null;

11
lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
View File

@ -523,14 +523,13 @@ public class RAMOnlyPostingsFormat extends PostingsFormat {
return -1;
}
@Override
public boolean hasPayload() {
return current.payloads != null && current.payloads[posUpto-1] != null;
}
@Override
public BytesRef getPayload() {
return new BytesRef(current.payloads[posUpto-1]);
if (current.payloads != null && current.payloads[posUpto-1] != null) {
return new BytesRef(current.payloads[posUpto-1]);
} else {
return null;
}
}
}

12
lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
View File

@ -365,15 +365,9 @@ public class AssertingAtomicReader extends FilterAtomicReader {
assert state != DocsEnumState.START : "getPayload() called before nextDoc()/advance()";
assert state != DocsEnumState.FINISHED : "getPayload() called after NO_MORE_DOCS";
assert positionCount > 0 : "getPayload() called before nextPosition()!";
return super.getPayload();
}
@Override
public boolean hasPayload() {
assert state != DocsEnumState.START : "hasPayload() called before nextDoc()/advance()";
assert state != DocsEnumState.FINISHED : "hasPayload() called after NO_MORE_DOCS";
assert positionCount > 0 : "hasPayload() called before nextPosition()!";
return super.hasPayload();
BytesRef payload = super.getPayload();
assert payload == null || payload.length > 0 : "getPayload() returned payload with invalid length!";
return payload;
}
}