Add GCJ native code for SegmentTermDocs.read(int[],int[]) to accellerate TermScorer. TermScorer and BooleanScorer are now usually a bit faster under GCJ than under Sun's JVM.

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150531 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Doug Cutting 2004-09-22 18:32:27 +00:00
parent 8ca05a3deb
commit 69d4dc5d06
16 changed files with 241 additions and 83 deletions

View File

@ -8,9 +8,17 @@ DEST=$(BUILD)/gcj
CORE=$(BUILD)/classes/java
SRC=.
CORE_HEADERS=$(CORE)/org/apache/lucene/store/IndexInput.h
GCJ_JAVA=$(wildcard $(SRC)/org/apache/lucene/store/*.java)
GCJ_OBJ=$(DEST)/lucene-gcj.a
CORE_HEADERS=\
$(CORE)/org/apache/lucene/store/IndexInput.h \
$(CORE)/org/apache/lucene/util/BitVector.h \
$(CORE)/org/apache/lucene/index/SegmentTermDocs.h
JAVA=$(wildcard $(SRC)/org/apache/lucene/*/*.java)
JAVA_HEADERS=$(subst $(SRC)/,$(DEST)/,$(subst .java,.h,$(JAVA)))
JAVA_OBJ=$(DEST)/lucene-gcj.a
CNI=$(wildcard $(SRC)/org/apache/lucene/*/*.cc)
CNI_OBJ=$(subst $(SRC)/,$(DEST)/,$(subst .cc,.o,$(CNI)))
CFLAGS ?= -O3 -ffast-math
GCJFLAGS ?= $(CFLAGS) -fno-bounds-check -fno-store-check
@ -36,6 +44,8 @@ $(DEST)/%.class : $(SRC)/%.java
mkdir -p $(dir $@)
gcj -C -I $(CORE) -d $(DEST) $<
.PRECIOUS : $(CORE)/%.h $(DEST)/%.h
$(CORE)/%.h : $(CORE)/%.class
gcjh --classpath=$(CORE) -d $(CORE) \
$(subst /,.,$(subst .class,,$(subst $(CORE)/,,$<)))
@ -44,20 +54,30 @@ $(DEST)/%.h : $(DEST)/%.class
gcjh --classpath=$(DEST) -d $(DEST) \
$(subst /,.,$(subst .class,,$(subst $(DEST)/,,$<)))
$(DEST)/%.o : $(SRC)/%.cc $(DEST)/%.h $(CORE_HEADERS)
$(DEST)/%.cc : $(DEST)/%.class
gcjh -stubs --classpath=$(DEST) -d $(DEST) \
$(subst /,.,$(subst .class,,$(subst $(DEST)/,,$<)))
$(DEST)/%.o : $(SRC)/%.cc $(DEST)/%.h $(CORE_HEADERS) $(JAVA_HEADERS)
g++ $(CFLAGS) -c -I $(CORE) -I $(DEST) -o $@ $<
$(GCJ_OBJ) : $(GCJ_JAVA)
$(DEST)/%.s : $(SRC)/%.cc $(DEST)/%.h $(CORE_HEADERS) $(JAVA_HEADERS)
g++ $(CFLAGS) -S -I $(CORE) -I $(DEST) -o $@ $<
$(JAVA_OBJ) : $(JAVA)
mkdir -p $(dir $@)
gcj $(GCJFLAGS) -c -I $(CORE) -I $(DEST) -o $@ $^
# list of all object code to be linked
OBJ = $(LUCENE_OBJ) $(GCJ_OBJ) $(DEST)/org/apache/lucene/store/GCJIndexInput.o
OBJ = $(LUCENE_OBJ) $(JAVA_OBJ) $(CNI_OBJ)
USE_GCJ_DIRECTORY =\
-Dorg.apache.lucene.FSDirectory.class=org.apache.lucene.store.GCJDirectory
PROPS = $(USE_GCJ_DIRECTORY)
USE_GCJ_SEGMENT_READER =\
-Dorg.apache.lucene.SegmentReader.class=org.apache.lucene.index.GCJSegmentReader
PROPS = $(USE_GCJ_DIRECTORY) $(USE_GCJ_SEGMENT_READER)
# link together various applications
$(BUILD)/indexFiles: $(OBJ)

View File

@ -0,0 +1,32 @@
package org.apache.lucene.index;
/**
* Copyright 2004 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.store.GCJDirectory;
class GCJSegmentReader extends SegmentReader {
public final TermDocs termDocs() throws IOException {
if (directory() instanceof GCJDirectory) {
return new GCJTermDocs(this);
} else {
return super.termDocs();
}
}
}

View File

@ -0,0 +1,56 @@
// This file was created by `gcjh -stubs'. -*- c++ -*-
//
// This file is intended to give you a head start on implementing native
// methods using CNI.
// Be aware: running `gcjh -stubs ' once more for this class may
// overwrite any edits you have made to this file.
#include <org/apache/lucene/index/GCJTermDocs.h>
#include <org/apache/lucene/store/GCJIndexInput.h>
#include <org/apache/lucene/store/IndexInput.h>
#include <org/apache/lucene/util/BitVector.h>
#include <gcj/cni.h>
#include <gcj/array.h>
using namespace ::std;
using namespace ::java::io;
using namespace ::gnu::gcj;
using namespace ::org::apache::lucene::index;
using namespace ::org::apache::lucene::store;
#define RAW(X) reinterpret_cast< RawData*>(X)
#define BYTES(X) reinterpret_cast< jbyte *>(X)
inline jint readVInt(jbyte*& bytes) {
jbyte b = *(bytes++);
jint i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
b = *(bytes++);
i |= (b & 0x7F) << shift;
}
return i;
}
jint GCJTermDocs::read(jintArray docs, jintArray freqs) {
jbyte* input = BYTES(((GCJIndexInput*)freqStream)->pointer);
jint length = docs->length;
jint i = 0;
while (i < length && count < df) {
unsigned int docCode = readVInt(input);
doc__ += docCode >> 1; // shift off low bit
if ((docCode & 1) != 0) // if low bit is set
freq__ = 1; // freq is one
else
freq__ = readVInt(input); // else read freq
count++;
if (deletedDocs == NULL || !deletedDocs->get(doc__)) {
elements(docs)[i] = doc__;
elements(freqs)[i] = freq__;
++i;
}
}
((GCJIndexInput*)freqStream)->pointer = RAW(input);
return i;
}

View File

@ -0,0 +1,28 @@
package org.apache.lucene.index;
/**
* Copyright 2004 The Apache Software Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
class GCJTermDocs extends SegmentTermDocs {
public GCJTermDocs(SegmentReader parent) {
super(parent);
}
public native int read(final int[] docs, final int[] freqs)
throws IOException;
}

View File

@ -28,8 +28,8 @@ public class GCJIndexInput extends IndexInput {
private String file;
private int fd;
private long fileLength;
private RawData data;
private RawData pointer;
public RawData data;
public RawData pointer;
private boolean isClone;
public GCJIndexInput(String file) throws IOException {

View File

@ -53,12 +53,6 @@ public abstract class IndexReader {
*/
protected IndexReader(Directory directory) {
this.directory = directory;
segmentInfos = null;
directoryOwner = false;
closeDirectory = false;
stale = false;
hasChanges = false;
writeLock = null;
}
/**
@ -70,24 +64,25 @@ public abstract class IndexReader {
* @param closeDirectory
*/
IndexReader(Directory directory, SegmentInfos segmentInfos, boolean closeDirectory) {
this.directory = directory;
this.segmentInfos = segmentInfos;
directoryOwner = true;
this.closeDirectory = closeDirectory;
stale = false;
hasChanges = false;
writeLock = null;
init(directory, segmentInfos, closeDirectory, true);
}
final private Directory directory;
final private boolean directoryOwner;
final private SegmentInfos segmentInfos;
void init(Directory directory, SegmentInfos segmentInfos, boolean closeDirectory, boolean directoryOwner) {
this.directory = directory;
this.segmentInfos = segmentInfos;
this.directoryOwner = directoryOwner;
this.closeDirectory = closeDirectory;
}
private Directory directory;
private boolean directoryOwner;
private boolean closeDirectory;
private SegmentInfos segmentInfos;
private Lock writeLock;
private boolean stale;
private boolean hasChanges;
final private boolean closeDirectory;
/** Returns an IndexReader reading the index in an FSDirectory in the named
path. */
@ -115,11 +110,11 @@ public abstract class IndexReader {
SegmentInfos infos = new SegmentInfos();
infos.read(directory);
if (infos.size() == 1) { // index is optimized
return new SegmentReader(infos, infos.info(0), closeDirectory);
return SegmentReader.get(infos, infos.info(0), closeDirectory);
} else {
IndexReader[] readers = new IndexReader[infos.size()];
for (int i = 0; i < infos.size(); i++)
readers[i] = new SegmentReader(infos.info(i));
readers[i] = SegmentReader.get(infos.info(i));
return new MultiReader(directory, infos, closeDirectory, readers);
}
}

View File

@ -406,7 +406,7 @@ public class IndexWriter {
final Vector segmentsToDelete = new Vector();
IndexReader sReader = null;
if (segmentInfos.size() == 1){ // add existing index, if any
sReader = new SegmentReader(segmentInfos.info(0));
sReader = SegmentReader.get(segmentInfos.info(0));
merger.add(sReader);
segmentsToDelete.addElement(sReader); // queue segment for deletion
}
@ -503,7 +503,7 @@ public class IndexWriter {
SegmentInfo si = segmentInfos.info(i);
if (infoStream != null)
infoStream.print(" " + si.name + " (" + si.docCount + " docs)");
IndexReader reader = new SegmentReader(si);
IndexReader reader = SegmentReader.get(si);
merger.add(reader);
if ((reader.directory() == this.directory) || // if we own the directory
(reader.directory() == this.ramDirectory))

View File

@ -35,7 +35,7 @@ import org.apache.lucene.util.BitVector;
*
* @version $Id$
*/
final class SegmentReader extends IndexReader {
class SegmentReader extends IndexReader {
private String segment;
FieldInfos fieldInfos;
@ -89,19 +89,46 @@ final class SegmentReader extends IndexReader {
private Hashtable norms = new Hashtable();
SegmentReader(SegmentInfos sis, SegmentInfo si, boolean closeDir)
throws IOException {
super(si.dir, sis, closeDir);
initialize(si);
/** The class which implements SegmentReader. */
private static final Class IMPL;
static {
try {
String name =
System.getProperty("org.apache.lucene.SegmentReader.class",
SegmentReader.class.getName());
IMPL = Class.forName(name);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
SegmentReader(SegmentInfo si) throws IOException {
super(si.dir);
initialize(si);
protected SegmentReader() { super(null); }
public static SegmentReader get(SegmentInfo si) throws IOException {
return get(si.dir, si, null, false, false);
}
public static SegmentReader get(SegmentInfos sis, SegmentInfo si,
boolean closeDir) throws IOException {
return get(si.dir, si, sis, closeDir, true);
}
public static SegmentReader get(Directory dir, SegmentInfo si,
SegmentInfos sis,
boolean closeDir, boolean ownDir)
throws IOException {
SegmentReader instance;
try {
instance = (SegmentReader)IMPL.newInstance();
} catch (Exception e) {
throw new RuntimeException(e);
}
instance.init(dir, sis, closeDir, ownDir);
instance.initialize(si);
return instance;
}
private void initialize(SegmentInfo si) throws IOException
{
private void initialize(SegmentInfo si) throws IOException {
segment = si.name;
// Use compound file directory for some files, if it exists
@ -132,7 +159,7 @@ final class SegmentReader extends IndexReader {
}
}
protected final void doCommit() throws IOException {
protected void doCommit() throws IOException {
if (deletedDocsDirty) { // re-write deleted
deletedDocs.write(directory(), segment + ".tmp");
directory().renameFile(segment + ".tmp", segment + ".del");
@ -154,7 +181,7 @@ final class SegmentReader extends IndexReader {
undeleteAll = false;
}
protected final void doClose() throws IOException {
protected void doClose() throws IOException {
fieldsReader.close();
tis.close();
@ -170,7 +197,7 @@ final class SegmentReader extends IndexReader {
cfsReader.close();
}
static final boolean hasDeletions(SegmentInfo si) throws IOException {
static boolean hasDeletions(SegmentInfo si) throws IOException {
return si.dir.fileExists(si.name + ".del");
}
@ -179,11 +206,11 @@ final class SegmentReader extends IndexReader {
}
static final boolean usesCompoundFile(SegmentInfo si) throws IOException {
static boolean usesCompoundFile(SegmentInfo si) throws IOException {
return si.dir.fileExists(si.name + ".cfs");
}
static final boolean hasSeparateNorms(SegmentInfo si) throws IOException {
static boolean hasSeparateNorms(SegmentInfo si) throws IOException {
String[] result = si.dir.list();
String pattern = si.name + ".s";
int patternLength = pattern.length();
@ -194,7 +221,7 @@ final class SegmentReader extends IndexReader {
return false;
}
protected final void doDelete(int docNum) {
protected void doDelete(int docNum) {
if (deletedDocs == null)
deletedDocs = new BitVector(maxDoc());
deletedDocsDirty = true;
@ -202,13 +229,13 @@ final class SegmentReader extends IndexReader {
deletedDocs.set(docNum);
}
protected final void doUndeleteAll() {
protected void doUndeleteAll() {
deletedDocs = null;
deletedDocsDirty = false;
undeleteAll = true;
}
final Vector files() throws IOException {
Vector files() throws IOException {
Vector files = new Vector(16);
final String ext[] = new String[]{
"cfs", "fnm", "fdx", "fdt", "tii", "tis", "frq", "prx", "del",
@ -235,34 +262,34 @@ final class SegmentReader extends IndexReader {
return files;
}
public final TermEnum terms() {
public TermEnum terms() {
return tis.terms();
}
public final TermEnum terms(Term t) throws IOException {
public TermEnum terms(Term t) throws IOException {
return tis.terms(t);
}
public final synchronized Document document(int n) throws IOException {
public synchronized Document document(int n) throws IOException {
if (isDeleted(n))
throw new IllegalArgumentException
("attempt to access a deleted document");
return fieldsReader.doc(n);
}
public final synchronized boolean isDeleted(int n) {
public synchronized boolean isDeleted(int n) {
return (deletedDocs != null && deletedDocs.get(n));
}
public final TermDocs termDocs() throws IOException {
public TermDocs termDocs() throws IOException {
return new SegmentTermDocs(this);
}
public final TermPositions termPositions() throws IOException {
public TermPositions termPositions() throws IOException {
return new SegmentTermPositions(this);
}
public final int docFreq(Term t) throws IOException {
public int docFreq(Term t) throws IOException {
TermInfo ti = tis.get(t);
if (ti != null)
return ti.docFreq;
@ -270,14 +297,14 @@ final class SegmentReader extends IndexReader {
return 0;
}
public final int numDocs() {
public int numDocs() {
int n = maxDoc();
if (deletedDocs != null)
n -= deletedDocs.count();
return n;
}
public final int maxDoc() {
public int maxDoc() {
return fieldsReader.size();
}
@ -339,7 +366,7 @@ final class SegmentReader extends IndexReader {
return norm.bytes;
}
protected final void doSetNorm(int doc, String field, byte value)
protected void doSetNorm(int doc, String field, byte value)
throws IOException {
Norm norm = (Norm) norms.get(field);
if (norm == null) // not an indexed field
@ -372,7 +399,7 @@ final class SegmentReader extends IndexReader {
}
}
private final void openNorms(Directory cfsDir) throws IOException {
private void openNorms(Directory cfsDir) throws IOException {
for (int i = 0; i < fieldInfos.size(); i++) {
FieldInfo fi = fieldInfos.fieldInfo(i);
if (fi.isIndexed) {
@ -388,7 +415,7 @@ final class SegmentReader extends IndexReader {
}
}
private final void closeNorms() throws IOException {
private void closeNorms() throws IOException {
synchronized (norms) {
Enumeration enumerator = norms.elements();
while (enumerator.hasMoreElements()) {

View File

@ -22,10 +22,10 @@ import org.apache.lucene.store.IndexInput;
class SegmentTermDocs implements TermDocs {
protected SegmentReader parent;
private IndexInput freqStream;
private int count;
private int df;
private BitVector deletedDocs;
protected IndexInput freqStream;
protected int count;
protected int df;
protected BitVector deletedDocs;
int doc = 0;
int freq;
@ -39,7 +39,7 @@ class SegmentTermDocs implements TermDocs {
private long skipPointer;
private boolean haveSkipped;
SegmentTermDocs(SegmentReader parent) {
protected SegmentTermDocs(SegmentReader parent) {
this.parent = parent;
this.freqStream = (IndexInput) parent.freqStream.clone();
this.deletedDocs = parent.deletedDocs;

View File

@ -73,8 +73,8 @@ class DocTest {
throws Exception {
Directory directory = FSDirectory.getDirectory("test", false);
SegmentReader r1 = new SegmentReader(new SegmentInfo(seg1, 1, directory));
SegmentReader r2 = new SegmentReader(new SegmentInfo(seg2, 1, directory));
SegmentReader r1 = SegmentReader.get(new SegmentInfo(seg1, 1, directory));
SegmentReader r2 = SegmentReader.get(new SegmentInfo(seg2, 1, directory));
SegmentMerger merger = new SegmentMerger(directory, merged);
merger.add(r1);
@ -89,7 +89,7 @@ class DocTest {
throws Exception {
Directory directory = FSDirectory.getDirectory("test", false);
SegmentReader reader =
new SegmentReader(new SegmentInfo(segment, 1, directory));
SegmentReader.get(new SegmentInfo(segment, 1, directory));
for (int i = 0; i < reader.numDocs(); i++)
System.out.println(reader.document(i));

View File

@ -177,8 +177,8 @@ public class TestDoc extends TestCase {
throws Exception {
Directory directory = FSDirectory.getDirectory(indexDir, false);
SegmentReader r1 = new SegmentReader(new SegmentInfo(seg1, 1, directory));
SegmentReader r2 = new SegmentReader(new SegmentInfo(seg2, 1, directory));
SegmentReader r1 = SegmentReader.get(new SegmentInfo(seg1, 1, directory));
SegmentReader r2 = SegmentReader.get(new SegmentInfo(seg2, 1, directory));
SegmentMerger merger =
new SegmentMerger(directory, merged);
@ -202,7 +202,7 @@ public class TestDoc extends TestCase {
throws Exception {
Directory directory = FSDirectory.getDirectory(indexDir, false);
SegmentReader reader =
new SegmentReader(new SegmentInfo(segment, 1, directory));
SegmentReader.get(new SegmentInfo(segment, 1, directory));
for (int i = 0; i < reader.numDocs(); i++)
out.println(reader.document(i));

View File

@ -56,7 +56,7 @@ public class TestDocumentWriter extends TestCase {
try {
writer.addDocument("test", testDoc);
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(new SegmentInfo("test", 1, dir));
SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 1, dir));
assertTrue(reader != null);
Document doc = reader.document(0);
assertTrue(doc != null);

View File

@ -44,8 +44,8 @@ public class TestMultiReader extends TestCase {
try {
sis.write(dir);
reader1 = new SegmentReader(new SegmentInfo("seg-1", 1, dir));
reader2 = new SegmentReader(new SegmentInfo("seg-2", 1, dir));
reader1 = SegmentReader.get(new SegmentInfo("seg-1", 1, dir));
reader2 = SegmentReader.get(new SegmentInfo("seg-2", 1, dir));
readers[0] = reader1;
readers[1] = reader2;
} catch (IOException e) {

View File

@ -50,8 +50,8 @@ public class TestSegmentMerger extends TestCase {
DocHelper.setupDoc(doc2);
DocHelper.writeDoc(merge2Dir, merge2Segment, doc2);
try {
reader1 = new SegmentReader(new SegmentInfo(merge1Segment, 1, merge1Dir));
reader2 = new SegmentReader(new SegmentInfo(merge2Segment, 1, merge2Dir));
reader1 = SegmentReader.get(new SegmentInfo(merge1Segment, 1, merge1Dir));
reader2 = SegmentReader.get(new SegmentInfo(merge2Segment, 1, merge2Dir));
} catch (IOException e) {
e.printStackTrace();
}
@ -80,7 +80,7 @@ public class TestSegmentMerger extends TestCase {
merger.closeReaders();
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
SegmentReader mergedReader = new SegmentReader(new SegmentInfo(mergedSegment, docsMerged, mergedDir));
SegmentReader mergedReader = SegmentReader.get(new SegmentInfo(mergedSegment, docsMerged, mergedDir));
assertTrue(mergedReader != null);
assertTrue(mergedReader.numDocs() == 2);
Document newDoc1 = mergedReader.document(0);

View File

@ -41,7 +41,7 @@ public class TestSegmentReader extends TestCase {
try {
DocHelper.setupDoc(testDoc);
DocHelper.writeDoc(dir, testDoc);
reader = new SegmentReader(new SegmentInfo("test", 1, dir));
reader = SegmentReader.get(new SegmentInfo("test", 1, dir));
} catch (IOException e) {
}
@ -84,7 +84,7 @@ public class TestSegmentReader extends TestCase {
DocHelper.setupDoc(docToDelete);
DocHelper.writeDoc(dir, "seg-to-delete", docToDelete);
try {
SegmentReader deleteReader = new SegmentReader(new SegmentInfo("seg-to-delete", 1, dir));
SegmentReader deleteReader = SegmentReader.get(new SegmentInfo("seg-to-delete", 1, dir));
assertTrue(deleteReader != null);
assertTrue(deleteReader.numDocs() == 1);
deleteReader.delete(0);

View File

@ -50,7 +50,7 @@ public class TestSegmentTermDocs extends TestCase {
public void testTermDocs() {
try {
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(new SegmentInfo("test", 1, dir));
SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 1, dir));
assertTrue(reader != null);
SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
assertTrue(segTermDocs != null);
@ -71,7 +71,7 @@ public class TestSegmentTermDocs extends TestCase {
public void testBadSeek() {
try {
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(new SegmentInfo("test", 3, dir));
SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 3, dir));
assertTrue(reader != null);
SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
assertTrue(segTermDocs != null);
@ -83,7 +83,7 @@ public class TestSegmentTermDocs extends TestCase {
}
try {
//After adding the document, we should be able to read it back in
SegmentReader reader = new SegmentReader(new SegmentInfo("test", 3, dir));
SegmentReader reader = SegmentReader.get(new SegmentInfo("test", 3, dir));
assertTrue(reader != null);
SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
assertTrue(segTermDocs != null);