SOLR-998: current lucene libs, SolrIndexReader introduction for FileFloatSource fix

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@743196 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -297,7 +297,10 @@ Other Changes
 
 16. SOLR-959: Refactored TestReplicationHandler to remove hardcoded port numbers (hossman, Akshay Ukey via shalin)
 
+17. Upgraded to Lucene 2.9-dev r742220 (yonik)
+
 Build
+
 ----------------------
  1. SOLR-776: Added in ability to sign artifacts via Ant for releases (gsingers)
 

lib/lucene-analyzers-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[a5d31a9409eaf23aa468d310c2efea973ff0b1c2] was removed in git history.
+AnyObjectId[233ed5db218f6903ac3c0ece661252d6407064b6] was removed in git history.
 Apache SVN contains full history.

lib/lucene-core-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[aff681d711db092c4a0223a4c83ed54bf2f1fc65] was removed in git history.
+AnyObjectId[31b0d33aa1da2bf13d6bf51e9be394a33b1ab0a3] was removed in git history.
 Apache SVN contains full history.

lib/lucene-highlighter-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[ec07b9376ad892ca3c65fcce6d4fe39e5842f287] was removed in git history.
+AnyObjectId[038a5f87ce9318fce477e7462e9b42453906061c] was removed in git history.
 Apache SVN contains full history.

lib/lucene-memory-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[252b5fc02fe9c59f5333284c4d31b617d0917833] was removed in git history.
+AnyObjectId[42c58436c087c6f1065ab4bdc595672a7e53c663] was removed in git history.
 Apache SVN contains full history.

lib/lucene-queries-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[3e54e93a41abdff89af393081f67b6675b7125ce] was removed in git history.
+AnyObjectId[65a1349b447909deaef0f3dc1a08d85382151633] was removed in git history.
 Apache SVN contains full history.

lib/lucene-snowball-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[461d1b7f1fc8850ef9bbde6e987866282a514db0] was removed in git history.
+AnyObjectId[ec5b95ed8d08e88804da07a13e954670e58b3afe] was removed in git history.
 Apache SVN contains full history.

lib/lucene-spellchecker-2.9-dev.jar

@@ -1,2 +1,2 @@
-AnyObjectId[582f579a54e931e0aab5900d6a1573633e105903] was removed in git history.
+AnyObjectId[62c06f36f735835d7effebce4ca377171d61f743] was removed in git history.
 Apache SVN contains full history.

src/java/org/apache/solr/search/SolrIndexReader.java

@@ -0,0 +1,395 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+
+import org.apache.lucene.index.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+
+/** Solr wrapper for IndexReader that contains extra context.
+ * This is currently experimental, for internal use only, and subject to change.
+ */
+public class SolrIndexReader extends FilterIndexReader {
+  private final SolrIndexReader[] subReaders;
+  private final SolrIndexReader parent;
+  private final int base; // docid offset of this reader within parent
+
+  // top level searcher for this reader tree
+  // a bit if a hack currently... searcher needs to set
+  SolrIndexSearcher searcher;
+
+  // Shared info about the wrapped reader.
+  private SolrReaderInfo info;
+
+  /** Recursively wrap an IndexReader in SolrIndexReader instances.
+   * @param in  the reader to wrap
+   * @param parent the parent, if any (null if none)
+   * @param base the docid offset in the parent (0 if top level)
+   */
+  public SolrIndexReader(IndexReader in, SolrIndexReader parent, int base) {
+    super(in);
+    assert(!(in instanceof SolrIndexReader));
+    this.parent = parent;
+    this.base = base;
+    IndexReader subs[] = in.getSequentialSubReaders();
+    subReaders = subs == null ? null : new SolrIndexReader[subs.length];
+    if (subs != null) {
+      int b=0;
+      for (int i=0; i<subReaders.length; i++) {
+        subReaders[i] = new SolrIndexReader(subs[i], this, b);
+        b += subReaders[i].maxDoc();
+      }
+    }
+  }
+
+  static String shortName(Object o) {
+    return o.getClass().getSimpleName()+ "@" + Integer.toHexString(o.hashCode());
+  }
+
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append("SolrIndexReader{this=").append(Integer.toHexString(this.hashCode()));
+    sb.append(",r=").append(shortName(in));
+    sb.append(",segments=");
+    sb.append(subReaders == null ? 1 : subReaders.length);
+    if (parent != null) {
+      sb.append(",parent=").append(parent.toString());
+    }
+    sb.append('}');
+    return sb.toString();
+  }
+
+  static void setSearcher(SolrIndexReader sr, SolrIndexSearcher searcher) {
+    sr.searcher = searcher;
+    SolrIndexReader[] readers = sr.getSequentialSubReaders();
+    if (readers == null) return;
+    for (SolrIndexReader r : readers) {
+      setSearcher(r, searcher);
+    }
+  }
+
+   private static void buildInfoMap(SolrIndexReader other, HashMap<IndexReader, SolrReaderInfo> map) {
+     if (other == null) return;
+     map.put(other.getWrappedReader(), other.info);
+     SolrIndexReader[] readers = other.getSequentialSubReaders();
+     if (readers == null) return;
+     for (SolrIndexReader r : readers) {
+       buildInfoMap(r, map);
+     }     
+   }
+
+   private static void setInfo(SolrIndexReader target, HashMap<IndexReader, SolrReaderInfo> map) {
+     SolrReaderInfo info = map.get(target.getWrappedReader());
+     if (info == null) info = new SolrReaderInfo(target.getWrappedReader());
+     target.info = info;
+     SolrIndexReader[] readers = target.getSequentialSubReaders();
+     if (readers == null) return;
+     for (SolrIndexReader r : readers) {
+       setInfo(r, map);
+     }     
+   }
+
+   /** Copies SolrReaderInfo instances from the source to this SolrIndexReader */
+   public void associateInfo(SolrIndexReader source) {
+     // seemed safer to not mess with reopen() but simply set
+     // one set of caches from another reader tree.
+     HashMap<IndexReader, SolrReaderInfo> map = new HashMap<IndexReader, SolrReaderInfo>();
+     buildInfoMap(source, map);
+     setInfo(this, map);
+   }
+
+  public IndexReader getWrappedReader() {
+    return in;
+  }
+
+  /** returns the parent reader, or null of none */
+  public SolrIndexReader getParent() {
+    return parent;
+  }
+
+   /** returns the docid offset within the parent reader */
+  public int getBase() {
+    return base;
+  }
+
+  @Override
+  public Directory directory() {
+    return in.directory();
+  }
+
+  @Override
+  public TermFreqVector[] getTermFreqVectors(int docNumber) throws IOException {
+    return in.getTermFreqVectors(docNumber);
+  }
+
+  @Override
+  public TermFreqVector getTermFreqVector(int docNumber, String field)
+          throws IOException {
+    return in.getTermFreqVector(docNumber, field);
+  }
+
+  @Override
+  public void getTermFreqVector(int docNumber, String field, TermVectorMapper mapper) throws IOException {
+    in.getTermFreqVector(docNumber, field, mapper);
+
+  }
+
+  @Override
+  public void getTermFreqVector(int docNumber, TermVectorMapper mapper) throws IOException {
+    in.getTermFreqVector(docNumber, mapper);
+  }
+
+  @Override
+  public int numDocs() {
+    return in.numDocs();
+  }
+
+  @Override
+  public int maxDoc() {
+    return in.maxDoc();
+  }
+
+  @Override
+  public Document document(int n, FieldSelector fieldSelector) throws CorruptIndexException, IOException {
+    return in.document(n, fieldSelector);
+  }
+
+  @Override
+  public boolean isDeleted(int n) {
+    return in.isDeleted(n);
+  }
+
+  @Override
+  public boolean hasDeletions() {
+    return in.hasDeletions();
+  }
+
+  @Override
+  protected void doUndeleteAll() throws CorruptIndexException, IOException {in.undeleteAll();}
+
+  @Override
+  public boolean hasNorms(String field) throws IOException {
+    return in.hasNorms(field);
+  }
+
+  @Override
+  public byte[] norms(String f) throws IOException {
+    return in.norms(f);
+  }
+
+  @Override
+  public void norms(String f, byte[] bytes, int offset) throws IOException {
+    in.norms(f, bytes, offset);
+  }
+
+  @Override
+  protected void doSetNorm(int d, String f, byte b) throws CorruptIndexException, IOException {
+    in.setNorm(d, f, b);
+  }
+
+  @Override
+  public TermEnum terms() throws IOException {
+    return in.terms();
+  }
+
+  @Override
+  public TermEnum terms(Term t) throws IOException {
+    return in.terms(t);
+  }
+
+  @Override
+  public int docFreq(Term t) throws IOException {
+    ensureOpen();
+    return in.docFreq(t);
+  }
+
+  @Override
+  public TermDocs termDocs() throws IOException {
+    ensureOpen();
+    return in.termDocs();
+  }
+
+  @Override
+  public TermDocs termDocs(Term term) throws IOException {
+    ensureOpen();
+    return in.termDocs(term);
+  }
+
+  @Override
+  public TermPositions termPositions() throws IOException {
+    ensureOpen();
+    return in.termPositions();
+  }
+
+  @Override
+  protected void doDelete(int n) throws  CorruptIndexException, IOException { in.deleteDocument(n); }
+
+  // Let FilterIndexReader handle commit()... we cannot override commit()
+  // or call in.commit() ourselves.
+  // protected void doCommit() throws IOException { in.commit(); }
+
+  @Override
+  protected void doClose() throws IOException { in.close(); }
+
+  @Override
+  public Collection getFieldNames(IndexReader.FieldOption fieldNames) {
+    return in.getFieldNames(fieldNames);
+  }
+
+  @Override
+  public long getVersion() {
+    return in.getVersion();
+  }
+
+  @Override
+  public boolean isCurrent() throws CorruptIndexException, IOException {
+    return in.isCurrent();
+  }
+
+  @Override
+  public boolean isOptimized() {
+    return in.isOptimized();
+  }
+
+  @Override
+  public SolrIndexReader[] getSequentialSubReaders() {
+    return subReaders;
+  }
+
+  @Override
+  public int hashCode() {
+    return in.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (o instanceof SolrIndexReader) {
+      o = ((SolrIndexReader)o).in;
+    }
+    return in.equals(o);
+  }
+
+  @Override
+  public SolrIndexReader reopen(boolean openReadOnly) throws IOException {
+    IndexReader r = in.reopen(openReadOnly);
+    if (r == in) {
+      return this;
+    }
+    SolrIndexReader sr = new SolrIndexReader(r, null, 0);
+    sr.associateInfo(this);
+    return sr;
+  }
+
+  @Override
+  public SolrIndexReader reopen() throws CorruptIndexException, IOException {
+    return reopen(true);
+  }
+
+  @Override
+  public void decRef() throws IOException {
+    in.decRef();
+  }
+
+  @Override
+  public void deleteDocument(int docNum) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    in.deleteDocument(docNum);
+  }
+
+  @Override
+  public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    return in.deleteDocuments(term);
+  }
+
+  @Override
+  public Document document(int n) throws CorruptIndexException, IOException {
+    return in.document(n);
+  }
+
+  @Override
+  public String getCommitUserData() {
+    return in.getCommitUserData();
+  }
+
+  @Override
+  public IndexCommit getIndexCommit() throws IOException {
+    return in.getIndexCommit();
+  }
+
+  @Override
+  public int getTermInfosIndexDivisor() {
+    return in.getTermInfosIndexDivisor();
+  }
+
+  @Override
+  public void incRef() {
+    in.incRef();
+  }
+
+  @Override
+  public int numDeletedDocs() {
+    return in.numDeletedDocs();
+  }
+
+  @Override
+  public void setNorm(int doc, String field, byte value) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    in.setNorm(doc, field, value);
+  }
+
+  @Override
+  public void setNorm(int doc, String field, float value) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    in.setNorm(doc, field, value);
+  }
+
+  @Override
+  public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException {
+    in.setTermInfosIndexDivisor(indexDivisor);
+  }
+
+  @Override
+  public TermPositions termPositions(Term term) throws IOException {
+    return in.termPositions(term);
+  }
+
+  @Override
+  public void undeleteAll() throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
+    in.undeleteAll();
+  }
+}
+
+
+
+/** SolrReaderInfo contains information that is the same for
+ * every SolrIndexReader that wraps the same IndexReader.
+ * Multiple SolrIndexReader instances will be accessing this
+ * class concurrently.
+ */
+class SolrReaderInfo {
+  private final IndexReader reader;
+  public SolrReaderInfo(IndexReader reader) {
+    this.reader = reader;
+  }
+  public IndexReader getReader() { return reader; }
+
+}

src/java/org/apache/solr/search/SolrIndexSearcher.java

@@ -67,7 +67,7 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
   private long registerTime = 0;
   private long warmupTime = 0;
   private final IndexSearcher searcher;
-  private final IndexReader reader;
+  private final SolrIndexReader reader;
   private final boolean closeReader;
 
   private final int queryResultWindowSize;
@@ -124,16 +124,23 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
     this.core = core;
     this.schema = schema;
     this.name = "Searcher@" + Integer.toHexString(hashCode()) + (name!=null ? " "+name : "");
-
     log.info("Opening " + this.name);
 
+    // wrap the reader
+    if (!(r instanceof SolrIndexReader)) {
+      reader = new SolrIndexReader(r, null, 0);
+      reader.associateInfo(null);
+    } else {
+      reader = (SolrIndexReader)r;
+    }
+    SolrIndexReader.setSearcher(reader, this);
+
     if (r.directory() instanceof FSDirectory) {
       FSDirectory fsDirectory = (FSDirectory) r.directory();
       indexDir = fsDirectory.getFile().getAbsolutePath();
     }
 
-    reader = r;
-    searcher = new IndexSearcher(r);
+    searcher = new IndexSearcher(reader);
     this.closeReader = closeReader;
     searcher.setSimilarity(schema.getSimilarity());
 
@@ -236,7 +243,7 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
   }
 
   /** Direct access to the IndexReader used by this searcher */
-  public IndexReader getReader() { return reader; }
+  public SolrIndexReader getReader() { return reader; }
   /** Direct access to the IndexSchema for use with this searcher */
   public IndexSchema getSchema() { return schema; }
   
@@ -1707,7 +1714,7 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
     lst.add("caching", cachingEnabled);
     lst.add("numDocs", reader.numDocs());
     lst.add("maxDoc", reader.maxDoc());
-    lst.add("readerImpl", reader.getClass().getSimpleName());
+    lst.add("reader", reader.toString());
     lst.add("readerDir", reader.directory());
     lst.add("indexVersion", reader.getVersion());
     lst.add("openedAt", new Date(openTime));

src/java/org/apache/solr/search/function/FileFloatSource.java

@@ -24,6 +24,7 @@ import org.apache.solr.core.SolrCore;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.schema.FieldType;
 import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrIndexReader;
 import org.apache.solr.util.VersionedFile;
 
 import java.io.*;
@@ -53,26 +54,37 @@ public class FileFloatSource extends ValueSource {
   }
 
   public DocValues getValues(IndexReader reader) throws IOException {
+    int offset = 0;
+    if (reader instanceof SolrIndexReader) {
+      SolrIndexReader r = (SolrIndexReader)reader;
+      while (r.getParent() != null) {
+        offset += r.getBase();
+        r = r.getParent();
+      }
+      reader = r;
+    }
+    final int off = offset;
+
     final float[] arr = getCachedFloats(reader);
     return new DocValues() {
       public float floatVal(int doc) {
-        return arr[doc];
+        return arr[doc + off];
       }
 
       public int intVal(int doc) {
-        return (int)arr[doc];
+        return (int)arr[doc + off];
       }
 
       public long longVal(int doc) {
-        return (long)arr[doc];
+        return (long)arr[doc + off];
       }
 
       public double doubleVal(int doc) {
-        return (double)arr[doc];
+        return (double)arr[doc + off];
       }
 
       public String strVal(int doc) {
-        return Float.toString(arr[doc]);
+        return Float.toString(arr[doc + off]);
       }
 
       public String toString(int doc) {