From 06434e5b5f8df05b37a2b795bd8f4ccd13103a10 Mon Sep 17 00:00:00 2001
From: Shai Erera <shaie@apache.org>
Date: Thu, 30 Jan 2014 13:12:10 +0000
Subject: [PATCH] LUCENE-5320: Add SearcherTaxonomyManager over Directory

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1562806 13f79535-47bb-0310-9956-ffa450edef68
---
 lucene/CHANGES.txt                            |   3 +
 .../taxonomy/SearcherTaxonomyManager.java     |  27 ++-
 .../taxonomy/TestSearcherTaxonomyManager.java | 209 ++++++++++++++----
 3 files changed, 190 insertions(+), 49 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2549d8e25f7..14c5abbdb2e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -122,6 +122,9 @@ New Features
 * LUCENE-5353: ShingleFilter's filler token should be configurable.
   (Ahmet Arslan, Simon Willnauer, Steve Rowe)
 
+* LUCENE-5320: Add SearcherTaxonomyManager over search and taxonomy index
+  directories (i.e. not only NRT). (Shai Erera)
+
 Build
 
 * LUCENE-5217,LUCENE-5420: Maven config: get dependencies from Ant+Ivy config;
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
index c4169ee1434..6de430c758c 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/SearcherTaxonomyManager.java
@@ -28,6 +28,7 @@ import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.search.SearcherManager;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 
 /**
@@ -69,9 +70,27 @@ public class SearcherTaxonomyManager extends ReferenceManager<SearcherTaxonomyMa
     this.searcherFactory = searcherFactory;
     this.taxoWriter = taxoWriter;
     DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);
-    current = new SearcherAndTaxonomy(SearcherManager.getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes)),
-                                      taxoReader);
-    taxoEpoch = taxoWriter.getTaxonomyEpoch();
+    current = new SearcherAndTaxonomy(SearcherManager.getSearcher(searcherFactory, DirectoryReader.open(writer, applyAllDeletes)), taxoReader);
+    this.taxoEpoch = taxoWriter.getTaxonomyEpoch();
+  }
+
+  /**
+   * Creates search and taxonomy readers over the corresponding directories.
+   * 
+   * <p>
+   * <b>NOTE:</b> you should only use this constructor if you commit and call
+   * {@link #maybeRefresh()} in the same thread. Otherwise it could lead to an
+   * unsync'd {@link IndexSearcher} and {@link TaxonomyReader} pair.
+   */
+  public SearcherTaxonomyManager(Directory indexDir, Directory taxoDir, SearcherFactory searcherFactory) throws IOException {
+    if (searcherFactory == null) {
+      searcherFactory = new SearcherFactory();
+    }
+    this.searcherFactory = searcherFactory;
+    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
+    current = new SearcherAndTaxonomy(SearcherManager.getSearcher(searcherFactory, DirectoryReader.open(indexDir)), taxoReader);
+    this.taxoWriter = null;
+    taxoEpoch = -1;
   }
 
   @Override
@@ -114,7 +133,7 @@ public class SearcherTaxonomyManager extends ReferenceManager<SearcherTaxonomyMa
       if (tr == null) {
         ref.taxonomyReader.incRef();
         tr = ref.taxonomyReader;
-      } else if (taxoWriter.getTaxonomyEpoch() != taxoEpoch) {
+      } else if (taxoWriter != null && taxoWriter.getTaxonomyEpoch() != taxoEpoch) {
         IOUtils.close(newReader, tr);
         throw new IllegalStateException("DirectoryTaxonomyWriter.replaceTaxonomy was called, which is not allowed when using SearcherTaxonomyManager");
       }
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
index f14dfde4e58..54bcd891e17 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestSearcherTaxonomyManager.java
@@ -37,12 +37,82 @@ import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager.SearcherAndTaxon
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util._TestUtil;
 
 public class TestSearcherTaxonomyManager extends FacetTestCase {
-  public void test() throws Exception {
+
+  private static class IndexerThread extends Thread {
+    
+    private IndexWriter w;
+    private FacetsConfig config;
+    private TaxonomyWriter tw;
+    private ReferenceManager<SearcherAndTaxonomy> mgr;
+    private int ordLimit;
+    private AtomicBoolean stop;
+
+    public IndexerThread(IndexWriter w, FacetsConfig config, TaxonomyWriter tw,
+        ReferenceManager<SearcherAndTaxonomy> mgr, int ordLimit, AtomicBoolean stop) {
+      this.w = w;
+      this.config = config;
+      this.tw = tw;
+      this.mgr = mgr;
+      this.ordLimit = ordLimit;
+      this.stop = stop;
+    }
+
+    @Override
+    public void run() {
+      try {
+        Set<String> seen = new HashSet<String>();
+        List<String> paths = new ArrayList<String>();
+        while (true) {
+          Document doc = new Document();
+          int numPaths = _TestUtil.nextInt(random(), 1, 5);
+          for(int i=0;i<numPaths;i++) {
+            String path;
+            if (!paths.isEmpty() && random().nextInt(5) != 4) {
+              // Use previous path
+              path = paths.get(random().nextInt(paths.size()));
+            } else {
+              // Create new path
+              path = null;
+              while (true) {
+                path = _TestUtil.randomRealisticUnicodeString(random());
+                if (path.length() != 0 && !seen.contains(path)) {
+                  seen.add(path);
+                  paths.add(path);
+                  break;
+                }
+              }
+            }
+            doc.add(new FacetField("field", path));
+          }
+          try {
+            w.addDocument(config.build(tw, doc));
+            if (mgr != null && random().nextDouble() < 0.1) {
+              w.commit();
+              tw.commit();
+              mgr.maybeRefresh();
+            }
+          } catch (IOException ioe) {
+            throw new RuntimeException(ioe);
+          }
+          
+          if (tw.getSize() >= ordLimit) {
+            break;
+          }
+        }
+      } finally {
+        stop.set(true);
+      }
+    }
+
+  }
+
+  public void testNRT() throws Exception {
     Directory dir = newDirectory();
     Directory taxoDir = newDirectory();
     final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -54,49 +124,7 @@ public class TestSearcherTaxonomyManager extends FacetTestCase {
     // How many unique facets to index before stopping:
     final int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
 
-    Thread indexer = new Thread() {
-        @Override
-        public void run() {
-          try {
-            Set<String> seen = new HashSet<String>();
-            List<String> paths = new ArrayList<String>();
-            while (true) {
-              Document doc = new Document();
-              int numPaths = _TestUtil.nextInt(random(), 1, 5);
-              for(int i=0;i<numPaths;i++) {
-                String path;
-                if (!paths.isEmpty() && random().nextInt(5) != 4) {
-                  // Use previous path
-                  path = paths.get(random().nextInt(paths.size()));
-                } else {
-                  // Create new path
-                  path = null;
-                  while (true) {
-                    path = _TestUtil.randomRealisticUnicodeString(random());
-                    if (path.length() != 0 && !seen.contains(path)) {
-                      seen.add(path);
-                      paths.add(path);
-                      break;
-                    }
-                  }
-                }
-                doc.add(new FacetField("field", path));
-              }
-              try {
-                w.addDocument(config.build(tw, doc));
-              } catch (IOException ioe) {
-                throw new RuntimeException(ioe);
-              }
-
-              if (tw.getSize() >= ordLimit) {
-                break;
-              }
-            }
-          } finally {
-            stop.set(true);
-          }
-        }
-      };
+    Thread indexer = new IndexerThread(w, config, tw, null, ordLimit, stop);
 
     final SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(w, true, null, tw);
 
@@ -160,8 +188,60 @@ public class TestSearcherTaxonomyManager extends FacetTestCase {
 
     IOUtils.close(mgr, tw, w, taxoDir, dir);
   }
+  
+  public void testDirectory() throws Exception {
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    final IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+    final DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+    // first empty commit
+    w.commit();
+    tw.commit();
+    final SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
+    final FacetsConfig config = new FacetsConfig();
+    config.setMultiValued("field", true);
+    final AtomicBoolean stop = new AtomicBoolean();
 
-  public void testReplaceTaxonomy() throws Exception {
+    // How many unique facets to index before stopping:
+    final int ordLimit = TEST_NIGHTLY ? 100000 : 6000;
+
+    Thread indexer = new IndexerThread(w, config, tw, mgr, ordLimit, stop);
+    indexer.start();
+
+    try {
+      while (!stop.get()) {
+        SearcherAndTaxonomy pair = mgr.acquire();
+        try {
+          //System.out.println("search maxOrd=" + pair.taxonomyReader.getSize());
+          FacetsCollector sfc = new FacetsCollector();
+          pair.searcher.search(new MatchAllDocsQuery(), sfc);
+          Facets facets = getTaxonomyFacetCounts(pair.taxonomyReader, config, sfc);
+          FacetResult result = facets.getTopChildren(10, "field");
+          if (pair.searcher.getIndexReader().numDocs() > 0) { 
+            //System.out.println(pair.taxonomyReader.getSize());
+            assertTrue(result.childCount > 0);
+            assertTrue(result.labelValues.length > 0);
+          }
+
+          //if (VERBOSE) {
+          //System.out.println("TEST: facets=" + FacetTestUtils.toString(results.get(0)));
+          //}
+        } finally {
+          mgr.release(pair);
+        }
+      }
+    } finally {
+      indexer.join();
+    }
+
+    if (VERBOSE) {
+      System.out.println("TEST: now stop");
+    }
+
+    IOUtils.close(mgr, tw, w, taxoDir, indexDir);
+  }
+  
+  public void testReplaceTaxonomyNRT() throws Exception {
     Directory dir = newDirectory();
     Directory taxoDir = newDirectory();
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
@@ -185,4 +265,43 @@ public class TestSearcherTaxonomyManager extends FacetTestCase {
 
     IOUtils.close(mgr, tw, w, taxoDir, dir);
   }
+  
+  public void testReplaceTaxonomyDirectory() throws Exception {
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
+    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
+    w.commit();
+    tw.commit();
+
+    Directory taxoDir2 = newDirectory();
+    DirectoryTaxonomyWriter tw2 = new DirectoryTaxonomyWriter(taxoDir2);
+    tw2.addCategory(new FacetLabel("a", "b"));
+    tw2.close();
+
+    SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);
+    SearcherAndTaxonomy pair = mgr.acquire();
+    try {
+      assertEquals(1, pair.taxonomyReader.getSize());
+    } finally {
+      mgr.release(pair);
+    }
+    
+    w.addDocument(new Document());
+    tw.replaceTaxonomy(taxoDir2);
+    taxoDir2.close();
+    w.commit();
+    tw.commit();
+
+    mgr.maybeRefresh();
+    pair = mgr.acquire();
+    try {
+      assertEquals(3, pair.taxonomyReader.getSize());
+    } finally {
+      mgr.release(pair);
+    }
+
+    IOUtils.close(mgr, tw, w, taxoDir, indexDir);
+  }
+
 }