From a02eb9ee42b66c1d6ce33da49f2b847482cd1d59 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 3 Sep 2009 13:02:16 +0000 Subject: [PATCH] LUCENE-1876: add missing package.html to some contribs git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@810923 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/lucene/collation/package.html | 182 ++++++++++++++++++ .../org/apache/lucene/store/je/package.html | 22 +++ .../org/apache/lucene/store/db/package.html | 22 +++ .../lucene/queryParser/analyzing/package.html | 22 +++ .../queryParser/complexPhrase/package.html | 22 +++ .../queryParser/precedence/package.html | 22 +++ .../lucene/spatial/geohash/package.html | 22 +++ .../apache/lucene/spatial/tier/package.html | 22 +++ .../lucene/wikipedia/analysis/package.html | 22 +++ .../org/apache/lucene/xmlparser/package.html | 22 +++ 10 files changed, 380 insertions(+) create mode 100644 contrib/collation/src/java/org/apache/lucene/collation/package.html create mode 100644 contrib/db/bdb-je/src/java/org/apache/lucene/store/je/package.html create mode 100644 contrib/db/bdb/src/java/org/apache/lucene/store/db/package.html create mode 100644 contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/package.html create mode 100644 contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/package.html create mode 100644 contrib/misc/src/java/org/apache/lucene/queryParser/precedence/package.html create mode 100644 contrib/spatial/src/java/org/apache/lucene/spatial/geohash/package.html create mode 100644 contrib/spatial/src/java/org/apache/lucene/spatial/tier/package.html create mode 100644 contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/package.html create mode 100644 contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/package.html diff --git a/contrib/collation/src/java/org/apache/lucene/collation/package.html b/contrib/collation/src/java/org/apache/lucene/collation/package.html new file mode 100644 index 00000000000..25d69e49165 --- /dev/null +++ b/contrib/collation/src/java/org/apache/lucene/collation/package.html @@ -0,0 +1,182 @@ + + + + + Lucene Collation Package + + +

+ CollationKeyFilter and ICUCollationKeyFilter + convert each token into its binary CollationKey using the + provided Collator, and then encode the CollationKey + as a String using + {@link org.apache.lucene.util.IndexableBinaryStringTools}, to allow it to be + stored as an index term. +

+

+ ICUCollationKeyFilter depends on ICU4J 4.0 to produce the + CollationKeys. icu4j-collation-4.0.jar, + a trimmed-down version of icu4j-4.0.jar that contains only the + code and data needed to support collation, is included in Lucene's Subversion + repository at contrib/collation/lib/. +

+ +

Use Cases

+ + + +

Example Usages

+ +

Farsi Range Queries

+
+  // "fa" Locale is not supported by Sun JDK 1.4 or 1.5
+  Collator collator = Collator.getInstance(new Locale("ar"));
+  CollationKeyAnalyzer analyzer = new CollationKeyAnalyzer(collator);
+  RAMDirectory ramDir = new RAMDirectory();
+  IndexWriter writer = new IndexWriter
+    (ramDir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+  Document doc = new Document();
+  doc.add(new Field("content", "\u0633\u0627\u0628", 
+                    Field.Store.YES, Field.Index.ANALYZED));
+  writer.addDocument(doc);
+  writer.close();
+  IndexSearcher is = new IndexSearcher(ramDir, true);
+
+  // The AnalyzingQueryParser in Lucene's contrib allows terms in range queries
+  // to be passed through an analyzer - Lucene's standard QueryParser does not
+  // allow this.
+  AnalyzingQueryParser aqp = new AnalyzingQueryParser("content", analyzer);
+  aqp.setLowercaseExpandedTerms(false);
+  
+  // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
+  // orders the U+0698 character before the U+0633 character, so the single
+  // indexed Term above should NOT be returned by a ConstantScoreRangeQuery
+  // with a Farsi Collator (or an Arabic one for the case when Farsi is not
+  // supported).
+  ScoreDoc[] result
+    = is.search(aqp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs;
+  assertEquals("The index Term should not be included.", 0, result.length);
+
 + +

Danish Sorting

+
+  Analyzer analyzer 
+    = new CollationKeyAnalyzer(Collator.getInstance(new Locale("da", "dk")));
+  RAMDirectory indexStore = new RAMDirectory();
+  IndexWriter writer = new IndexWriter 
+    (indexStore, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+  String[] tracer = new String[] { "A", "B", "C", "D", "E" };
+  String[] data = new String[] { "HAT", "HUT", "H\u00C5T", "H\u00D8T", "HOT" };
+  String[] sortedTracerOrder = new String[] { "A", "E", "B", "D", "C" };
+  for (int i = 0 ; i < data.length ; ++i) {
+    Document doc = new Document();
+    doc.add(new Field("tracer", tracer[i], Field.Store.YES, Field.Index.NO));
+    doc.add(new Field("contents", data[i], Field.Store.NO, Field.Index.ANALYZED));
+    writer.addDocument(doc);
+  }
+  writer.close();
+  Searcher searcher = new IndexSearcher(indexStore, true);
+  Sort sort = new Sort();
+  sort.setSort(new SortField("contents", SortField.STRING));
+  Query query = new MatchAllDocsQuery();
+  ScoreDoc[] result = searcher.search(query, null, 1000, sort).scoreDocs;
+  for (int i = 0 ; i < result.length ; ++i) {
+    Document doc = searcher.doc(result[i].doc);
+    assertEquals(sortedTracerOrder[i], doc.getValues("tracer")[0]);
+  }
+
 + +

Turkish Case Normalization

+
+  Collator collator = Collator.getInstance(new Locale("tr", "TR"));
+  collator.setStrength(Collator.PRIMARY);
+  Analyzer analyzer = new CollationKeyAnalyzer(collator);
+  RAMDirectory ramDir = new RAMDirectory();
+  IndexWriter writer = new IndexWriter
+    (ramDir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+  Document doc = new Document();
+  doc.add(new Field("contents", "DIGY", Field.Store.NO, Field.Index.ANALYZED));
+  writer.addDocument(doc);
+  writer.close();
+  IndexSearcher is = new IndexSearcher(ramDir, true);
+  QueryParser parser = new QueryParser("contents", analyzer);
+  Query query = parser.parse("d\u0131gy");   // U+0131: dotless i
+  ScoreDoc[] result = is.search(query, null, 1000).scoreDocs;
+  assertEquals("The index Term should be included.", 1, result.length);
+
 + +

Caveats and Comparisons

+

+ WARNING: Make sure you use exactly the same + Collator at index and query time -- CollationKeys + are only comparable when produced by + the same Collator. Since {@link java.text.RuleBasedCollator}s + are not independently versioned, it is unsafe to search against stored + CollationKeys unless the following are exactly the same (best + practice is to store this information with the index and check that they + remain the same at query time): +

+
    +
  1. JVM vendor
    2. +
  2. JVM version, including patch version
    3. +
  3. + The language (and country and variant, if specified) of the Locale + used when constructing the collator via + {@link java.text.Collator#getInstance(java.util.Locale)}. +
    4. +
  4. + The collation strength used - see {@link java.text.Collator#setStrength(int)} +
    5. +
+

+ ICUCollationKeyFilter uses ICU4J's Collator, which + makes its version available, thus allowing collation to be versioned + independently from the JVM. ICUCollationKeyFilter is also + significantly faster and generates significantly shorter keys than + CollationKeyFilter. See + http://site.icu-project.org/charts/collation-icu4j-sun for key + generation timing and key length comparisons between ICU4J and + java.text.Collator over several languages. +

+

+ CollationKeys generated by java.text.Collators are + not compatible with those those generated by ICU Collators. Specifically, if + you use CollationKeyFilter to generate index terms, do not use + ICUCollationKeyFilter on the query side, or vice versa. +

+
+
+ + diff --git a/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/package.html b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/package.html new file mode 100644 index 00000000000..21d0a39cbe2 --- /dev/null +++ b/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/package.html @@ -0,0 +1,22 @@ + + + + +Berkeley DB Java Edition based implementation of {@link org.apache.lucene.store.Directory Directory}. + + diff --git a/contrib/db/bdb/src/java/org/apache/lucene/store/db/package.html b/contrib/db/bdb/src/java/org/apache/lucene/store/db/package.html new file mode 100644 index 00000000000..2d499a4e6f9 --- /dev/null +++ b/contrib/db/bdb/src/java/org/apache/lucene/store/db/package.html @@ -0,0 +1,22 @@ + + + + +Berkeley DB 4.3 based implementation of {@link org.apache.lucene.store.Directory Directory}. + + diff --git a/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/package.html b/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/package.html new file mode 100644 index 00000000000..2785a6bee18 --- /dev/null +++ b/contrib/misc/src/java/org/apache/lucene/queryParser/analyzing/package.html @@ -0,0 +1,22 @@ + + + + +QueryParser that passes Fuzzy-, Prefix-, Range-, and WildcardQuerys through the given analyzer. + + diff --git a/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/package.html b/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/package.html new file mode 100644 index 00000000000..ade19fc0e7e --- /dev/null +++ b/contrib/misc/src/java/org/apache/lucene/queryParser/complexPhrase/package.html @@ -0,0 +1,22 @@ + + + + +QueryParser which permits complex phrase query syntax eg "(john jon jonathan~) peters*" + + diff --git a/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/package.html b/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/package.html new file mode 100644 index 00000000000..fdc3a30651d --- /dev/null +++ b/contrib/misc/src/java/org/apache/lucene/queryParser/precedence/package.html @@ -0,0 +1,22 @@ + + + + +QueryParser designed to handle operator precedence in a more sensible fashion than the default QueryParser. + + diff --git a/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/package.html b/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/package.html new file mode 100644 index 00000000000..4999361f344 --- /dev/null +++ b/contrib/spatial/src/java/org/apache/lucene/spatial/geohash/package.html @@ -0,0 +1,22 @@ + + + + +Support for Geohash encoding, decoding, and filtering. + + diff --git a/contrib/spatial/src/java/org/apache/lucene/spatial/tier/package.html b/contrib/spatial/src/java/org/apache/lucene/spatial/tier/package.html new file mode 100644 index 00000000000..333b7948e57 --- /dev/null +++ b/contrib/spatial/src/java/org/apache/lucene/spatial/tier/package.html @@ -0,0 +1,22 @@ + + + + +Support for filtering based upon geographic location. + + diff --git a/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/package.html b/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/package.html new file mode 100644 index 00000000000..7b23869e1d5 --- /dev/null +++ b/contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/package.html @@ -0,0 +1,22 @@ + + + + +Tokenizer that is aware of Wikipedia syntax. + + diff --git a/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/package.html b/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/package.html new file mode 100644 index 00000000000..8740ffd7a0f --- /dev/null +++ b/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/package.html @@ -0,0 +1,22 @@ + + + + +Parser that produces Lucene Query objects from XML streams. + +