mirror of https://github.com/apache/lucene.git
423 lines
18 KiB
DTD
423 lines
18 KiB
DTD
<!--
|
|
<h3>Background</h3>
|
|
This DTD describes the XML syntax used to perform advanced searches using the core Lucene search engine. The motivation behind the XML query syntax is:
|
|
<ol>
|
|
<li>To open up Lucene functionality to clients other than Java</li>
|
|
<li>To offer a form of expressing queries that can easily be
|
|
<ul>
|
|
<li>Persisted for logging/auditing purposes</li>
|
|
<li>Changed by editing text query templates (XSLT) without requiring a recompile/redeploy of applications</li>
|
|
<li>Serialized across networks (without requiring Java bytecode for Query logic deployed on clients)</li>
|
|
</ul>
|
|
</li>
|
|
<li>To provide a shorthand way of expressing query logic which echos the logical tree structure of query objects more closely than reading procedural Java query construction code</li>
|
|
<li>To bridge the growing gap between Lucene query/filtering functionality and the set of functionality accessible throught the standard Lucene QueryParser syntax</li>
|
|
<li>To provide a simply extensible syntax that does not require complex parser skills such as knowledge of JavaCC syntax</li>
|
|
</ol>
|
|
|
|
|
|
<h3>Syntax overview</h3>
|
|
Search syntax consists of two types of elements:
|
|
<ul>
|
|
<li><i>Queries</i></li>
|
|
<li><i>Filters</i></li>
|
|
</ul>
|
|
|
|
<h4>Queries</h4>
|
|
The root of any XML search must be a <i>Query</i> type element used to select content.
|
|
Queries typically score matches on documents using a number of different factors in order to provide relevant results first.
|
|
One common example of a query tag is the <a href="#UserQuery">UserQuery</a> element which uses the standard
|
|
Lucene QueryParser to parse Google-style search syntax provided by end users.
|
|
|
|
<h4>Filters</h4>
|
|
Unlike Queries, <i>Filters</i> are not used to select or score content - they are simply used to filter <i>Query</i> output (see <a href="#FilteredQuery">FilteredQuery</a> for an example use of query filtering).
|
|
Because Filters simply offer a yes/no decision for each document in the index their output can be efficiently cached in memory as a <a href="http://java.sun.com/j2se/1.4.2/docs/api/java/util/BitSet.html">Bitset</a> for
|
|
subsequent reuse (see <a href="#CachedFilter">CachedFilter</a> tag).
|
|
|
|
<h4>Nesting elements</h4>
|
|
Many of the the elements can nest other elements to produce queries/filters of an arbitrary depth and complexity.
|
|
The <a href="#BooleanQuery">BooleanQuery</a> element is one such example which provides a means for combining other queries (including other BooleanQueries) using Boolean
|
|
logic to determine mandatory or optional elements.
|
|
|
|
|
|
<h3>Advanced topics</h3>
|
|
<h4>Advanced positional testing - span queries</h4>
|
|
The <i>SpanQuery</i> class of queries allow for complex positional tests which not only look for certain combinations of words but in particular
|
|
positions in relation to each other and the documents containing them.
|
|
|
|
|
|
CoreParser.java is the Java class that encapsulates this parser behaviour.
|
|
|
|
|
|
@title Core Lucene
|
|
-->
|
|
|
|
<!-- @hidden Define core types of XML elements -->
|
|
<!ENTITY % coreSpanQueries "SpanOr|SpanNear|SpanOrTerms|SpanFirst|SpanNot|SpanTerm|BoostingTermQuery" >
|
|
<!ENTITY % coreQueries "BooleanQuery|UserQuery|FilteredQuery|TermQuery|TermsQuery|MatchAllDocsQuery|ConstantScoreQuery|BoostingTermQuery" >
|
|
<!ENTITY % coreFilters "RangeFilter|CachedFilter" >
|
|
|
|
<!-- @hidden Allow for extensions -->
|
|
<!ENTITY % extendedSpanQueries1 " " >
|
|
<!ENTITY % extendedQueries1 " " >
|
|
<!ENTITY % extendedFilters1 " " >
|
|
|
|
<!ENTITY % spanQueries "%coreSpanQueries;%extendedSpanQueries1;" >
|
|
<!ENTITY % queries "%coreQueries;|%spanQueries;%extendedQueries1;" >
|
|
|
|
|
|
<!ENTITY % filters "%coreFilters;%extendedFilters1;" >
|
|
|
|
<!--
|
|
BooleanQuerys implement Boolean logic which controls how multiple Clauses should be interpreted.
|
|
Some clauses may represent optional Query criteria while others represent mandatory criteria.
|
|
@example
|
|
<em>Find articles about banks, preferably talking about mergers but nothing to do with "sumitomo"</em>
|
|
%
|
|
<BooleanQuery fieldName="contents">
|
|
<Clause occurs="should">
|
|
<TermQuery>merger</TermQuery>
|
|
</Clause>
|
|
<Clause occurs="mustnot">
|
|
<TermQuery>sumitomo</TermQuery>
|
|
</Clause>
|
|
<Clause occurs="must">
|
|
<TermQuery>bank</TermQuery>
|
|
</Clause>
|
|
</BooleanQuery>
|
|
|
|
%
|
|
-->
|
|
<!ELEMENT BooleanQuery (Clause)+>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST BooleanQuery boost CDATA "1.0">
|
|
<!-- fieldName can optionally be defined here as a default attribute used by all child elements -->
|
|
<!ATTLIST BooleanQuery fieldName CDATA #IMPLIED>
|
|
<!-- The "Coordination factor" rewards documents that contain more of the optional clauses in this list. This flag can be used to turn off this factor. -->
|
|
<!ATTLIST BooleanQuery disableCoord (true | false) "false">
|
|
<!-- The minimum number of optional clauses that should be present in any one document before it is considered to be a match. -->
|
|
<!ATTLIST BooleanQuery minimumNumberShouldMatch CDATA "0">
|
|
|
|
<!-- NOTE: "Clause" tag has 2 modes of use - inside <BooleanQuery> in which case only "query" types can be
|
|
child elements - while in a <BooleanFilter> clause only "filter" types can be contained.
|
|
@hidden TODO: Change BooleanFilterBuilder and BooleanQueryBuilder to auto-wrap choice of query or filters. This type of
|
|
code already exists in CachedFilter so could be reused.
|
|
-->
|
|
<!ELEMENT Clause (%queries;|%filters;)>
|
|
<!-- Controls if the clause is optional (should), mandatory (must) or unacceptable (mustNot) -->
|
|
<!ATTLIST Clause occurs (should | must | mustnot) "should">
|
|
|
|
|
|
<!-- Caches any nested query or filter in an LRU (Least recently used) Cache. Cached queries, like filters, are turned into
|
|
Bitsets at a cost of 1 bit per document in the index. The memory cost of a cached query/filter is therefore numberOfDocsinIndex/8 bytes.
|
|
Queries that are cached as filters obviously retain none of the scoring information associated with results - they retain just
|
|
a Boolean yes/no record of which documents matched.
|
|
@example
|
|
<em>Search for documents about banks from the last 10 years - caching the commonly-used "last 10 year" filter as a BitSet in
|
|
RAM to eliminate the cost of building this filter from disk for every query</em>
|
|
%
|
|
<FilteredQuery>
|
|
<Query>
|
|
<UserQuery>bank</UserQuery>
|
|
</Query>
|
|
<Filter>
|
|
<CachedFilter>
|
|
<RangeFilter fieldName="date" lowerTerm="19970101" upperTerm="20070101"/>
|
|
</CachedFilter>
|
|
</Filter>
|
|
</FilteredQuery>
|
|
%
|
|
|
|
-->
|
|
<!ELEMENT CachedFilter (%queries;|%filters;)>
|
|
|
|
|
|
|
|
<!--
|
|
Passes content directly through to the standard LuceneQuery parser see "Lucene Query Syntax"
|
|
@example
|
|
<em>Search for documents about John Smith or John Doe using standard LuceneQuerySyntax</em>
|
|
%
|
|
<UserQuery>"John Smith" OR "John Doe"</UserQuery>
|
|
%
|
|
|
|
-->
|
|
<!ELEMENT UserQuery (#PCDATA)>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST UserQuery boost CDATA "1.0">
|
|
<!-- fieldName can optionally be defined here to change the default field used in the QueryParser -->
|
|
<!ATTLIST UserQuery fieldName CDATA #IMPLIED>
|
|
|
|
<!-- A query which is used to match all documents. This has a couple of uses:
|
|
<ol>
|
|
<li> as a Clause in a BooleanQuery who's only other clause
|
|
is a "mustNot" match (Lucene requires at least one positive clause) and..</li>
|
|
<li> in a FilteredQuery where a Filter tag is effectively being
|
|
used to select content rather than it's usual role of filtering the results of a query.</li>
|
|
</ol>
|
|
|
|
@example
|
|
<em>Effectively use a Filter as a query </em>
|
|
%
|
|
<FilteredQuery>
|
|
<Query>
|
|
<MatchAllDocsQuery/>
|
|
</Query>
|
|
<Filter>
|
|
<RangeFilter fieldName="date" lowerTerm="19870409" upperTerm="19870412"/>
|
|
</Filter>
|
|
</FilteredQuery>
|
|
%
|
|
|
|
-->
|
|
<!ELEMENT MatchAllDocsQuery EMPTY>
|
|
|
|
<!-- a single term query - no analysis is done of the child text
|
|
@example
|
|
<em>Match on a primary key</em>
|
|
%
|
|
<TermQuery fieldName="primaryKey">13424</TermQuery>
|
|
%
|
|
-->
|
|
<!ELEMENT TermQuery (#PCDATA)>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST TermQuery boost CDATA "1.0">
|
|
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
|
|
<!ATTLIST TermQuery fieldName CDATA #IMPLIED>
|
|
|
|
|
|
<!--
|
|
A boosted term query - no analysis is done of the child text. Also a span member.
|
|
|
|
(Text below is copied from the javadocs of BoostingTermQuery)
|
|
|
|
The BoostingTermQuery is very similar to the {@link org.apache.lucene.search.spans.SpanTermQuery} except
|
|
that it factors in the value of the payload located at each of the positions where the
|
|
{@link org.apache.lucene.index.Term} occurs.
|
|
|
|
In order to take advantage of this, you must override {@link org.apache.lucene.search.Similarity#scorePayload(String, byte[],int,int)}
|
|
which returns 1 by default.
|
|
|
|
Payload scores are averaged across term occurrences in the document.
|
|
|
|
@see org.apache.lucene.search.Similarity#scorePayload(String, byte[], int, int)
|
|
-->
|
|
<!ELEMENT BoostingTermQuery (#PCDATA)>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST TermQuery boost CDATA "1.0">
|
|
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
|
|
<!ATTLIST TermQuery fieldName CDATA #IMPLIED>
|
|
|
|
|
|
|
|
<!--
|
|
The equivalent of a BooleanQuery with multiple optional TermQuery clauses.
|
|
Child text is analyzed using a field-specific choice of Analyzer to produce a set of terms that are ORed together in Boolean logic.
|
|
Unlike UserQuery element, this does not parse any special characters to control fuzzy/phrase/boolean logic and as such is incapable
|
|
of producing a Query parse error given any user input
|
|
@example
|
|
<em>Match on text from a database description (which may contain characters that
|
|
are illegal characters in the standard Lucene Query syntax used in the UserQuery tag</em>
|
|
%
|
|
<TermsQuery fieldName="description">Smith & Sons (Ltd) : incorporated 1982</TermsQuery>
|
|
%
|
|
-->
|
|
<!ELEMENT TermsQuery (#PCDATA)>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST TermsQuery boost CDATA "1.0">
|
|
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
|
|
<!ATTLIST TermsQuery fieldName CDATA #IMPLIED>
|
|
<!-- The "Coordination factor" rewards documents that contain more of the terms in this list. This flag can be used to turn off this factor. -->
|
|
<!ATTLIST TermsQuery disableCoord (true | false) "false">
|
|
<!-- The minimum number of terms that should be present in any one document before it is considered to be a match. -->
|
|
<!ATTLIST TermsQuery minimumNumberShouldMatch CDATA "0">
|
|
|
|
|
|
<!--
|
|
Runs a Query and filters results to only those query matches that also match the Filter element.
|
|
@example
|
|
<em>Find all documents about Lucene that have a status of "published"</em>
|
|
%
|
|
<FilteredQuery>
|
|
<Query>
|
|
<UserQuery>Lucene</UserQuery>
|
|
</Query>
|
|
<Filter>
|
|
<TermsFilter fieldName="status">published</TermsFilter>
|
|
</Filter>
|
|
</FilteredQuery>
|
|
%
|
|
-->
|
|
<!ELEMENT FilteredQuery (Query,Filter)>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST FilteredQuery boost CDATA "1.0">
|
|
<!-- Used to identify a nested Query element inside another container element. NOT a top-level query tag -->
|
|
<!ELEMENT Query (%queries;)>
|
|
<!-- The choice of Filter that MUST also be matched -->
|
|
<!ELEMENT Filter (%filters;)>
|
|
|
|
<!--
|
|
Filter used to limit query results to documents matching a range of field values
|
|
@example
|
|
<em>Search for documents about banks from the last 10 years</em>
|
|
%
|
|
<FilteredQuery>
|
|
<Query>
|
|
<UserQuery>bank</UserQuery>
|
|
</Query>
|
|
<Filter>
|
|
<RangeFilter fieldName="date" lowerTerm="19970101" upperTerm="20070101"/>
|
|
</Filter>
|
|
</FilteredQuery>
|
|
%
|
|
-->
|
|
<!ELEMENT RangeFilter EMPTY>
|
|
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
|
|
<!ATTLIST RangeFilter fieldName CDATA #IMPLIED>
|
|
<!-- The lower-most term value for this field (must be <= upperTerm) -->
|
|
<!ATTLIST RangeFilter lowerTerm CDATA #REQUIRED>
|
|
<!-- The upper-most term value for this field (must be >= lowerTerm) -->
|
|
<!ATTLIST RangeFilter upperTerm CDATA #REQUIRED>
|
|
<!-- Controls if the lowerTerm in the range is part of the allowed set of values -->
|
|
<!ATTLIST RangeFilter includeLower (true | false) "true">
|
|
<!-- Controls if the upperTerm in the range is part of the allowed set of values -->
|
|
<!ATTLIST RangeFilter includeUpper (true | false) "true">
|
|
|
|
|
|
|
|
<!-- A single term used in a SpanQuery. These clauses are the building blocks for more complex "span" queries which test word proximity
|
|
@example <em>Find documents using terms close to each other about mining and accidents</em>
|
|
%
|
|
<SpanNear slop="8" inOrder="false" fieldName="text">
|
|
<SpanOr>
|
|
<SpanTerm>killed</SpanTerm>
|
|
<SpanTerm>died</SpanTerm>
|
|
<SpanTerm>dead</SpanTerm>
|
|
</SpanOr>
|
|
<SpanOr>
|
|
<SpanTerm>miner</SpanTerm>
|
|
<SpanTerm>mining</SpanTerm>
|
|
<SpanTerm>miners</SpanTerm>
|
|
</SpanOr>
|
|
</SpanNear>
|
|
%
|
|
-->
|
|
<!ELEMENT SpanTerm (#PCDATA)>
|
|
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
|
|
<!ATTLIST SpanTerm fieldName CDATA #REQUIRED>
|
|
|
|
<!-- A field-specific analyzer is used here to parse the child text provided in this tag. The SpanTerms produced are ORed in terms of Boolean logic
|
|
@example <em>Use SpanOrTerms as a more convenient/succinct way of expressing multiple choices of SpanTerms. This example looks for reports
|
|
using words describing a fatality near to references to miners</em>
|
|
%
|
|
<SpanNear slop="8" inOrder="false" fieldName="text">
|
|
<SpanOrTerms>killed died death dead deaths</SpanOrTerms>
|
|
<SpanOrTerms>miner mining miners</SpanOrTerms>
|
|
</SpanNear>
|
|
%
|
|
-->
|
|
<!ELEMENT SpanOrTerms (#PCDATA)>
|
|
<!-- fieldName must be defined here or is taken from the most immediate parent XML element that defines a "fieldName" attribute -->
|
|
<!ATTLIST SpanOrTerms fieldName CDATA #REQUIRED>
|
|
|
|
<!-- Takes any number of child queries from the Span family
|
|
@example <em>Find documents using terms close to each other about mining and accidents</em>
|
|
%
|
|
<SpanNear slop="8" inOrder="false" fieldName="text">
|
|
<SpanOr>
|
|
<SpanTerm>killed</SpanTerm>
|
|
<SpanTerm>died</SpanTerm>
|
|
<SpanTerm>dead</SpanTerm>
|
|
</SpanOr>
|
|
<SpanOr>
|
|
<SpanTerm>miner</SpanTerm>
|
|
<SpanTerm>mining</SpanTerm>
|
|
<SpanTerm>miners</SpanTerm>
|
|
</SpanOr>
|
|
</SpanNear>
|
|
%
|
|
|
|
-->
|
|
<!ELEMENT SpanOr (%spanQueries;)* >
|
|
|
|
<!-- Takes any number of child queries from the Span family and tests for proximity
|
|
@hidden TODO SpanNear missing "boost attr (could add to SpanBuilderBase)
|
|
-->
|
|
<!ELEMENT SpanNear (%spanQueries;)* >
|
|
<!-- defines the maximum distance between Span elements where distance is expressed as word number, not byte offset
|
|
@example <em>Find documents using terms within 8 words of each other talking about mining and accidents</em>
|
|
%
|
|
<SpanNear slop="8" inOrder="false" fieldName="text">
|
|
<SpanOr>
|
|
<SpanTerm>killed</SpanTerm>
|
|
<SpanTerm>died</SpanTerm>
|
|
<SpanTerm>dead</SpanTerm>
|
|
</SpanOr>
|
|
<SpanOr>
|
|
<SpanTerm>miner</SpanTerm>
|
|
<SpanTerm>mining</SpanTerm>
|
|
<SpanTerm>miners</SpanTerm>
|
|
</SpanOr>
|
|
</SpanNear>
|
|
%
|
|
-->
|
|
<!ATTLIST SpanNear slop CDATA #REQUIRED>
|
|
<!-- Controls if matching terms have to appear in the order listed or can be reversed -->
|
|
<!ATTLIST SpanNear inOrder (true | false) "true">
|
|
|
|
<!-- Looks for a SpanQuery match occuring near the beginning of a document
|
|
|
|
@example
|
|
<em>Find letters where the first 50 words talk about a resignation:</em>
|
|
%
|
|
<SpanFirst end="50">
|
|
<SpanOrTerms fieldName="text">resigning resign leave</SpanOrTerms>
|
|
</SpanFirst>
|
|
%
|
|
|
|
-->
|
|
<!ELEMENT SpanFirst (%spanQueries;) >
|
|
<!-- Controls the end of the region considered in a document's field (expressed in word number, not byte offset) -->
|
|
<!ATTLIST SpanFirst end CDATA #REQUIRED>
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST SpanFirst boost CDATA "1.0">
|
|
|
|
<!-- Finds documents matching a SpanQuery but not if matching another SpanQuery
|
|
@example <em>Find documents talking about social services but not containing the word "public"</em>
|
|
%
|
|
<SpanNot fieldName="text">
|
|
<Include>
|
|
<SpanNear slop="2" inOrder="true">
|
|
<SpanTerm>social</SpanTerm>
|
|
<SpanTerm>services</SpanTerm>
|
|
</SpanNear>
|
|
</Include>
|
|
<Exclude>
|
|
<SpanTerm>public</SpanTerm>
|
|
</Exclude>
|
|
</SpanNot>
|
|
%
|
|
|
|
-->
|
|
<!ELEMENT SpanNot (Include,Exclude) >
|
|
<!-- The SpanQuery to find -->
|
|
<!ELEMENT Include (%spanQueries;) >
|
|
<!-- The SpanQuery to be avoided -->
|
|
<!ELEMENT Exclude (%spanQueries;) >
|
|
|
|
|
|
<!-- a utility tag to wrap any filter as a query
|
|
@example <em> Find all documents from the last 10 years </em>
|
|
%
|
|
<ConstantScoreQuery>
|
|
<RangeFilter fieldName="date" lowerTerm="19970101" upperTerm="20070101"/>
|
|
</ConstantScoreQuery>
|
|
%
|
|
-->
|
|
<!ELEMENT ConstantScoreQuery (%filters;)* >
|
|
<!-- Optional boost for matches on this query. Values > 1 -->
|
|
<!ATTLIST ConstantScoreQuery boost CDATA "1.0">
|
|
|
|
|
|
|