SOLR-1229: fixes for deletedPkQuery

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@792963 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Erik Hatcher 2009-07-10 14:55:28 +00:00
parent 6457b52936
commit a7c25f8813
8 changed files with 918 additions and 12 deletions

View File

@ -246,6 +246,8 @@ Bug Fixes
26.SOLR-1146: ConcurrentModificationException in DataImporter.getStatusMessages 26.SOLR-1146: ConcurrentModificationException in DataImporter.getStatusMessages
(Walter Ferrara, Noble Paul via shalin) (Walter Ferrara, Noble Paul via shalin)
27.SOLR-1229: Fixes for deletedPkQuery, particularly when using transformed Solr unique id's
(Lance Norskog, Noble Paul via ehatcher)
Documentation Documentation

View File

@ -68,7 +68,7 @@ public abstract class AbstractDataImportHandlerTest extends
protected void runDeltaImport(String dataConfig) throws Exception { protected void runDeltaImport(String dataConfig) throws Exception {
LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import", LocalSolrQueryRequest request = lrf.makeRequest("command", "delta-import",
"debug", "on", "clean", "true", "commit", "true", "dataConfig", "debug", "on", "clean", "false", "commit", "true", "dataConfig",
dataConfig); dataConfig);
h.query("/dataimport", request); h.query("/dataimport", request);
} }

View File

@ -78,6 +78,8 @@ public class DataConfig {
public String pk; public String pk;
public String pkMappingFromSchema;
public String dataSource; public String dataSource;
public Map<String, String> allAttributes; public Map<String, String> allAttributes;

View File

@ -106,7 +106,8 @@ public class DataImporter {
Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>(); Map<String, DataConfig.Field> fields = new HashMap<String, DataConfig.Field>();
initEntity(e, fields, false); initEntity(e, fields, false);
verifyWithSchema(fields); verifyWithSchema(fields);
} identifyPk(e);
}
} }
private void verifyWithSchema(Map<String, DataConfig.Field> fields) { private void verifyWithSchema(Map<String, DataConfig.Field> fields) {
@ -145,6 +146,22 @@ public class DataImporter {
} }
} }
private void identifyPk(DataConfig.Entity entity) {
String schemaPk = schema.getUniqueKeyField().getName();
//if no fields are mentioned . solr uniqeKey is same as dih 'pk'
entity.pkMappingFromSchema = schemaPk;
for (DataConfig.Field field : entity.fields) {
if(field.getName().equals(schemaPk)) {
entity.pkMappingFromSchema = field.column;
//get the corresponding column mapping for the solr uniqueKey
// But if there are multiple columns mapping to the solr uniqueKey, it will fail
// so , in one off cases we may need pk
break;
}
}
}
void loadDataConfig(String configFile) { void loadDataConfig(String configFile) {
try { try {

View File

@ -273,16 +273,11 @@ public class DocBuilder {
while (iter.hasNext()) { while (iter.hasNext()) {
Map<String, Object> map = iter.next(); Map<String, Object> map = iter.next();
Object key = null; Object key = null;
if(root.pk != null){ if(root.pkMappingFromSchema != null){
key = map.get(root.pk); key = map.get(root.pkMappingFromSchema);
} } else if(root.pk != null){
if(key == null && map.size() ==1){ key = map.get(root.pk);
//iterating through the map just to get the first and only item }
for (Map.Entry<String, Object> e : map.entrySet()) {
key = e.getValue();
break;
}
}
if(key == null) { if(key == null) {
LOG.warn("no key was available for deleteted pk query"); LOG.warn("no key was available for deleteted pk query");
continue; continue;

View File

@ -0,0 +1,295 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* <p>
* Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
* </p>
*
*
* @version $Id: TestSqlEntityProcessor2.java 723824 2008-12-05 19:14:11Z shalin $
* @since solr 1.3
*/
public class TestSqlEntityProcessorDelta extends AbstractDataImportHandlerTest {
private static final String FULLIMPORT_QUERY = "select * from x";
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
@Override
public String getSchemaFile() {
return "dataimport-schema.xml";
}
@Override
public String getSolrConfigFile() {
return "dataimport-solrconfig.xml";
}
@Override
public void setUp() throws Exception {
super.setUp();
}
@Override
public void tearDown() throws Exception {
super.tearDown();
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runFullImport(dataConfig_delta);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
add1document();
}
// WORKS
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_delete() throws Exception {
add1document();
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
MockDataSource.setIterator(DELTA_QUERY, Collections
.EMPTY_LIST.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_empty() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
// WORKS
@Test
@SuppressWarnings("unchecked")
public void XtestCompositePk_DeltaImport_replace_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
assertQ(req("id:1"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_nodelta() throws Exception {
add1document();
MockDataSource.clearCache();
MockDataSource.setIterator(DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("id:1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
assertQ(req("id:2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
private static String dataConfig_delta = "<dataConfig>\n"
+ " <document>\n"
+ " <entity name=\"x\" pk=\"id\" transformer=\"TemplateTransformer\""
+ " query=\"" + FULLIMPORT_QUERY + "\""
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
+ " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n"
+ " <field column=\"id\" name=\"id\"/>\n"
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
+ " <field column=\"desc\" />\n"
+ " </entity>\n" + " </entity>\n"
+ " </document>\n" + "</dataConfig>\n";
}

View File

@ -0,0 +1,291 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.dataimport;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* <p>
* Test for SqlEntityProcessor which checks variations in primary key names and deleted ids
* </p>
*
*
* @version $Id: TestSqlEntityProcessor2.java 723824 2008-12-05 19:14:11Z shalin $
* @since solr 1.3
*/
public class TestSqlEntityProcessorDelta2 extends AbstractDataImportHandlerTest {
private static final String FULLIMPORT_QUERY = "select * from x";
private static final String DELTA_QUERY = "select id from x where last_modified > NOW";
private static final String DELETED_PK_QUERY = "select id from x where last_modified > NOW AND deleted='true'";
@Override
public String getSchemaFile() {
return "dataimport-solr_id-schema.xml";
}
@Override
public String getSolrConfigFile() {
return "dataimport-solrconfig.xml";
}
@Override
public void setUp() throws Exception {
super.setUp();
}
@Override
public void tearDown() throws Exception {
super.tearDown();
}
@SuppressWarnings("unchecked")
private void add1document() throws Exception {
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator(FULLIMPORT_QUERY, parentRow.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runFullImport(dataConfig_delta2);
assertQ(req("*:* OR add1document"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_FullImport() throws Exception {
add1document();
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_delete() throws Exception {
add1document();
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY, deletedRow.iterator());
MockDataSource.setIterator(DELTA_QUERY, Collections
.EMPTY_LIST.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_empty() throws Exception {
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY, deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "hello"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_empty"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void XtestCompositePk_DeltaImport_replace_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_replace_delete"), "//*[@numFound='0']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_replace_nodelete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
MockDataSource.setIterator(DELETED_PK_QUERY, Collections
.EMPTY_LIST.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "1"));
MockDataSource.setIterator("select * from x where id='1'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='1'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("desc:hello OR XtestCompositePk_DeltaImport_replace_nodelete"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_add"), "//*[@numFound='2']");
assertQ(req("solr_id:prefix-1"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='1']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_nodelta() throws Exception {
add1document();
MockDataSource.clearCache();
MockDataSource.setIterator(DELTA_QUERY,
Collections.EMPTY_LIST.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-1 OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
assertQ(req("desc:hello OR testCompositePk_DeltaImport_nodelta"), "//*[@numFound='1']");
}
@Test
@SuppressWarnings("unchecked")
public void testCompositePk_DeltaImport_add_delete() throws Exception {
add1document();
MockDataSource.clearCache();
List deltaRow = new ArrayList();
deltaRow.add(createMap("id", "2"));
MockDataSource.setIterator(DELTA_QUERY,
deltaRow.iterator());
List deletedRow = new ArrayList();
deletedRow.add(createMap("id", "1"));
MockDataSource.setIterator(DELETED_PK_QUERY,
deletedRow.iterator());
List parentRow = new ArrayList();
parentRow.add(createMap("id", "2"));
MockDataSource.setIterator("select * from x where id='2'", parentRow
.iterator());
List childRow = new ArrayList();
childRow.add(createMap("desc", "goodbye"));
MockDataSource.setIterator("select * from y where y.A='2'", childRow
.iterator());
super.runDeltaImport(dataConfig_delta2);
assertQ(req("*:* OR XtestCompositePk_DeltaImport_add_delete"), "//*[@numFound='1']");
assertQ(req("solr_id:prefix-2"), "//*[@numFound='1']");
assertQ(req("desc:hello"), "//*[@numFound='0']");
assertQ(req("desc:goodbye"), "//*[@numFound='1']");
}
private static String dataConfig_delta2 = "<dataConfig>\n"
+ " <document>\n"
+ " <entity name=\"x\" pk=\"id\" transformer=\"TemplateTransformer\""
+ " query=\"" + FULLIMPORT_QUERY + "\""
+ " deletedPkQuery=\"" + DELETED_PK_QUERY + "\""
+ " deltaImportQuery=\"select * from x where id='${dataimporter.delta.id}'\""
+ " deltaQuery=\"" + DELTA_QUERY + "\">\n"
+ " <field column=\"solr_id\" template=\"prefix-${x.id}\"/>\n"
+ " <entity name=\"y\" query=\"select * from y where y.A='${x.id}'\">\n"
+ " <field column=\"desc\" />\n"
+ " </entity>\n" + " </entity>\n"
+ " </document>\n" + "</dataConfig>\n";
}

View File

@ -0,0 +1,304 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
This is the Solr schema file. This file should be named "schema.xml" and
should be in the conf directory under the solr home
(i.e. ./solr/conf/schema.xml by default)
or located where the classloader for the Solr webapp can find it.
This example schema is the recommended starting point for users.
It should be kept correct and concise, usable out-of-the-box.
For more information, on how to customize this file, please see
http://wiki.apache.org/solr/SchemaXml
-->
<schema name="test" version="1.1">
<!-- attribute "name" is the name of this schema and is only used for display purposes.
Applications should change this to reflect the nature of the search collection.
version="1.1" is Solr's version number for the schema syntax and semantics. It should
not normally be changed by applications.
1.0: multiValued attribute did not exist, all fields are multiValued by nature
1.1: multiValued attribute introduced, false by default -->
<types>
<!-- field type definitions. The "name" attribute is
just a label to be used by field definitions. The "class"
attribute and any other attributes determine the real
behavior of the fieldType.
Class names starting with "solr" refer to java classes in the
org.apache.solr.analysis package.
-->
<!-- The StrField type is not analyzed, but indexed/stored verbatim.
- StrField and TextField support an optional compressThreshold which
limits compression (if enabled in the derived fields) to values which
exceed a certain size (in characters).
-->
<fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
<!-- boolean type: "true" or "false" -->
<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true" omitNorms="true"/>
<!-- The optional sortMissingLast and sortMissingFirst attributes are
currently supported on types that are sorted internally as strings.
- If sortMissingLast="true", then a sort on this field will cause documents
without the field to come after documents with the field,
regardless of the requested sort order (asc or desc).
- If sortMissingFirst="true", then a sort on this field will cause documents
without the field to come before documents with the field,
regardless of the requested sort order.
- If sortMissingLast="false" and sortMissingFirst="false" (the default),
then default lucene sorting will be used which places docs without the
field first in an ascending sort and last in a descending sort.
-->
<!-- numeric field types that store and index the text
value verbatim (and hence don't support range queries, since the
lexicographic ordering isn't equal to the numeric ordering) -->
<fieldType name="integer" class="solr.IntField" omitNorms="true"/>
<fieldType name="long" class="solr.LongField" omitNorms="true"/>
<fieldType name="float" class="solr.FloatField" omitNorms="true"/>
<fieldType name="double" class="solr.DoubleField" omitNorms="true"/>
<!-- Numeric field types that manipulate the value into
a string value that isn't human-readable in its internal form,
but with a lexicographic ordering the same as the numeric ordering,
so that range queries work correctly. -->
<fieldType name="sint" class="solr.SortableIntField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="slong" class="solr.SortableLongField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="sfloat" class="solr.SortableFloatField" sortMissingLast="true" omitNorms="true"/>
<fieldType name="sdouble" class="solr.SortableDoubleField" sortMissingLast="true" omitNorms="true"/>
<!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
is a more restricted form of the canonical representation of dateTime
http://www.w3.org/TR/xmlschema-2/#dateTime
The trailing "Z" designates UTC time and is mandatory.
Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
All other components are mandatory.
Expressions can also be used to denote calculations that should be
performed relative to "NOW" to determine the value, ie...
NOW/HOUR
... Round to the start of the current hour
NOW-1DAY
... Exactly 1 day prior to now
NOW/DAY+6MONTHS+3DAYS
... 6 months and 3 days in the future from the start of
the current day
Consult the DateField javadocs for more information.
-->
<fieldType name="date" class="solr.DateField" sortMissingLast="true" omitNorms="true"/>
<!-- The "RandomSortField" is not used to store or search any
data. You can declare fields of this type it in your schema
to generate psuedo-random orderings of your docs for sorting
purposes. The ordering is generated based on the field name
and the version of the index, As long as the index version
remains unchanged, and the same field name is reused,
the ordering of the docs will be consistent.
If you want differend psuedo-random orderings of documents,
for the same version of the index, use a dynamicField and
change the name
-->
<fieldType name="random" class="solr.RandomSortField" indexed="true" />
<!-- solr.TextField allows the specification of custom text analyzers
specified as a tokenizer and a list of token filters. Different
analyzers may be specified for indexing and querying.
The optional positionIncrementGap puts space between multiple fields of
this type on the same document, with the purpose of preventing false phrase
matching across fields.
For more info on customizing your analyzer chain, please see
http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
-->
<!-- One can also specify an existing Analyzer class that has a
default constructor via the class attribute on the analyzer element
<fieldType name="text_greek" class="solr.TextField">
<analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
</fieldType>
-->
<!-- A text field that only splits on whitespace for exact matching of words -->
<fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
</analyzer>
</fieldType>
<!-- A text field that uses WordDelimiterFilter to enable splitting and matching of
words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
Synonyms and stopwords are customized by external files, and stemming is enabled.
Duplicate tokens at the same position (which may result from Stemmed Synonyms or
WordDelim parts) are removed.
-->
<fieldType name="text" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<!-- in this example, we will only use synonyms at query time
<filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-->
<!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<!--<filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>-->
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<!--<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>-->
<!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
<filter class="solr.LowerCaseFilterFactory"/>
<!--<filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>-->
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
<!-- Less flexible matching, but less false matches. Probably not ideal for product names,
but may be good for SKUs. Can insert dashes in the wrong place and still match. -->
<fieldType name="textTight" class="solr.TextField" positionIncrementGap="100" >
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory"/>
<!--<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>-->
<!--<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>-->
<filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
<filter class="solr.LowerCaseFilterFactory"/>
<!--<filter class="solr.EnglishPorterFilterFactory" protected="protwords.txt"/>-->
<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
</analyzer>
</fieldType>
<!-- This is an example of using the KeywordTokenizer along
With various TokenFilterFactories to produce a sortable field
that does not include some properties of the source text
-->
<fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
<analyzer>
<!-- KeywordTokenizer does no actual tokenizing, so the entire
input string is preserved as a single token
-->
<tokenizer class="solr.KeywordTokenizerFactory"/>
<!-- The LowerCase TokenFilter does what you expect, which can be
when you want your sorting to be case insensitive
-->
<filter class="solr.LowerCaseFilterFactory" />
<!-- The TrimFilter removes any leading or trailing whitespace -->
<filter class="solr.TrimFilterFactory" />
<!-- The PatternReplaceFilter gives you the flexibility to use
Java Regular expression to replace any sequence of characters
matching a pattern with an arbitrary replacement string,
which may include back refrences to portions of the orriginal
string matched by the pattern.
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
http://java.sun.com/j2se/1.5.0/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
/>
</analyzer>
</fieldType>
<!-- since fields of this type are by default not stored or indexed, any data added to
them will be ignored outright
-->
<fieldtype name="ignored" stored="false" indexed="false" class="solr.StrField" />
</types>
<fields>
<!-- Valid attributes for fields:
name: mandatory - the name for the field
type: mandatory - the name of a previously defined type from the <types> section
indexed: true if this field should be indexed (searchable or sortable)
stored: true if this field should be retrievable
compressed: [false] if this field should be stored using gzip compression
(this will only apply if the field type is compressable; among
the standard field types, only TextField and StrField are)
multiValued: true if this field may contain multiple values per document
omitNorms: (expert) set to true to omit the norms associated with
this field (this disables length normalization and index-time
boosting for the field, and saves some memory). Only full-text
fields or fields that need an index-time boost need norms.
termVectors: [false] set to true to store the term vector for a given field.
When using MoreLikeThis, fields used for similarity should be stored for
best performance.
-->
<field name="solr_id" type="string" indexed="true" stored="true" required="true" />
<field name="desc" type="string" indexed="true" stored="true" multiValued="true" />
<field name="date" type="date" indexed="true" stored="true" />
<field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
will be used if the name matches any of the patterns.
RESTRICTION: the glob-like pattern in the name attribute must have
a "*" only at the start or the end.
EXAMPLE: name="*_i" will match any field ending in _i (like myid_i, z_i)
Longer patterns will be matched first. if equal size patterns
both match, the first appearing in the schema will be used. -->
<dynamicField name="*_i" type="sint" indexed="true" stored="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true"/>
<dynamicField name="*_l" type="slong" indexed="true" stored="true"/>
<dynamicField name="*_t" type="text" indexed="true" stored="true"/>
<dynamicField name="*_b" type="boolean" indexed="true" stored="true"/>
<dynamicField name="*_f" type="sfloat" indexed="true" stored="true"/>
<dynamicField name="*_d" type="sdouble" indexed="true" stored="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="true"/>
<dynamicField name="random*" type="random" />
<!-- uncomment the following to ignore any fields that don't already match an existing
field name or dynamic field, rather than reporting them as an error.
alternately, change the type="ignored" to some other type e.g. "text" if you want
unknown fields indexed and/or stored by default -->
<!--dynamicField name="*" type="ignored" /-->
</fields>
<!-- Field to use to determine and enforce document uniqueness.
Unless this field is marked with required="false", it will be a required field
-->
<uniqueKey>solr_id</uniqueKey>
<!-- field for the QueryParser to use when an explicit fieldname is absent -->
<defaultSearchField>desc</defaultSearchField>
<!-- SolrQueryParser configuration: defaultOperator="AND|OR" -->
<solrQueryParser defaultOperator="OR"/>
</schema>