diff --git a/CHANGES.txt b/CHANGES.txt index 8e2f905b899..108edddc60d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -161,6 +161,12 @@ New Features 26. SOLR-170: StandardRequestHandler now supports a "sort" parameter. Using the ';' syntax is still supported, but it is recommended to transition to the new syntax. (ryan) + +27. SOLR-181: The index schema now supports "required" fields. Attempts + to add a document without a required field will fail, returning a + descriptive error message. By default, the uniqueKey field is + a required field. This can be disabled by setting required=false + in schema.xml. (Greg Ludington via ryan) Changes in runtime behavior 1. Highlighting using DisMax will only pick up terms from the main diff --git a/example/solr/conf/schema.xml b/example/solr/conf/schema.xml index 38902e46a6a..4195bd896c4 100755 --- a/example/solr/conf/schema.xml +++ b/example/solr/conf/schema.xml @@ -234,7 +234,7 @@ fields or fields that need an index-time boost need norms. --> - + @@ -291,7 +291,9 @@ - + id diff --git a/src/java/org/apache/solr/schema/FieldProperties.java b/src/java/org/apache/solr/schema/FieldProperties.java index 08b012b3461..379ad7ec83a 100644 --- a/src/java/org/apache/solr/schema/FieldProperties.java +++ b/src/java/org/apache/solr/schema/FieldProperties.java @@ -43,13 +43,15 @@ abstract class FieldProperties { final static int MULTIVALUED = 0x00000200; final static int SORT_MISSING_FIRST = 0x00000400; final static int SORT_MISSING_LAST = 0x00000800; - + + final static int REQUIRED = 0x00001000; + static final String[] propertyNames = { "indexed", "tokenized", "stored", "binary", "compressed", "omitNorms", "termVectors", "termPositions", "termOffsets", "multiValued", - "sortMissingFirst","sortMissingLast" + "sortMissingFirst","sortMissingLast","required" }; static final Map propertyMap = new HashMap(); diff --git a/src/java/org/apache/solr/schema/IndexSchema.java b/src/java/org/apache/solr/schema/IndexSchema.java index fb669ba197a..3584169216d 100644 --- a/src/java/org/apache/solr/schema/IndexSchema.java +++ b/src/java/org/apache/solr/schema/IndexSchema.java @@ -19,7 +19,6 @@ package org.apache.solr.schema; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Similarity; @@ -92,6 +91,7 @@ public final class IndexSchema { private final HashMap fields = new HashMap(); private final HashMap fieldTypes = new HashMap(); private final List fieldsWithDefaultValue = new ArrayList(); + private final Collection requiredFields = new HashSet(); /** * Provides direct access to the Map containing all explicit @@ -118,6 +118,12 @@ public final class IndexSchema { */ public List getFieldsWithDefaultValue() { return fieldsWithDefaultValue; } + /** + * Provides direct access to the List containing all required fields. This + * list contains all fields with default values. + */ + public Collection getRequiredFields() { return requiredFields; } + private Similarity similarity; /** @@ -338,6 +344,8 @@ public final class IndexSchema { } + // Hang on to the fields that say if they are required -- this lets us set a reasonable default for the unique key + Map explicitRequiredProp = new HashMap(); ArrayList dFields = new ArrayList(); expression = "/schema/fields/field | /schema/fields/dynamicField"; nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET); @@ -358,6 +366,9 @@ public final class IndexSchema { } Map args = DOMUtil.toMapExcept(attrs, "name", "type"); + if( args.get( "required" ) != null ) { + explicitRequiredProp.put( name, Boolean.valueOf( args.get( "required" ) ) ); + } SchemaField f = SchemaField.create(name,ft,args); @@ -366,7 +377,11 @@ public final class IndexSchema { log.fine("field defined: " + f); if( f.getDefaultValue() != null ) { log.fine(name+" contains default value: " + f.getDefaultValue()); - fieldsWithDefaultValue.add( f ); + fieldsWithDefaultValue.add( f ); + } + if (f.isRequired()) { + log.fine(name+" is required in this schema"); + requiredFields.add(f); } } else if (node.getNodeName().equals("dynamicField")) { dFields.add(new DynamicField(f)); @@ -376,6 +391,11 @@ public final class IndexSchema { throw new RuntimeException("Unknown field type"); } } + + //fields with default values are by definition required + //add them to required fields, and we only have to loop once + // in DocumentBuilder.getDoc() + requiredFields.addAll(getFieldsWithDefaultValue()); // OK, now sort the dynamic fields largest to smallest size so we don't get // any false matches. We want to act like a compiler tool and try and match @@ -423,6 +443,12 @@ public final class IndexSchema { uniqueKeyFieldName=uniqueKeyField.getName(); uniqueKeyFieldType=uniqueKeyField.getType(); log.info("unique key field: "+uniqueKeyFieldName); + + // Unless the uniqueKeyField is marked 'required=false' then make sure it exists + if( Boolean.FALSE != explicitRequiredProp.get( uniqueKeyFieldName ) ) { + uniqueKeyField.required = true; + requiredFields.add(uniqueKeyField); + } } /////////////// parse out copyField commands /////////////// @@ -824,3 +850,4 @@ public final class IndexSchema { } + diff --git a/src/java/org/apache/solr/schema/SchemaField.java b/src/java/org/apache/solr/schema/SchemaField.java index 18bccc01568..b3a9cc35bcb 100644 --- a/src/java/org/apache/solr/schema/SchemaField.java +++ b/src/java/org/apache/solr/schema/SchemaField.java @@ -37,6 +37,7 @@ public final class SchemaField extends FieldProperties { final FieldType type; final int properties; final String defaultValue; + boolean required = false; // this can't be final since it may be changed dynamically /** Create a new SchemaField with the given name and type, @@ -64,6 +65,9 @@ public final class SchemaField extends FieldProperties { this.type = type; this.properties = properties; this.defaultValue = defaultValue; + + // initalize with the required property flag + required = (properties & REQUIRED) !=0; } public String getName() { return name; } @@ -80,6 +84,7 @@ public final class SchemaField extends FieldProperties { public boolean sortMissingFirst() { return (properties & SORT_MISSING_FIRST)!=0; } public boolean sortMissingLast() { return (properties & SORT_MISSING_LAST)!=0; } public boolean isCompressed() { return (properties & COMPRESSED)!=0; } + public boolean isRequired() { return required; } // things that should be determined by field type, not set as options boolean isTokenized() { return (properties & TOKENIZED)!=0; } @@ -89,10 +94,12 @@ public final class SchemaField extends FieldProperties { return type.createField(this,val,boost); } + @Override public String toString() { return name + "{type="+type.getTypeName() + ((defaultValue==null)?"":(",default="+defaultValue)) + ",properties=" + propertiesToString(properties) + + ( required ? ", required=true" : "" ) + "}"; } @@ -111,7 +118,7 @@ public final class SchemaField extends FieldProperties { } - static SchemaField create(String name, FieldType ft, Map props) { + static SchemaField create(String name, FieldType ft, Map props) { int trueProps = parseProperties(props,true); int falseProps = parseProperties(props,false); @@ -175,3 +182,5 @@ public final class SchemaField extends FieldProperties { + + diff --git a/src/java/org/apache/solr/update/DocumentBuilder.java b/src/java/org/apache/solr/update/DocumentBuilder.java index 0fe1991fd0d..0778f8ebb40 100644 --- a/src/java/org/apache/solr/update/DocumentBuilder.java +++ b/src/java/org/apache/solr/update/DocumentBuilder.java @@ -17,13 +17,15 @@ package org.apache.solr.update; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.solr.core.SolrException; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; -import org.apache.solr.core.SolrException; - -import java.util.HashMap; /** * @author yonik @@ -104,15 +106,37 @@ public class DocumentBuilder { } // specific to this type of document builder - public Document getDoc() { - - // Check for default fields in our schema... - for( SchemaField field : schema.getFieldsWithDefaultValue() ) { - if( doc.getField( field.getName() ) == null ) { - doc.add( field.createField( field.getDefaultValue(), 1.0f ) ); + public Document getDoc() throws IllegalArgumentException { + + // Check for all required fields -- Note, all fields with a + // default value are defacto 'required' fields. + List missingFields = new ArrayList( schema.getRequiredFields().size() ); + for (SchemaField field : schema.getRequiredFields()) { + if (doc.getField(field.getName() ) == null) { + if (field.getDefaultValue() != null) { + doc.add( field.createField( field.getDefaultValue(), 1.0f ) ); + } else { + missingFields.add(field.getName()); + } } } - + + if (missingFields.size() > 0) { + StringBuilder builder = new StringBuilder(); + // add the uniqueKey if possible + if( schema.getUniqueKeyField() != null ) { + String n = schema.getUniqueKeyField().getName(); + String v = doc.get( n ); + builder.append( "Document ["+n+"="+v+"] " ); + } + builder.append("missing required fields: " ); + for (String field : missingFields) { + builder.append(field); + builder.append(" "); + } + throw new SolrException(400, builder.toString()); + } + Document ret = doc; doc=null; return ret; } diff --git a/src/java/org/apache/solr/util/AbstractSolrTestCase.java b/src/java/org/apache/solr/util/AbstractSolrTestCase.java index 6f901d11443..5a4cc27505b 100644 --- a/src/java/org/apache/solr/util/AbstractSolrTestCase.java +++ b/src/java/org/apache/solr/util/AbstractSolrTestCase.java @@ -126,16 +126,36 @@ public abstract class AbstractSolrTestCase extends TestCase { public void assertU(String update) { assertU(null, update); } - + /** Validates an update XML String is successful */ public void assertU(String message, String update) { + checkUpdateU(message, update, true); + } + + /** Validates an update XML String failed + */ + public void assertFailedU(String update) { + assertFailedU(null, update); + } + + /** Validates an update XML String failed + */ + public void assertFailedU(String message, String update) { + checkUpdateU(message, update, false); + } + + /** Checks the success or failure of an update message + */ + private void checkUpdateU(String message, String update, boolean shouldSucceed) { try { String m = (null == message) ? "" : message + " "; - - String res = h.validateUpdate(update); - if (null != res) { - fail(m + "update was not successful: " + res); + if (shouldSucceed) { + String res = h.validateUpdate(update); + if (res != null) fail(m + "update was not successful: " + res); + } else { + String res = h.validateErrorUpdate(update); + if (res != null) fail(m + "update succeeded, but should have failed: " + res); } } catch (SAXException e) { throw new RuntimeException("Invalid XML", e); @@ -284,6 +304,4 @@ public abstract class AbstractSolrTestCase extends TestCase { } return f.delete(); } - - } diff --git a/src/java/org/apache/solr/util/TestHarness.java b/src/java/org/apache/solr/util/TestHarness.java index d1f424c245f..e4b7fbb48ce 100644 --- a/src/java/org/apache/solr/util/TestHarness.java +++ b/src/java/org/apache/solr/util/TestHarness.java @@ -138,9 +138,33 @@ public class TestHarness { * @return null if succesful, otherwise the XML response to the update */ public String validateUpdate(String xml) throws SAXException { + return checkUpdateStatus(xml, "0"); + } + + /** + * Validates that an "update" (add, commit or optimize) results in success. + * + * :TODO: currently only deals with one add/doc at a time, this will need changed if/when SOLR-2 is resolved + * + * @param xml The XML of the update + * @return null if succesful, otherwise the XML response to the update + */ + public String validateErrorUpdate(String xml) throws SAXException { + return checkUpdateStatus(xml, "1"); + } + + /** + * Validates that an "update" (add, commit or optimize) results in success. + * + * :TODO: currently only deals with one add/doc at a time, this will need changed if/when SOLR-2 is resolved + * + * @param xml The XML of the update + * @return null if succesful, otherwise the XML response to the update + */ + public String checkUpdateStatus(String xml, String code) throws SAXException { try { String res = update(xml); - String valid = validateXPath(res, "//result[@status=0]" ); + String valid = validateXPath(res, "//result[@status="+code+"]" ); return (null == valid) ? null : res; } catch (XPathExpressionException e) { throw new RuntimeException @@ -148,7 +172,6 @@ public class TestHarness { } } - /** * Validates that an add of a single document results in success. * diff --git a/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java b/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java new file mode 100644 index 00000000000..70fa40e4eb8 --- /dev/null +++ b/src/test/org/apache/solr/schema/NotRequiredUniqueKeyTest.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; + +import org.apache.solr.core.SolrCore; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.util.AbstractSolrTestCase; + +/** + * This is a simple test to make sure the unique key is not required + * when it is specified as 'false' + * + * It needs its own file so it can load a special schema file + */ +public class NotRequiredUniqueKeyTest extends AbstractSolrTestCase { + + @Override public String getSchemaFile() { return "schema-not-required-unique-key.xml"; } + @Override public String getSolrConfigFile() { return "solrconfig.xml"; } + + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + + public void testSchemaLoading() + { + SolrCore core = SolrCore.getSolrCore(); + IndexSchema schema = core.getSchema(); + SchemaField uniqueKey = schema.getUniqueKeyField(); + + assertFalse( uniqueKey.isRequired() ); + + assertFalse( schema.getRequiredFields().contains( uniqueKey ) ); + } +} diff --git a/src/test/org/apache/solr/schema/RequiredFieldsTest.java b/src/test/org/apache/solr/schema/RequiredFieldsTest.java new file mode 100644 index 00000000000..c47176fb614 --- /dev/null +++ b/src/test/org/apache/solr/schema/RequiredFieldsTest.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; + +import org.apache.solr.core.SolrCore; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.util.AbstractSolrTestCase; + +/** + * + * @author Greg Ludington + */ +public class RequiredFieldsTest extends AbstractSolrTestCase { + + @Override public String getSchemaFile() { return "schema-required-fields.xml"; } + @Override public String getSolrConfigFile() { return "solrconfig.xml"; } + + @Override + public void setUp() throws Exception { + super.setUp(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + } + + + public void testRequiredFieldsConfig() { + + SolrCore core = SolrCore.getSolrCore(); + IndexSchema schema = core.getSchema(); + SchemaField uniqueKey = schema.getUniqueKeyField(); + + // Make sure the uniqueKey is required + assertTrue( uniqueKey.isRequired() ); + assertTrue( schema.getRequiredFields().contains( uniqueKey ) ); + + // we specified one required field, but all devault valued fields are also required + Collection requiredFields =schema.getRequiredFields(); + int numDefaultFields = schema.getFieldsWithDefaultValue().size(); + assertEquals( numDefaultFields+1+1, requiredFields.size()); // also the uniqueKey + } + + public void testRequiredFieldsSingleAdd() { + // Add a single document + assertU("adding document", + adoc("id", "529", "name", "document with id, name, and subject", "field_t", "what's inside?", "subject", "info")); + assertU(commit()); + + // Check it it is in the index + assertQ("should find one", req("id:529") ,"//result[@numFound=1]" ); + + // Add another document without the required subject field, which + // has a configured defaultValue of "Stuff" + assertU("adding a doc without field w/ configured default", + adoc("id", "530", "name", "document with id and name", "field_t", "what's inside?")); + assertU(commit()); + + // Add another document without a subject, which has a default in schema + String subjectDefault = SolrCore.getSolrCore().getSchema().getField("subject").getDefaultValue(); + assertNotNull("subject has no default value", subjectDefault); + assertQ("should find one with subject="+subjectDefault, req("id:530 subject:"+subjectDefault) ,"//result[@numFound=1]" ); + + // Add another document without a required name, which has no default + assertNull(SolrCore.getSolrCore().getSchema().getField("name").getDefaultValue()); + assertFailedU("adding doc without required field", + adoc("id", "531", "subject", "no name document", "field_t", "what's inside?") ); + assertU(commit()); + + // Check to make sure this submission did not succeed + assertQ("should not find any", req("id:531") ,"//result[@numFound=0]" ); + } + + public void testAddMultipleDocumentsWithErrors() { + //Add three documents at once to make sure the baseline succeeds + assertU("adding 3 documents", + "" +doc("id", "601", "name", "multiad one", "field_t", "what's inside?", "subject", "info") + + doc("id", "602", "name", "multiad two", "field_t", "what's inside?", "subject", "info") + + doc("id", "603", "name", "multiad three", "field_t", "what's inside?", "subject", "info") + + ""); + assertU(commit()); + + // Check that they are in the index + assertQ("should find three", req("name:multiad") ,"//result[@numFound=3]" ); + + // Add three documents at once, with the middle one missing a field that has a default + assertU("adding 3 docs, with 2nd one missing a field that has a default value", + "" +doc("id", "601", "name", "nosubject batch one", "field_t", "what's inside?", "subject", "info") + + doc("id", "602", "name", "nosubject batch two", "field_t", "what's inside?") + + doc("id", "603", "name", "nosubject batch three", "field_t", "what's inside?", "subject", "info") + + ""); + assertU(commit()); + + // Since the missing field had a devault value, + // All three should have made it into the index + assertQ("should find three", req("name:nosubject") ,"//result[@numFound=3]" ); + + // Add three documents at once, with the middle with a bad field definition, + // to establish the baselinie behavior for errors in a multi-ad submission + assertFailedU("adding 3 documents, with 2nd one with undefined field", + "" +doc("id", "801", "name", "baddef batch one", "field_t", "what's inside?", "subject", "info") + + doc("id", "802", "field_t", "name", "baddef batch two", "what's inside?", "subject", "info", "GaRbAgeFiElD", "garbage") + + doc("id", "803", "name", "baddef batch three", "field_t", "what's inside?", "subject", "info") + + ""); + assertU(commit()); + + // Check that only docs before the error should be in the index + assertQ("should find one", req("name:baddef") ,"//result[@numFound=1]" ); + + // Add three documents at once, with the middle one missing a required field that has no default + assertFailedU("adding 3 docs, with 2nd one missing required field", + "" +doc("id", "701", "name", "noname batch one", "field_t", "what's inside?", "subject", "info") + + doc("id", "702", "field_t", "what's inside?", "subject", "info") + + doc("id", "703", "name", "noname batch batch three", "field_t", "what's inside?", "subject", "info") + + ""); + + assertU(commit()); + + // Check that only docs before the error should be in the index + assertQ("should find one", req("name:noname") ,"//result[@numFound=1]" ); + } +} diff --git a/src/test/test-files/solr/conf/schema-not-required-unique-key.xml b/src/test/test-files/solr/conf/schema-not-required-unique-key.xml new file mode 100644 index 00000000000..7853fbfad8d --- /dev/null +++ b/src/test/test-files/solr/conf/schema-not-required-unique-key.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + subject + id + diff --git a/src/test/test-files/solr/conf/schema-required-fields.xml b/src/test/test-files/solr/conf/schema-required-fields.xml new file mode 100644 index 00000000000..29d26a2705e --- /dev/null +++ b/src/test/test-files/solr/conf/schema-required-fields.xml @@ -0,0 +1,434 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text + id + + + + + + + + + + + + + + diff --git a/src/test/test-files/solr/conf/schema.xml b/src/test/test-files/solr/conf/schema.xml index db83cf567bc..f71cd46f972 100644 --- a/src/test/test-files/solr/conf/schema.xml +++ b/src/test/test-files/solr/conf/schema.xml @@ -298,7 +298,7 @@ - +