From 14c9f28f57bc102c6346a4bc6a58b57816a55735 Mon Sep 17 00:00:00 2001 From: Steven Rowe Date: Thu, 7 Mar 2013 22:41:18 +0000 Subject: [PATCH] SOLR-4537: Clean up schema information REST API git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1454141 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 29 ++++++++++++ .../apache/solr/rest/BaseFieldResource.java | 20 ++++---- .../rest/DynamicFieldCollectionResource.java | 39 ++++++++------- .../solr/rest/FieldCollectionResource.java | 47 +++++++++++-------- .../org/apache/solr/rest/FieldResource.java | 3 +- .../org/apache/solr/schema/IndexSchema.java | 3 +- .../rest/TestFieldCollectionResource.java | 37 ++++++++++++--- 7 files changed, 117 insertions(+), 61 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index bcbf37f60e4..0dcdb051cc7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -38,6 +38,35 @@ TBD... Detailed Change List ---------------------- +================== 4.3.0 ================== + +Versions of Major Components +--------------------- +Apache Tika 1.3 +Carrot2 3.6.2 +Velocity 1.7 and Velocity Tools 2.0 +Apache UIMA 2.3.1 +Apache ZooKeeper 3.4.5 + +Upgrading from Solr 4.2.0 +---------------------- + +(No upgrade instructions yet) + +Detailed Change List +---------------------- + +New Features +---------------------- + +Bug Fixes +---------------------- + +Other Changes +---------------------- + +* SOLR-4537: Clean up schema information REST API. (Steve Rowe) + ================== 4.2.0 ================== Versions of Major Components diff --git a/solr/core/src/java/org/apache/solr/rest/BaseFieldResource.java b/solr/core/src/java/org/apache/solr/rest/BaseFieldResource.java index 2e0bd719c2d..c31f1821959 100644 --- a/solr/core/src/java/org/apache/solr/rest/BaseFieldResource.java +++ b/solr/core/src/java/org/apache/solr/rest/BaseFieldResource.java @@ -18,14 +18,10 @@ package org.apache.solr.rest; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.SimpleOrderedMap; -import org.apache.solr.schema.CopyField; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.restlet.resource.ResourceException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.LinkedHashSet; /** @@ -38,10 +34,10 @@ abstract class BaseFieldResource extends BaseSchemaResource { private static final String SHOW_DEFAULTS = "showDefaults"; - private LinkedHashMap> requestedFields; + private LinkedHashSet requestedFields; private boolean showDefaults; - protected LinkedHashMap> getRequestedFields() { + protected LinkedHashSet getRequestedFields() { return requestedFields; } @@ -70,10 +66,10 @@ abstract class BaseFieldResource extends BaseSchemaResource { if (null != flParam) { String[] fields = flParam.trim().split("[,\\s]+"); if (fields.length > 0) { - requestedFields = new LinkedHashMap>(); + requestedFields = new LinkedHashSet(); for (String field : fields) { - if ( ! field.isEmpty()) { - requestedFields.put(field, null); + if ( ! field.trim().isEmpty()) { + requestedFields.add(field.trim()); } } } @@ -93,9 +89,9 @@ abstract class BaseFieldResource extends BaseSchemaResource { SimpleOrderedMap properties = field.getNamedPropertyValues(showDefaults); if ( ! getSchema().getFields().containsKey(field.getName())) { String dynamicBase = getSchema().getDynamicPattern(field.getName()); + // Add dynamicBase property if it's different from the field name. if ( ! field.getName().equals(dynamicBase)) { - // Don't add dynamicBase property if it's identical to the field name. - properties.add(DYNAMIC_BASE, getSchema().getDynamicPattern(field.getName())); + properties.add(DYNAMIC_BASE, dynamicBase); } } if (field == getSchema().getUniqueKeyField()) { diff --git a/solr/core/src/java/org/apache/solr/rest/DynamicFieldCollectionResource.java b/solr/core/src/java/org/apache/solr/rest/DynamicFieldCollectionResource.java index f221b721d9c..bdf6bdcee5b 100644 --- a/solr/core/src/java/org/apache/solr/rest/DynamicFieldCollectionResource.java +++ b/solr/core/src/java/org/apache/solr/rest/DynamicFieldCollectionResource.java @@ -21,6 +21,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; import org.restlet.representation.Representation; import org.restlet.resource.ResourceException; @@ -28,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,32 +57,29 @@ public class DynamicFieldCollectionResource extends BaseFieldResource implements public Representation get() { try { - SchemaField[] dynamicFields = getSchema().getDynamicFieldPrototypes(); - List> props = new ArrayList>(dynamicFields.length); - if (null != getRequestedFields()) { + List> props = new ArrayList>(); + if (null == getRequestedFields()) { + for (IndexSchema.DynamicField dynamicField : getSchema().getDynamicFields()) { + if ( ! dynamicField.getRegex().startsWith(INTERNAL_POLY_FIELD_PREFIX)) { // omit internal polyfields + props.add(getFieldProperties(dynamicField.getPrototype())); + } + } + } else { if (0 == getRequestedFields().size()) { String message = "Empty " + CommonParams.FL + " parameter value"; throw new SolrException(ErrorCode.BAD_REQUEST, message); } - for (SchemaField prototype : dynamicFields) { - if (getRequestedFields().containsKey(prototype.getName())) { - getRequestedFields().put(prototype.getName(), getFieldProperties(prototype)); - } + Map dynamicFieldsByName = new HashMap(); + for (IndexSchema.DynamicField dynamicField : getSchema().getDynamicFields()) { + dynamicFieldsByName.put(dynamicField.getRegex(), dynamicField.getPrototype()); } // Use the same order as the fl parameter - for (Map.Entry> requestedField : getRequestedFields().entrySet()) { - SimpleOrderedMap fieldProperties = requestedField.getValue(); - // Should there be some form of error condition - // if one or more of the requested fields were not found? - if (null != fieldProperties) { - props.add(fieldProperties); - } - } - } else { - for (SchemaField prototype : dynamicFields) { - // omit internal polyfields - if ( ! prototype.getName().startsWith(INTERNAL_POLY_FIELD_PREFIX)) { - props.add(getFieldProperties(prototype)); + for (String dynamicFieldName : getRequestedFields()) { + final SchemaField dynamicSchemaField = dynamicFieldsByName.get(dynamicFieldName); + if (null == dynamicSchemaField) { + log.info("Requested dynamic field '" + dynamicFieldName + "' not found."); + } else { + props.add(getFieldProperties(dynamicSchemaField)); } } } diff --git a/solr/core/src/java/org/apache/solr/rest/FieldCollectionResource.java b/solr/core/src/java/org/apache/solr/rest/FieldCollectionResource.java index b2d69f1973d..36b548337bc 100644 --- a/solr/core/src/java/org/apache/solr/rest/FieldCollectionResource.java +++ b/solr/core/src/java/org/apache/solr/rest/FieldCollectionResource.java @@ -21,29 +21,37 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.SchemaField; import org.restlet.representation.Representation; import org.restlet.resource.ResourceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; /** * This class responds to requests at /solr/(corename)/schema/fields *

- * One query parameters are supported: + * Two query parameters are supported: *

    *
  • * "fl": a comma- and/or space-separated list of fields to send properties * for in the response, rather than the default: all of them. *
    • + *
  • + * "includeDynamic": if the "fl" parameter is specified, matching dynamic + * fields are included in the response and identified with the "dynamicBase" + * property. If the "fl" parameter is not specified, the "includeDynamic" + * query parameter is ignored. + *
    • *
*/ public class FieldCollectionResource extends BaseFieldResource implements GETable { private static final Logger log = LoggerFactory.getLogger(FieldCollectionResource.class); + private boolean includeDynamic; public FieldCollectionResource() { super(); @@ -52,40 +60,39 @@ public class FieldCollectionResource extends BaseFieldResource implements GETabl @Override public void doInit() throws ResourceException { super.doInit(); + includeDynamic = getSolrRequest().getParams().getBool(INCLUDE_DYNAMIC_PARAM, false); } @Override public Representation get() { try { - // Get all explicitly defined fields from the schema - Set fieldNames = new HashSet(getSchema().getFields().keySet()); - - final List> fieldCollectionProperties = new ArrayList>(fieldNames.size()); - + final List> props = new ArrayList>(); if (null == getRequestedFields()) { + SortedSet fieldNames = new TreeSet(getSchema().getFields().keySet()); for (String fieldName : fieldNames) { - fieldCollectionProperties.add(getFieldProperties(getSchema().getFieldOrNull(fieldName))); + props.add(getFieldProperties(getSchema().getFields().get(fieldName))); } } else { if (0 == getRequestedFields().size()) { String message = "Empty " + CommonParams.FL + " parameter value"; throw new SolrException(ErrorCode.BAD_REQUEST, message); } - for (String field : fieldNames) { - if (getRequestedFields().containsKey(field)) { - getRequestedFields().put(field, getFieldProperties(getSchema().getFieldOrNull(field))); - } - } // Use the same order as the fl parameter - for (SimpleOrderedMap fieldProperties : getRequestedFields().values()) { - // Should there be some form of error condition - // if one or more of the requested fields were not found? - if (null != fieldProperties) { - fieldCollectionProperties.add(fieldProperties); + for (String fieldName : getRequestedFields()) { + final SchemaField field; + if (includeDynamic) { + field = getSchema().getFieldOrNull(fieldName); + } else { + field = getSchema().getFields().get(fieldName); + } + if (null == field) { + log.info("Requested field '" + fieldName + "' not found."); + } else { + props.add(getFieldProperties(field)); } } } - getSolrResponse().add(SchemaRestApi.FIELDS, fieldCollectionProperties); + getSolrResponse().add(SchemaRestApi.FIELDS, props); } catch (Exception e) { getSolrResponse().setException(e); } diff --git a/solr/core/src/java/org/apache/solr/rest/FieldResource.java b/solr/core/src/java/org/apache/solr/rest/FieldResource.java index 423d8c54835..cbb1d2ae747 100644 --- a/solr/core/src/java/org/apache/solr/rest/FieldResource.java +++ b/solr/core/src/java/org/apache/solr/rest/FieldResource.java @@ -68,11 +68,10 @@ public class FieldResource extends BaseFieldResource implements GETable { final String message = "Field name is missing"; throw new SolrException(ErrorCode.BAD_REQUEST, message); } else { - SchemaField field; + final SchemaField field; if (includeDynamic) { field = getSchema().getFieldOrNull(fieldName); } else { - // Don't look for matches among dynamic fields field = getSchema().getFields().get(fieldName); } if (null == field) { diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java index f02da185008..7660f55022c 100644 --- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java @@ -841,7 +841,8 @@ public final class IndexSchema { public final static class DynamicField extends DynamicReplacement { - final SchemaField prototype; + private final SchemaField prototype; + public SchemaField getPrototype() { return prototype; } DynamicField(SchemaField prototype) { super(prototype.name); diff --git a/solr/core/src/test/org/apache/solr/rest/TestFieldCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/TestFieldCollectionResource.java index 40c256182e2..feee49608c5 100644 --- a/solr/core/src/test/org/apache/solr/rest/TestFieldCollectionResource.java +++ b/solr/core/src/test/org/apache/solr/rest/TestFieldCollectionResource.java @@ -24,9 +24,9 @@ public class TestFieldCollectionResource extends SchemaRestletTestBase { @Test public void testGetAllFields() throws Exception { assertQ("/schema/fields?indent=on&wt=xml", - "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'custstopfilt'", - "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = 'lowerfilt'", - "(/response/arr[@name='fields']/lst/str[@name='name'])[3] = 'test_basictv'", + "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'HTMLstandardtok'", + "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = 'HTMLwhitetok'", + "(/response/arr[@name='fields']/lst/str[@name='name'])[3] = '_version_'", "count(//copySources/str) = count(//copyDests/str)"); } @@ -37,6 +37,31 @@ public class TestFieldCollectionResource extends SchemaRestletTestBase { "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'id'", "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = '_version_'"); } + + @Test + public void testGetThreeFieldsDontIncludeDynamic() throws IOException { + // + assertQ("/schema/fields?indent=on&wt=xml&fl=id,_version_,price_i", + "count(/response/arr[@name='fields']/lst/str[@name='name']) = 2", + "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'id'", + "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = '_version_'"); + } + + @Test + public void testGetThreeFieldsIncludeDynamic() throws IOException { + assertQ("/schema/fields?indent=on&wt=xml&fl=id,_version_,price_i&includeDynamic=on", + + "count(/response/arr[@name='fields']/lst/str[@name='name']) = 3", + + "(/response/arr[@name='fields']/lst/str[@name='name'])[1] = 'id'", + + "(/response/arr[@name='fields']/lst/str[@name='name'])[2] = '_version_'", + + "(/response/arr[@name='fields']/lst/str[@name='name'])[3] = 'price_i'", + + "/response/arr[@name='fields']/lst[ str[@name='name']='price_i' " + +" and str[@name='dynamicBase']='*_i']"); + } @Test public void testNotFoundFields() throws IOException { @@ -48,9 +73,9 @@ public class TestFieldCollectionResource extends SchemaRestletTestBase { @Test public void testJsonGetAllFields() throws Exception { assertJQ("/schema/fields?indent=on", - "/fields/[0]/name=='custstopfilt'", - "/fields/[1]/name=='lowerfilt'", - "/fields/[2]/name=='test_basictv'"); + "/fields/[0]/name=='HTMLstandardtok'", + "/fields/[1]/name=='HTMLwhitetok'", + "/fields/[2]/name=='_version_'"); } @Test