SOLR-5010: clean up plus POST support on schema/copyfields

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1501047 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Grant Ingersoll 2013-07-09 02:24:39 +00:00
parent b8858ab1ca
commit 450854e0c5
6 changed files with 234 additions and 46 deletions

View File

@ -17,37 +17,46 @@ package org.apache.solr.rest.schema;
*/
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.rest.GETable;
import org.apache.solr.rest.POSTable;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.ManagedIndexSchema;
import org.noggit.ObjectBuilder;
import org.restlet.data.MediaType;
import org.restlet.representation.Representation;
import org.restlet.resource.ResourceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* This class responds to requests at /solr/(corename)/schema/copyfields
*
* <p/>
*
*
* To restrict the set of copyFields in the response, specify one or both
* of the following as query parameters, with values as space and/or comma
* separated dynamic or explicit field names:
*
*
* <ul>
* <li>dest.fl: include copyFields that have one of these as a destination</li>
* <li>source.fl: include copyFields that have one of these as a source</li>
* </ul>
*
*
* If both dest.fl and source.fl are given as query parameters, the copyfields
* in the response will be restricted to those that match any of the destinations
* in dest.fl and also match any of the sources in source.fl.
*/
public class CopyFieldCollectionResource extends BaseFieldResource implements GETable {
public class CopyFieldCollectionResource extends BaseFieldResource implements GETable, POSTable {
private static final Logger log = LoggerFactory.getLogger(CopyFieldCollectionResource.class);
private static final String SOURCE_FIELD_LIST = IndexSchema.SOURCE + "." + CommonParams.FL;
private static final String DESTINATION_FIELD_LIST = IndexSchema.DESTINATION + "." + CommonParams.FL;
@ -94,4 +103,78 @@ public class CopyFieldCollectionResource extends BaseFieldResource implements GE
return new SolrOutputRepresentation();
}
@Override
public Representation post(Representation entity) throws ResourceException {
try {
if (!getSchema().isMutable()) {
final String message = "This IndexSchema is not mutable.";
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message);
} else {
if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) {
String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted."
+ " Request has media type " + entity.getMediaType().toString() + ".";
log.error(message);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message);
} else {
Object object = ObjectBuilder.fromJSON(entity.getText());
if (!(object instanceof List)) {
String message = "Invalid JSON type " + object.getClass().getName() + ", expected List of the form"
+ " (ignore the backslashes): [{\"source\":\"foo\",\"dest\":\"comma-separated list of targets\"}, {...}, ...]";
log.error(message);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message);
} else {
List<Map<String, Object>> list = (List<Map<String, Object>>) object;
Map<String, Collection<String>> fieldsToCopy = new HashMap<>();
ManagedIndexSchema oldSchema = (ManagedIndexSchema) getSchema();
Set<String> malformed = new HashSet<>();
for (Map<String,Object> map : list) {
String fieldName = (String)map.get(IndexSchema.SOURCE);
if (null == fieldName) {
String message = "Missing '" + IndexSchema.SOURCE + "' mapping.";
log.error(message);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message);
}
String destinations = (String)map.get(IndexSchema.DESTINATION);
if (destinations == null) {
String message = "Missing '" + IndexSchema.DESTINATION + "' mapping.";
log.error(message);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message);
}
String [] splits = destinations.split(",");
Set<String> destinationSet = new HashSet<>();
if (splits != null && splits.length > 0){
for (int i = 0; i < splits.length; i++) {
destinationSet.add(splits[i].trim());
}
fieldsToCopy.put(fieldName, destinationSet);
} else {
malformed.add(fieldName);
}
}
if (malformed.size() > 0){
StringBuilder message = new StringBuilder("Malformed destination(s) for: ");
for (String s : malformed) {
message.append(s).append(", ");
}
if (message.length() > 2) {
message.setLength(message.length() - 2);//drop the last ,
}
log.error(message.toString().trim());
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message.toString().trim());
}
IndexSchema newSchema = oldSchema.addCopyFields(fieldsToCopy);
if (newSchema != null) {
getSolrCore().setLatestSchema(newSchema);
}
}
}
}
} catch (Exception e) {
getSolrResponse().setException(e);
}
handlePostExecution(log);
return new SolrOutputRepresentation();
}
}

View File

@ -33,9 +33,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
@ -44,22 +47,22 @@ import java.util.TreeSet;
* <p/>
* Two query parameters are supported:
* <ul>
* <li>
* "fl": a comma- and/or space-separated list of fields to send properties
* for in the response, rather than the default: all of them.
* </li>
* <li>
* "includeDynamic": if the "fl" parameter is specified, matching dynamic
* fields are included in the response and identified with the "dynamicBase"
* property. If the "fl" parameter is not specified, the "includeDynamic"
* query parameter is ignored.
* </li>
* <li>
* "fl": a comma- and/or space-separated list of fields to send properties
* for in the response, rather than the default: all of them.
* </li>
* <li>
* "includeDynamic": if the "fl" parameter is specified, matching dynamic
* fields are included in the response and identified with the "dynamicBase"
* property. If the "fl" parameter is not specified, the "includeDynamic"
* query parameter is ignored.
* </li>
* </ul>
*/
public class FieldCollectionResource extends BaseFieldResource implements GETable,POSTable {
public class FieldCollectionResource extends BaseFieldResource implements GETable, POSTable {
private static final Logger log = LoggerFactory.getLogger(FieldCollectionResource.class);
private boolean includeDynamic;
public FieldCollectionResource() {
super();
}
@ -109,65 +112,78 @@ public class FieldCollectionResource extends BaseFieldResource implements GETabl
return new SolrOutputRepresentation();
}
@Override
public Representation post(Representation entity) {
try {
if ( ! getSchema().isMutable()) {
if (!getSchema().isMutable()) {
final String message = "This IndexSchema is not mutable.";
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
if (null == entity.getMediaType()) {
entity.setMediaType(MediaType.APPLICATION_JSON);
}
if ( ! entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) {
if (!entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) {
String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted."
+ " Request has media type " + entity.getMediaType().toString() + ".";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
Object object = ObjectBuilder.fromJSON(entity.getText());
Map<String, String> copyFields = new HashMap<>();
if ( ! (object instanceof List)) {
if (!(object instanceof List)) {
String message = "Invalid JSON type " + object.getClass().getName() + ", expected List of the form"
+ " (ignore the backslashes): [{\"name\":\"foo\",\"type\":\"text_general\", ...}, {...}, ...]";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
List<Map<String,Object>> list = (List<Map<String,Object>>)object;
List<Map<String, Object>> list = (List<Map<String, Object>>) object;
List<SchemaField> newFields = new ArrayList<SchemaField>();
IndexSchema oldSchema = getSchema();
for (Map<String,Object> map : list) {
String fieldName = (String)map.remove(IndexSchema.NAME);
Map<String, Collection<String>> copyFields = new HashMap<>();
Set<String> malformed = new HashSet<>();
for (Map<String, Object> map : list) {
String fieldName = (String) map.remove(IndexSchema.NAME);
if (null == fieldName) {
String message = "Missing '" + IndexSchema.NAME + "' mapping.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
}
String fieldType = (String)map.remove(IndexSchema.TYPE);
String fieldType = (String) map.remove(IndexSchema.TYPE);
if (null == fieldType) {
String message = "Missing '" + IndexSchema.TYPE + "' mapping.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
}
// copyFields:"comma separated list of destination fields"
String copyTo = (String)map.get(IndexSchema.COPY_FIELDS);
if (copyTo != null){
String copyTo = (String) map.get(IndexSchema.COPY_FIELDS);
if (copyTo != null) {
map.remove(IndexSchema.COPY_FIELDS);
copyFields.put(fieldName, copyTo);
String[] splits = copyTo.split(",");
Set<String> destinations = new HashSet<>();
if (splits != null && splits.length > 0) {
for (int i = 0; i < splits.length; i++) {
destinations.add(splits[i].trim());
}
copyFields.put(fieldName, destinations);
} else{
malformed.add(fieldName);
}
}
newFields.add(oldSchema.newField(fieldName, fieldType, map));
}
IndexSchema newSchema = oldSchema.addFields(newFields);
for (Map.Entry<String, String> entry : copyFields.entrySet()) {
//key is the source, value is a comma separated list of targets
String [] splits = entry.getValue().split(",");
if (splits != null && splits.length > 0){
for (int i = 0; i < splits.length; i++) {
newSchema.registerCopyField(entry.getKey(), splits[i].trim());
}
if (malformed.size() > 0){
StringBuilder message = new StringBuilder("Malformed destination(s) for: ");
for (String s : malformed) {
message.append(s).append(", ");
}
if (message.length() > 2) {
message.setLength(message.length() - 2);//drop the last ,
}
log.error(message.toString().trim());
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, message.toString().trim());
}
IndexSchema newSchema = oldSchema.addFields(newFields, copyFields);
getSolrCore().setLatestSchema(newSchema);
}
}

View File

@ -150,8 +150,17 @@ public class FieldResource extends BaseFieldResource implements GETable, PUTable
if (copyTo != null) {
map.remove(IndexSchema.COPY_FIELDS);
String [] tmp = copyTo.split(",");
copyFieldNames = new HashSet<>(tmp.length);
Collections.addAll(copyFieldNames, tmp);
if (tmp != null && tmp.length > 0) {
copyFieldNames = new HashSet<>(tmp.length);
for (int i = 0; i < tmp.length; i++) {
copyFieldNames.add(tmp[i].trim());
}
} else {
//the user specified copy fields, but then passed in something invalid
String msg = "Invalid " + IndexSchema.COPY_FIELDS + " for field: " + fieldName;
log.error(msg);
throw new SolrException(ErrorCode.BAD_REQUEST, msg);
}
}
SchemaField newField = oldSchema.newField(fieldName, fieldType, map);
IndexSchema newSchema = oldSchema.addField(newField, copyFieldNames);

View File

@ -1440,7 +1440,7 @@ public class IndexSchema {
* Copies this schema, adds the given field to the copy, then persists the new schema.
*
* @param newField the SchemaField to add
* @param copyFieldNames 0 or more names of targets to copy this field to
* @param copyFieldNames 0 or more names of targets to copy this field to. The targets must already exist.
* @return a new IndexSchema based on this schema with newField added
* @see #newField(String, String, Map)
*/
@ -1467,7 +1467,7 @@ public class IndexSchema {
* Copies this schema, adds the given fields to the copy, then persists the new schema.
*
* @param newFields the SchemaFields to add
* @param copyFieldNames 0 or more names of targets to copy this field to
* @param copyFieldNames 0 or more names of targets to copy this field to. The target fields must already exist.
* @return a new IndexSchema based on this schema with newFields added
* @see #newField(String, String, Map)
*/
@ -1477,6 +1477,17 @@ public class IndexSchema {
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Copies this schema and adds the new copy fields to the copy, then persists the new schema
* @param copyFields Key is the name of the source field name, value is a collection of target field names. Fields must exist.
* @return The new Schema with the copy fields added
*/
public IndexSchema addCopyFields(Map<String, Collection<String>> copyFields){
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Returns a SchemaField if the given fieldName does not already
* exist in this schema, and does not match any dynamic fields

View File

@ -166,12 +166,12 @@ public final class ManagedIndexSchema extends IndexSchema {
}
@Override
public IndexSchema addField(SchemaField newField) {
public ManagedIndexSchema addField(SchemaField newField) {
return addFields(Arrays.asList(newField));
}
@Override
public IndexSchema addField(SchemaField newField, Collection<String> copyFieldNames) {
public ManagedIndexSchema addField(SchemaField newField, Collection<String> copyFieldNames) {
return addFields(Arrays.asList(newField), Collections.singletonMap(newField.getName(), copyFieldNames));
}
@ -182,12 +182,12 @@ public final class ManagedIndexSchema extends IndexSchema {
}
@Override
public IndexSchema addFields(Collection<SchemaField> newFields) {
public ManagedIndexSchema addFields(Collection<SchemaField> newFields) {
return addFields(newFields, Collections.<String, Collection<String>>emptyMap());
}
@Override
public IndexSchema addFields(Collection<SchemaField> newFields, Map<String, Collection<String>> copyFieldNames) {
public ManagedIndexSchema addFields(Collection<SchemaField> newFields, Map<String, Collection<String>> copyFieldNames) {
ManagedIndexSchema newSchema = null;
if (isMutable) {
boolean success = false;
@ -242,6 +242,39 @@ public final class ManagedIndexSchema extends IndexSchema {
return newSchema;
}
@Override
public ManagedIndexSchema addCopyFields(Map<String, Collection<String>> copyFields) {
ManagedIndexSchema newSchema = null;
if (isMutable) {
boolean success = false;
while (!success) { // optimistic concurrency
// even though fields is volatile, we need to synchronize to avoid two addCopyFields
// happening concurrently (and ending up missing one of them)
synchronized (getSchemaUpdateLock()) {
newSchema = shallowCopy(true);
for (Map.Entry<String, Collection<String>> entry : copyFields.entrySet()) {
//Key is the name of the field, values are the destinations
for (String destination : entry.getValue()) {
newSchema.registerCopyField(entry.getKey(), destination);
}
}
//TODO: move this common stuff out to shared methods
// Run the callbacks on SchemaAware now that everything else is done
for (SchemaAware aware : newSchema.schemaAware) {
aware.inform(newSchema);
}
newSchema.refreshAnalyzers();
success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
if (success) {
log.debug("Added copy fields for {} sources", copyFields.size());
}
}
}
}
return newSchema;
}
@Override
public SchemaField newField(String fieldName, String fieldType, Map<String,?> options) {
SchemaField sf;

View File

@ -122,7 +122,13 @@ public class TestManagedSchemaFieldResource extends RestTestBase {
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldB",
"count(/response/arr[@name='copyFields']/lst) = 1"
);
//some bad usages
assertJPut("/schema/fields/fieldB",
"{\"type\":\"text\",\"stored\":\"false\", \"copyFields\":\",,,\"}",
"/error/msg==\"Invalid copyFields for field: fieldB\"");
assertJPut("/schema/fields/fieldC",
"{\"type\":\"text\",\"stored\":\"false\", \"copyFields\":\"some_nonexistent_field\"}",
"/error/msg==\"copyField dest :\\'some_nonexistent_field\\' is not an explicit field and doesn\\'t match a dynamicField.\"");
}
@Test
@ -192,7 +198,37 @@ public class TestManagedSchemaFieldResource extends RestTestBase {
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldF",
"count(/response/arr[@name='copyFields']/lst) = 2"
);
//some bad usages
assertJPost("/schema/fields",
"[{\"name\":\"fieldX\",\"type\":\"text\",\"stored\":\"false\"},"
+ "{\"name\":\"fieldY\",\"type\":\"text\",\"stored\":\"false\"},"
+ " {\"name\":\"fieldZ\",\"type\":\"text\",\"stored\":\"false\", \"copyFields\":\",,,\"}]",
"/error/msg==\"Malformed destination(s) for: fieldZ\"");
assertJPost("/schema/fields",
"[{\"name\":\"fieldX\",\"type\":\"text\",\"stored\":\"false\"},"
+ "{\"name\":\"fieldY\",\"type\":\"text\",\"stored\":\"false\"},"
+ " {\"name\":\"fieldZ\",\"type\":\"text\",\"stored\":\"false\", \"copyFields\":\"some_nonexistent_field\"}]",
"/error/msg==\"copyField dest :\\'some_nonexistent_field\\' is not an explicit field and doesn\\'t match a dynamicField.\"");
}
@Test
public void testPostCopyFields() throws Exception {
assertJPost("/schema/fields",
"[{\"name\":\"fieldA\",\"type\":\"text\",\"stored\":\"false\"},"
+ "{\"name\":\"fieldB\",\"type\":\"text\",\"stored\":\"false\"},"
+ "{\"name\":\"fieldC\",\"type\":\"text\",\"stored\":\"false\"},"
+ "{\"name\":\"fieldD\",\"type\":\"text\",\"stored\":\"false\"},"
+ " {\"name\":\"fieldE\",\"type\":\"text\",\"stored\":\"false\"}]",
"/responseHeader/status==0");
assertJPost("/schema/copyfields", "[{\"source\":\"fieldA\", \"dest\":\"fieldB\"},{\"source\":\"fieldD\", \"dest\":\"fieldC, fieldE\"}]", "/responseHeader/status==0");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldA",
"count(/response/arr[@name='copyFields']/lst) = 1");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=fieldD",
"count(/response/arr[@name='copyFields']/lst) = 2");
assertJPost("/schema/copyfields", "[{\"source\":\"fieldD\", \"dest\":\",,,\"}]", "/error/msg==\"Malformed destination(s) for: fieldD\"");
assertJPost("/schema/copyfields", "[{\"source\":\"some_nonexistent_field\", \"dest\":\"fieldA\"}]", "/error/msg==\"copyField source :\\'some_nonexistent_field\\' is not a glob and doesn\\'t match any explicit field or dynamicField.\"");
assertJPost("/schema/copyfields", "[{\"source\":\"fieldD\", \"dest\":\"some_nonexistent_field\"}]", "/error/msg==\"copyField dest :\\'some_nonexistent_field\\' is not an explicit field and doesn\\'t match a dynamicField.\"");
}
}