Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-5097: Schema API: Add REST support for adding dynamic fields to the schema. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1622135 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Steven Rowe 2014-09-02 21:25:05 +00:00
parent 419fdd3b8c
commit d88b031c89
11 changed files with 1065 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -137,6 +137,9 @@ New Features
* SOLR-6365: specify appends, defaults, invariants outside of the request handler.
(Noble Paul, Erik Hatcher, shalin)
* SOLR-5097: Schema API: Add REST support for adding dynamic fields to the schema.
(Steve Rowe)
Bug Fixes
----------------------

20
solr/core/src/java/org/apache/solr/rest/schema/BaseFieldResource.java
View File

@ -24,6 +24,7 @@ import org.apache.solr.schema.SchemaField;
import org.restlet.resource.ResourceException;
import java.util.LinkedHashSet;
import java.util.Map;
/**
@ -98,4 +99,23 @@ abstract class BaseFieldResource extends BaseSolrResource {
}
return properties;
}
// protected access on this class triggers a bug in javadoc generation caught by
// documentation-link: "BROKEN LINK" reported in javadoc for classes using
// NewFieldArguments because the link target file is BaseFieldResource.NewFieldArguments,
// but the actual file is BaseFieldResource$NewFieldArguments.
static class NewFieldArguments {
private String name;
private String type;
Map<String,Object> map;
NewFieldArguments(String name, String type, Map<String,Object> map) {
this.name = name;
this.type = type;
this.map = map;
}
public String getName() { return name; }
public String getType() { return type; }
public Map<String, Object> getMap() { return map; }
}
}

112
solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldCollectionResource.java
View File

@ -21,14 +21,20 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.rest.GETable;
import org.apache.solr.rest.POSTable;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.solr.schema.SchemaField;
import org.noggit.ObjectBuilder;
import org.restlet.data.MediaType;
import org.restlet.representation.Representation;
import org.restlet.resource.ResourceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -40,7 +46,7 @@ import java.util.Map;
* and/or space separated list of dynamic field patterns in the "fl" query
* parameter.
*/
public class DynamicFieldCollectionResource extends BaseFieldResource implements GETable {
public class DynamicFieldCollectionResource extends BaseFieldResource implements GETable, POSTable {
private static final Logger log = LoggerFactory.getLogger(DynamicFieldCollectionResource.class);
public DynamicFieldCollectionResource() {
@ -90,4 +96,108 @@ public class DynamicFieldCollectionResource extends BaseFieldResource implements
return new SolrOutputRepresentation();
}
@Override
public Representation post(Representation entity) {
try {
if ( ! getSchema().isMutable()) {
final String message = "This IndexSchema is not mutable.";
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
if (null == entity.getMediaType()) {
entity.setMediaType(MediaType.APPLICATION_JSON);
}
if ( ! entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) {
String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted."
+ " Request has media type " + entity.getMediaType().toString() + ".";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
Object object = ObjectBuilder.fromJSON(entity.getText());
if ( ! (object instanceof List)) {
String message = "Invalid JSON type " + object.getClass().getName() + ", expected List of the form"
+ " (ignore the backslashes): [{\"name\":\"*_foo\",\"type\":\"text_general\", ...}, {...}, ...]";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
List<Map<String,Object>> list = (List<Map<String,Object>>)object;
List<SchemaField> newDynamicFields = new ArrayList<>();
List<NewFieldArguments> newDynamicFieldArguments = new ArrayList<>();
ManagedIndexSchema oldSchema = (ManagedIndexSchema)getSchema();
Map<String,Collection<String>> copyFields = new HashMap<>();
for (Map<String,Object> map : list) {
String fieldNamePattern = (String)map.remove(IndexSchema.NAME);
if (null == fieldNamePattern) {
String message = "Missing '" + IndexSchema.NAME + "' mapping.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
}
String fieldType = (String)map.remove(IndexSchema.TYPE);
if (null == fieldType) {
String message = "Missing '" + IndexSchema.TYPE + "' mapping.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
}
// copyFields:"comma separated list of destination fields"
Object copies = map.get(IndexSchema.COPY_FIELDS);
List<String> copyTo = null;
if (copies != null) {
if (copies instanceof List){
copyTo = (List<String>)copies;
} else if (copies instanceof String){
copyTo = Collections.singletonList(copies.toString());
} else {
String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
}
}
if (copyTo != null) {
map.remove(IndexSchema.COPY_FIELDS);
copyFields.put(fieldNamePattern, copyTo);
}
newDynamicFields.add(oldSchema.newDynamicField(fieldNamePattern, fieldType, map));
newDynamicFieldArguments.add(new NewFieldArguments(fieldNamePattern, fieldType, map));
}
boolean firstAttempt = true;
boolean success = false;
while ( ! success) {
try {
if ( ! firstAttempt) {
// If this isn't the first attempt, we must have failed due to
// the schema changing in Zk during optimistic concurrency control.
// In that case, rerun creating the new fields, because they may
// fail now due to changes in the schema. This behavior is consistent
// with what would happen if we locked the schema and the other schema
// change went first.
newDynamicFields.clear();
for (NewFieldArguments args : newDynamicFieldArguments) {
newDynamicFields.add(oldSchema.newDynamicField(args.getName(), args.getType(), args.getMap()));
}
}
firstAttempt = false;
synchronized (oldSchema.getSchemaUpdateLock()) {
IndexSchema newSchema = oldSchema.addDynamicFields(newDynamicFields, copyFields);
if (null != newSchema) {
getSolrCore().setLatestSchema(newSchema);
success = true;
} else {
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add dynamic fields.");
}
}
} catch (ManagedIndexSchema.SchemaChangedInZkException e) {
log.debug("Schema changed while processing request, retrying");
oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema();
}
}
}
}
}
} catch (Exception e) {
getSolrResponse().setException(e);
}
handlePostExecution(log);
return new SolrOutputRepresentation();
}
}

103
solr/core/src/java/org/apache/solr/rest/schema/DynamicFieldResource.java
View File

@ -20,20 +20,27 @@ package org.apache.solr.rest.schema;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.rest.GETable;
import org.apache.solr.rest.PUTable;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.solr.schema.SchemaField;
import org.noggit.ObjectBuilder;
import org.restlet.data.MediaType;
import org.restlet.representation.Representation;
import org.restlet.resource.ResourceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.UnsupportedEncodingException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* This class responds to requests at /solr/(corename)/schema/dynamicfields/(pattern)
* where pattern is a field name pattern (with an asterisk at the beginning or the end).
*/
public class DynamicFieldResource extends BaseFieldResource implements GETable {
public class DynamicFieldResource extends BaseFieldResource implements GETable, PUTable {
private static final Logger log = LoggerFactory.getLogger(DynamicFieldResource.class);
private String fieldNamePattern;
@ -86,4 +93,98 @@ public class DynamicFieldResource extends BaseFieldResource implements GETable {
return new SolrOutputRepresentation();
}
/**
* Accepts JSON add dynamic field request
*/
@Override
public Representation put(Representation entity) {
try {
if ( ! getSchema().isMutable()) {
final String message = "This IndexSchema is not mutable.";
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
if (null == entity.getMediaType()) {
entity.setMediaType(MediaType.APPLICATION_JSON);
}
if ( ! entity.getMediaType().equals(MediaType.APPLICATION_JSON, true)) {
String message = "Only media type " + MediaType.APPLICATION_JSON.toString() + " is accepted."
+ " Request has media type " + entity.getMediaType().toString() + ".";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
Object object = ObjectBuilder.fromJSON(entity.getText());
if ( ! (object instanceof Map)) {
String message = "Invalid JSON type " + object.getClass().getName() + ", expected Map of the form"
+ " (ignore the backslashes): {\"type\":\"text_general\", ...}, either with or"
+ " without a \"name\" mapping. If the \"name\" is specified, it must match the"
+ " name given in the request URL: /schema/dynamicfields/(name)";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
Map<String,Object> map = (Map<String,Object>)object;
if (1 == map.size() && map.containsKey(IndexSchema.DYNAMIC_FIELD)) {
map = (Map<String,Object>)map.get(IndexSchema.DYNAMIC_FIELD);
}
String bodyFieldName;
if (null != (bodyFieldName = (String)map.remove(IndexSchema.NAME))
&& ! fieldNamePattern.equals(bodyFieldName)) {
String message = "Dynamic field name in the request body '" + bodyFieldName
+ "' doesn't match dynamic field name in the request URL '" + fieldNamePattern + "'";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
String fieldType;
if (null == (fieldType = (String) map.remove(IndexSchema.TYPE))) {
String message = "Missing '" + IndexSchema.TYPE + "' mapping.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
} else {
ManagedIndexSchema oldSchema = (ManagedIndexSchema)getSchema();
Object copies = map.get(IndexSchema.COPY_FIELDS);
List<String> copyFieldNames = null;
if (copies != null) {
if (copies instanceof List) {
copyFieldNames = (List<String>)copies;
} else if (copies instanceof String) {
copyFieldNames = Collections.singletonList(copies.toString());
} else {
String message = "Invalid '" + IndexSchema.COPY_FIELDS + "' type.";
log.error(message);
throw new SolrException(ErrorCode.BAD_REQUEST, message);
}
}
if (copyFieldNames != null) {
map.remove(IndexSchema.COPY_FIELDS);
}
boolean success = false;
while ( ! success) {
try {
SchemaField newDynamicField = oldSchema.newDynamicField(fieldNamePattern, fieldType, map);
synchronized (oldSchema.getSchemaUpdateLock()) {
IndexSchema newSchema = oldSchema.addDynamicField(newDynamicField, copyFieldNames);
if (null != newSchema) {
getSolrCore().setLatestSchema(newSchema);
success = true;
} else {
throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to add dynamic field.");
}
}
} catch (ManagedIndexSchema.SchemaChangedInZkException e) {
log.debug("Schema changed while processing request, retrying");
oldSchema = (ManagedIndexSchema)getSolrCore().getLatestSchema();
}
}
}
}
}
}
}
} catch (Exception e) {
getSolrResponse().setException(e);
}
handlePostExecution(log);
return new SolrOutputRepresentation();
}
}

15
solr/core/src/java/org/apache/solr/rest/schema/FieldCollectionResource.java
View File

@ -219,19 +219,4 @@ public class FieldCollectionResource extends BaseFieldResource implements GETabl
return new SolrOutputRepresentation();
}
private static class NewFieldArguments {
private String name;
private String type;
Map<String, Object> map;
NewFieldArguments(String name, String type, Map<String, Object> map){
this.name = name;
this.type = type;
this.map = map;
}
public String getName() { return name; }
public String getType() { return type; }
public Map<String, Object> getMap() { return map; }
}
}

158
solr/core/src/java/org/apache/solr/schema/IndexSchema.java
View File

@ -521,7 +521,7 @@ public class IndexSchema {
}
}
// /schema/defaultSearchField/@text()
// /schema/defaultSearchField/text()
expression = stepsToPath(SCHEMA, DEFAULT_SEARCH_FIELD, TEXT_FUNCTION);
node = (Node) xpath.evaluate(expression, document, XPathConstants.NODE);
if (node==null) {
@ -668,21 +668,8 @@ public class IndexSchema {
requiredFields.add(f);
}
} else if (node.getNodeName().equals(DYNAMIC_FIELD)) {
if( f.getDefaultValue() != null ) {
throw new SolrException(ErrorCode.SERVER_ERROR,
DYNAMIC_FIELD + " can not have a default value: " + name);
}
if ( f.isRequired() ) {
throw new SolrException(ErrorCode.SERVER_ERROR,
DYNAMIC_FIELD + " can not be required: " + name);
}
if (isValidFieldGlob(name)) {
// make sure nothing else has the same path
addDynamicField(dFields, f);
} else {
String msg = "Dynamic field name '" + name
+ "' should have either a leading or a trailing asterisk, and no others.";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
if (isValidDynamicField(dFields, f)) {
addDynamicFieldNoDupCheck(dFields, f);
}
} else {
// we should never get here
@ -695,18 +682,25 @@ public class IndexSchema {
// in DocumentBuilder.getDoc()
requiredFields.addAll(fieldsWithDefaultValue);
// OK, now sort the dynamic fields largest to smallest size so we don't get
// any false matches. We want to act like a compiler tool and try and match
// the largest string possible.
Collections.sort(dFields);
log.trace("Dynamic Field Ordering:" + dFields);
// stuff it in a normal array for faster access
dynamicFields = dFields.toArray(new DynamicField[dFields.size()]);
dynamicFields = dynamicFieldListToSortedArray(dFields);
return explicitRequiredProp;
}
/**
* Sort the dynamic fields and stuff them in a normal array for faster access.
*/
protected static DynamicField[] dynamicFieldListToSortedArray(List<DynamicField> dynamicFieldList) {
// Avoid creating the array twice by converting to an array first and using Arrays.sort(),
// rather than Collections.sort() then converting to an array, since Collections.sort()
// copies to an array first, then sets each collection member from the array.
DynamicField[] dFields = dynamicFieldList.toArray(new DynamicField[dynamicFieldList.size()]);
Arrays.sort(dFields);
log.trace("Dynamic Field Ordering:" + Arrays.toString(dFields));
return dFields;
}
/**
* Loads the copy fields
@ -773,15 +767,29 @@ public class IndexSchema {
return false;
}
private void addDynamicField(List<DynamicField> dFields, SchemaField f) {
if (isDuplicateDynField(dFields, f)) {
String msg = "[schema.xml] Duplicate DynamicField definition for '" + f.getName() + "'";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
} else {
addDynamicFieldNoDupCheck(dFields, f);
protected boolean isValidDynamicField(List<DynamicField> dFields, SchemaField f) {
String glob = f.getName();
if (f.getDefaultValue() != null) {
throw new SolrException(ErrorCode.SERVER_ERROR,
DYNAMIC_FIELD + " can not have a default value: " + glob);
}
if (f.isRequired()) {
throw new SolrException(ErrorCode.SERVER_ERROR,
DYNAMIC_FIELD + " can not be required: " + glob);
}
if ( ! isValidFieldGlob(glob)) {
String msg = "Dynamic field name '" + glob
+ "' should have either a leading or a trailing asterisk, and no others.";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
if (isDuplicateDynField(dFields, f)) {
String msg = "[schema.xml] Duplicate DynamicField definition for '" + glob + "'";
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
return true;
}
/**
* Register one or more new Dynamic Fields with the Schema.
* @param fields The sequence of {@link org.apache.solr.schema.SchemaField}
@ -796,8 +804,7 @@ public class IndexSchema {
addDynamicFieldNoDupCheck(dynFields, field);
}
}
Collections.sort(dynFields);
dynamicFields = dynFields.toArray(new DynamicField[dynFields.size()]);
dynamicFields = dynamicFieldListToSortedArray(dynFields);
}
private void addDynamicFieldNoDupCheck(List<DynamicField> dFields, SchemaField f) {
@ -805,7 +812,7 @@ public class IndexSchema {
log.debug("dynamic field defined: " + f);
}
private boolean isDuplicateDynField(List<DynamicField> dFields, SchemaField f) {
protected boolean isDuplicateDynField(List<DynamicField> dFields, SchemaField f) {
for (DynamicField df : dFields) {
if (df.getRegex().equals(f.name)) return true;
}
@ -1534,6 +1541,69 @@ public class IndexSchema {
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Copies this schema, adds the given dynamic field to the copy, then persists the
* new schema. Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newDynamicField the SchemaField to add
* @return a new IndexSchema based on this schema with newField added
* @see #newDynamicField(String, String, Map)
*/
public IndexSchema addDynamicField(SchemaField newDynamicField) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Copies this schema, adds the given dynamic field to the copy, then persists the
* new schema. Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newDynamicField the SchemaField to add
* @param copyFieldNames 0 or more names of targets to copy this field to. The targets must already exist.
* @return a new IndexSchema based on this schema with newDynamicField added
* @see #newDynamicField(String, String, Map)
*/
public IndexSchema addDynamicField(SchemaField newDynamicField, Collection<String> copyFieldNames) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Copies this schema, adds the given dynamic fields to the copy, then persists the
* new schema. Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newDynamicFields the SchemaFields to add
* @return a new IndexSchema based on this schema with newDynamicFields added
* @see #newDynamicField(String, String, Map)
*/
public IndexSchema addDynamicFields(Collection<SchemaField> newDynamicFields) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Copies this schema, adds the given dynamic fields to the copy, then persists the
* new schema. Requires synchronizing on the object returned by
* {@link #getSchemaUpdateLock()}.
*
* @param newDynamicFields the SchemaFields to add
* @param copyFieldNames 0 or more names of targets to copy this field to. The target fields must already exist.
* @return a new IndexSchema based on this schema with newDynamicFields added
* @see #newDynamicField(String, String, Map)
*/
public IndexSchema addDynamicFields
(Collection<SchemaField> newDynamicFields, Map<String, Collection<String>> copyFieldNames) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Copies this schema and adds the new copy fields to the copy, then
* persists the new schema. Requires synchronizing on the object returned by
@ -1566,6 +1636,24 @@ public class IndexSchema {
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Returns a SchemaField if the given dynamic field glob does not already
* exist in this schema, and does not match any dynamic fields
* in this schema. The resulting SchemaField can be used in a call
* to {@link #addField(SchemaField)}.
*
* @param fieldNamePattern the pattern for the dynamic field to add
* @param fieldType the field type for the new field
* @param options the options to use when creating the SchemaField
* @return The created SchemaField
* @see #addField(SchemaField)
*/
public SchemaField newDynamicField(String fieldNamePattern, String fieldType, Map<String,?> options) {
String msg = "This IndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
/**
* Returns the schema update lock that should be synchronzied on
* to update the schema. Only applicable to mutable schemas.

134
solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
View File

@ -40,10 +40,12 @@ import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */
@ -174,16 +176,6 @@ public final class ManagedIndexSchema extends IndexSchema {
return success;
}
@Override
public ManagedIndexSchema addField(SchemaField newField) {
return addFields(Arrays.asList(newField));
}
@Override
public ManagedIndexSchema addField(SchemaField newField, Collection<String> copyFieldNames) {
return addFields(Arrays.asList(newField), Collections.singletonMap(newField.getName(), copyFieldNames));
}
public class FieldExistsException extends SolrException {
public FieldExistsException(ErrorCode code, String msg) {
super(code, msg);
@ -196,6 +188,16 @@ public final class ManagedIndexSchema extends IndexSchema {
}
}
@Override
public ManagedIndexSchema addField(SchemaField newField) {
return addFields(Arrays.asList(newField));
}
@Override
public ManagedIndexSchema addField(SchemaField newField, Collection<String> copyFieldNames) {
return addFields(Arrays.asList(newField), Collections.singletonMap(newField.getName(), copyFieldNames));
}
@Override
public ManagedIndexSchema addFields(Collection<SchemaField> newFields) {
return addFields(newFields, Collections.<String, Collection<String>>emptyMap());
@ -232,19 +234,83 @@ public final class ManagedIndexSchema extends IndexSchema {
newSchema.registerCopyField(newField.getName(), copyField);
}
}
}
// Run the callbacks on SchemaAware now that everything else is done
for (SchemaAware aware : newSchema.schemaAware) {
aware.inform(newSchema);
// Run the callbacks on SchemaAware now that everything else is done
for (SchemaAware aware : newSchema.schemaAware) {
aware.inform(newSchema);
}
newSchema.refreshAnalyzers();
success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
if (success) {
log.debug("Added field(s): {}", newFields);
} else {
log.error("Failed to add field(s): {}", newFields);
newSchema = null;
}
} else {
String msg = "This ManagedIndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
return newSchema;
}
@Override
public IndexSchema addDynamicField(SchemaField newDynamicField) {
return addDynamicFields(Arrays.asList(newDynamicField));
}
@Override
public IndexSchema addDynamicField(SchemaField newDynamicField, Collection<String> copyFieldNames) {
return addDynamicFields(Arrays.asList(newDynamicField),
Collections.singletonMap(newDynamicField.getName(), copyFieldNames));
}
@Override
public ManagedIndexSchema addDynamicFields(Collection<SchemaField> newDynamicFields) {
return addDynamicFields(newDynamicFields, Collections.<String,Collection<String>>emptyMap());
}
@Override
public ManagedIndexSchema addDynamicFields(Collection<SchemaField> newDynamicFields,
Map<String,Collection<String>> copyFieldNames) {
ManagedIndexSchema newSchema = null;
if (isMutable) {
boolean success = false;
if (copyFieldNames == null){
copyFieldNames = Collections.emptyMap();
}
newSchema = shallowCopy(true);
for (SchemaField newDynamicField : newDynamicFields) {
List<DynamicField> dFields = new ArrayList<>(Arrays.asList(newSchema.dynamicFields));
if (isDuplicateDynField(dFields, newDynamicField)) {
String msg = "Dynamic field '" + newDynamicField.getName() + "' already exists.";
throw new FieldExistsException(ErrorCode.BAD_REQUEST, msg);
}
newSchema.refreshAnalyzers();
success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
if (success) {
log.debug("Added field(s): {}", newFields);
} else {
log.error("Failed to add field(s): {}", newFields);
dFields.add(new DynamicField(newDynamicField));
newSchema.dynamicFields = dynamicFieldListToSortedArray(dFields);
Collection<String> copyFields = copyFieldNames.get(newDynamicField.getName());
if (copyFields != null) {
for (String copyField : copyFields) {
newSchema.registerCopyField(newDynamicField.getName(), copyField);
}
}
}
// Run the callbacks on SchemaAware now that everything else is done
for (SchemaAware aware : newSchema.schemaAware) {
aware.inform(newSchema);
}
newSchema.refreshAnalyzers();
success = newSchema.persistManagedSchema(false); // don't just create - update it if it already exists
if (success) {
log.debug("Added dynamic field(s): {}", newDynamicFields);
} else {
log.error("Failed to add dynamic field(s): {}", newDynamicFields);
}
} else {
String msg = "This ManagedIndexSchema is not mutable.";
log.error(msg);
@ -316,6 +382,36 @@ public final class ManagedIndexSchema extends IndexSchema {
return sf;
}
@Override
public SchemaField newDynamicField(String fieldNamePattern, String fieldType, Map<String,?> options) {
SchemaField sf;
if (isMutable) {
try {
FieldType type = getFieldTypeByName(fieldType);
if (null == type) {
String msg = "Dynamic field '" + fieldNamePattern + "': Field type '" + fieldType + "' not found.";
log.error(msg);
throw new SolrException(ErrorCode.BAD_REQUEST, msg);
}
sf = SchemaField.create(fieldNamePattern, type, options);
if ( ! isValidDynamicField(Arrays.asList(dynamicFields), sf)) {
String msg = "Invalid dynamic field '" + fieldNamePattern + "'";
log.error(msg);
throw new SolrException(ErrorCode.BAD_REQUEST, msg);
}
} catch (SolrException e) {
throw e;
} catch (Exception e) {
throw new SolrException(ErrorCode.BAD_REQUEST, e);
}
} else {
String msg = "This ManagedIndexSchema is not mutable.";
log.error(msg);
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
}
return sf;
}
/**
* Called from ZkIndexSchemaReader to merge the fields from the serialized managed schema
* on ZooKeeper with the local managed schema.

7
solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldCollectionResource.java
View File

@ -59,4 +59,11 @@ public class TestDynamicFieldCollectionResource extends SolrRestletTestBase {
"/dynamicFields/[0]/name=='*_i'",
"/dynamicFields/[1]/name=='*_s'");
}
@Test
public void testJsonPostFieldsToNonMutableIndexSchema() throws Exception {
assertJPost("/schema/dynamicfields",
"[{\"name\":\"foobarbaz\", \"type\":\"text_general\", \"stored\":\"false\"}]",
"/error/msg=='This IndexSchema is not mutable.'");
}
}

7
solr/core/src/test/org/apache/solr/rest/schema/TestDynamicFieldResource.java
View File

@ -68,4 +68,11 @@ public class TestDynamicFieldResource extends SolrRestletTestBase {
"/dynamicField/required==false",
"/dynamicField/tokenized==false");
}
@Test
public void testJsonPutFieldToNonMutableIndexSchema() throws Exception {
assertJPut("/schema/dynamicfields/newfield_*",
"{\"type\":\"text_general\", \"stored\":\"false\"}",
"/error/msg=='This IndexSchema is not mutable.'");
}
}

364
solr/core/src/test/org/apache/solr/rest/schema/TestManagedSchemaDynamicFieldResource.java Normal file
View File

@ -0,0 +1,364 @@
package org.apache.solr.rest.schema;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Pattern;
import org.apache.commons.io.FileUtils;
import org.apache.solr.util.RestTestBase;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.restlet.ext.servlet.ServerServlet;
public class TestManagedSchemaDynamicFieldResource extends RestTestBase {
private static File tmpSolrHome;
private static File tmpConfDir;
private static final String collection = "collection1";
private static final String confDir = collection + "/conf";
@Before
public void before() throws Exception {
tmpSolrHome = createTempDir();
tmpConfDir = new File(tmpSolrHome, confDir);
FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile());
final SortedMap<ServletHolder,String> extraServlets = new TreeMap<>();
final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...'
System.setProperty("managed.schema.mutable", "true");
System.setProperty("enable.update.log", "false");
createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml",
"/solr", true, extraServlets);
}
@After
public void after() throws Exception {
if (jetty != null) {
jetty.stop();
jetty = null;
}
server = null;
restTestHarness = null;
}
@Test
public void testAddDynamicFieldBadFieldType() throws Exception {
assertJPut("/schema/dynamicfields/*_newdynamicfield",
json( "{'type':'not_in_there_at_all','stored':false}" ),
"/error/msg==\"Dynamic field \\'*_newdynamicfield\\': Field type \\'not_in_there_at_all\\' not found.\"");
}
@Test
public void testAddDynamicFieldMismatchedName() throws Exception {
assertJPut("/schema/dynamicfields/*_newdynamicfield",
json( "{'name':'*_something_else','type':'text','stored':false}" ),
"/error/msg=='///regex:\\\\*_newdynamicfield///'");
}
@Test
public void testAddDynamicFieldBadProperty() throws Exception {
assertJPut("/schema/dynamicfields/*_newdynamicfield",
json( "{'type':'text','no_property_with_this_name':false}" ),
"/error/msg==\"java.lang.IllegalArgumentException: Invalid field property: no_property_with_this_name\"");
}
@Test
public void testAddDynamicField() throws Exception {
assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml",
"count(/response/lst[@name='newdynamicfield_*']) = 0",
"/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
"/response/lst[@name='error']/int[@name='code'] = '404'");
assertJPut("/schema/dynamicfields/newdynamicfield_*",
json("{'type':'text','stored':false}"),
"/responseHeader/status==0");
assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'");
assertU(adoc("newdynamicfield_A", "value1 value2", "id", "123"));
assertU(commit());
assertQ("/select?q=newdynamicfield_A:value1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/result[@name='response'][@numFound='1']",
"count(/response/result[@name='response']/doc/*) = 1",
"/response/result[@name='response']/doc/str[@name='id'][.='123']");
}
@Test
public void testAddDynamicFieldWithMulipleOptions() throws Exception {
assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 0",
"/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
"/response/lst[@name='error']/int[@name='code'] = '404'");
assertJPut("/schema/dynamicfields/newdynamicfield_*",
json("{'type':'text_en','stored':true,'indexed':false}"),
"/responseHeader/status==0");
File managedSchemaFile = new File(tmpConfDir, "managed-schema");
assertTrue(managedSchemaFile.exists());
String managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8");
Pattern newdynamicfieldStoredTrueIndexedFalsePattern
= Pattern.compile( "<dynamicField name=\"newdynamicfield_\\*\" type=\"text_en\""
+ "(?=.*stored=\"true\")(?=.*indexed=\"false\").*/>");
assertTrue(newdynamicfieldStoredTrueIndexedFalsePattern.matcher(managedSchemaContents).find());
assertQ("/schema/dynamicfields/newdynamicfield_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/lst[@name='dynamicField']/str[@name='name'] = 'newdynamicfield_*'",
"/response/lst[@name='dynamicField']/str[@name='type'] = 'text_en'",
"/response/lst[@name='dynamicField']/bool[@name='indexed'] = 'false'",
"/response/lst[@name='dynamicField']/bool[@name='stored'] = 'true'");
assertU(adoc("newdynamicfield_A", "value1 value2", "id", "1234"));
assertU(commit());
assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 0",
"/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
"/response/lst[@name='error']/int[@name='code'] = '404'");
assertJPut("/schema/dynamicfields/newdynamicfield2_*",
json("{'type':'text_en','stored':true,'indexed':true,'multiValued':true}"),
"/responseHeader/status==0");
managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8");
Pattern newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern
= Pattern.compile( "<dynamicField name=\"newdynamicfield2_\\*\" type=\"text_en\" "
+ "(?=.*stored=\"true\")(?=.*indexed=\"true\")(?=.*multiValued=\"true\").*/>");
assertTrue(newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern.matcher(managedSchemaContents).find());
assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/lst[@name='dynamicField']/str[@name='name'] = 'newdynamicfield2_*'",
"/response/lst[@name='dynamicField']/str[@name='type'] = 'text_en'",
"/response/lst[@name='dynamicField']/bool[@name='indexed'] = 'true'",
"/response/lst[@name='dynamicField']/bool[@name='stored'] = 'true'",
"/response/lst[@name='dynamicField']/bool[@name='multiValued'] = 'true'");
assertU(adoc("newdynamicfield2_A", "value1 value2", "newdynamicfield2_A", "value3 value4", "id", "5678"));
assertU(commit());
assertQ("/select?q=newdynamicfield2_A:value3",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/result[@name='response'][@numFound='1']",
"count(/response/result[@name='response']/doc) = 1",
"/response/result[@name='response']/doc/str[@name='id'][.='5678']");
}
@Test
public void testAddDynamicFieldCollectionWithMultipleOptions() throws Exception {
assertQ("/schema/dynamicfields?indent=on&wt=xml",
"count(/response/arr[@name='dynamicFields']/lst/str[@name]) > 0", // there are fields
"count(/response/arr[@name='dynamicFields']/lst/str[starts-with(@name,'newfield')]) = 0"); // but none named newfield*
assertJPost("/schema/dynamicfields",
json("[{'name':'newdynamicfield_*','type':'text_en','stored':true,'indexed':false}]"),
"/responseHeader/status==0");
File managedSchemaFile = new File(tmpConfDir, "managed-schema");
assertTrue(managedSchemaFile.exists());
String managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8");
Pattern newfieldStoredTrueIndexedFalsePattern
= Pattern.compile( "<dynamicField name=\"newdynamicfield_\\*\" type=\"text_en\""
+ "(?=.*stored=\"true\")(?=.*indexed=\"false\").*/>");
assertTrue(newfieldStoredTrueIndexedFalsePattern.matcher(managedSchemaContents).find());
assertQ("/schema/dynamicfields?indent=on&wt=xml",
"/response/arr[@name='dynamicFields']/lst"
+ "[str[@name='name']='newdynamicfield_*' and str[@name='type']='text_en'"
+ " and bool[@name='stored']='true' and bool[@name='indexed']='false']");
assertU(adoc("newdynamicfield_A", "value1 value2", "id", "789"));
assertU(commit());
assertJPost("/schema/dynamicfields",
json("[{'name':'newdynamicfield2_*','type':'text_en','stored':true,'indexed':true,'multiValued':true}]"),
"/responseHeader/status==0");
managedSchemaContents = FileUtils.readFileToString(managedSchemaFile, "UTF-8");
Pattern newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern
= Pattern.compile( "<dynamicField name=\"newdynamicfield2_\\*\" type=\"text_en\" "
+ "(?=.*stored=\"true\")(?=.*indexed=\"true\")(?=.*multiValued=\"true\").*/>");
assertTrue(newdynamicfield2StoredTrueIndexedTrueMultiValuedTruePattern.matcher(managedSchemaContents).find());
assertQ("/schema/dynamicfields?indent=on&wt=xml",
"/response/arr[@name='dynamicFields']/lst"
+ "[str[@name='name']='newdynamicfield2_*' and str[@name='type']='text_en'"
+ " and bool[@name='stored']='true' and bool[@name='indexed']='true' and bool[@name='multiValued']='true']");
assertU(adoc("newdynamicfield2_A", "value1 value2", "newdynamicfield2_A", "value3 value4", "id", "790"));
assertU(commit());
assertQ("/select?q=newdynamicfield2_A:value3",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/result[@name='response'][@numFound='1']",
"count(/response/result[@name='response']/doc) = 1",
"/response/result[@name='response']/doc/str[@name='id'][.='790']");
}
@Test
public void testAddCopyField() throws Exception {
assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 0",
"/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
"/response/lst[@name='error']/int[@name='code'] = '404'");
assertJPut("/schema/dynamicfields/dynamicfieldA_*",
json("{'type':'text','stored':false}"),
"/responseHeader/status==0");
assertJPut("/schema/dynamicfields/dynamicfieldB_*",
json("{'type':'text','stored':false, 'copyFields':['dynamicfieldA_*']}"),
"/responseHeader/status==0");
assertJPut("/schema/dynamicfields/dynamicfieldC_*",
json("{'type':'text','stored':false, 'copyFields':'dynamicfieldA_*'}"),
"/responseHeader/status==0");
assertQ("/schema/dynamicfields/dynamicfieldB_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldB_*",
"count(/response/arr[@name='copyFields']/lst) = 1");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldC_*",
"count(/response/arr[@name='copyFields']/lst) = 1");
//fine to pass in empty list, just won't do anything
assertJPut("/schema/dynamicfields/dynamicfieldD_*",
json("{'type':'text','stored':false, 'copyFields':[]}"),
"/responseHeader/status==0");
//some bad usages
assertJPut("/schema/dynamicfields/dynamicfieldF_*",
json("{'type':'text','stored':false, 'copyFields':['some_nonexistent_dynamicfield_ignore_exception_*']}"),
"/error/msg==\"copyField dest :\\'some_nonexistent_dynamicfield_ignore_exception_*\\' is not an explicit field and doesn\\'t match a dynamicField.\"");
}
@Test
public void testPostMultipleDynamicFields() throws Exception {
assertQ("/schema/dynamicfields/newdynamicfield1_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 0",
"/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
"/response/lst[@name='error']/int[@name='code'] = '404'");
assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 0",
"/response/lst[@name='responseHeader']/int[@name='status'] = '404'",
"/response/lst[@name='error']/int[@name='code'] = '404'");
assertJPost("/schema/dynamicfields",
json( "[{'name':'newdynamicfield1_*','type':'text','stored':false},"
+ " {'name':'newdynamicfield2_*','type':'text','stored':false}]"),
"/responseHeader/status==0");
assertQ("/schema/dynamicfields/newdynamicfield1_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'");
assertQ("/schema/dynamicfields/newdynamicfield2_*?indent=on&wt=xml",
"count(/response/lst[@name='dynamicField']) = 1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'");
assertU(adoc("newdynamicfield1_A", "value1 value2", "id", "123"));
assertU(adoc("newdynamicfield2_A", "value3 value4", "id", "456"));
assertU(commit());
assertQ("/select?q=newdynamicfield1_A:value1",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/result[@name='response'][@numFound='1']",
"count(/response/result[@name='response']/doc/*) = 1",
"/response/result[@name='response']/doc/str[@name='id'][.='123']");
assertQ("/select?q=newdynamicfield2_A:value3",
"/response/lst[@name='responseHeader']/int[@name='status'] = '0'",
"/response/result[@name='response'][@numFound='1']",
"count(/response/result[@name='response']/doc/*) = 1",
"/response/result[@name='response']/doc/str[@name='id'][.='456']");
}
@Test
public void testPostCopy() throws Exception {
assertJPost("/schema/dynamicfields",
json( "[{'name':'dynamicfieldA_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldB_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldC_*','type':'text','stored':false, 'copyFields':['dynamicfieldB_*']}]"),
"/responseHeader/status==0");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldC_*",
"count(/response/arr[@name='copyFields']/lst) = 1");
assertJPost("/schema/dynamicfields",
json( "[{'name':'dynamicfieldD_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldE_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldF_*','type':'text','stored':false, 'copyFields':['dynamicfieldD_*','dynamicfieldE_*']},"
+ " {'name':'dynamicfieldG_*','type':'text','stored':false, 'copyFields':'dynamicfieldD_*'}]"),//single
"/responseHeader/status==0");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldF_*",
"count(/response/arr[@name='copyFields']/lst) = 2");
//passing in an empty list is perfectly acceptable, it just won't do anything
assertJPost("/schema/dynamicfields",
json( "[{'name':'dynamicfieldX_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldY_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldZ_*','type':'text','stored':false, 'copyFields':[]}]"),
"/responseHeader/status==0");
//some bad usages
assertJPost("/schema/dynamicfields",
json( "[{'name':'dynamicfieldH_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldI_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldJ_*','type':'text','stored':false, 'copyFields':['some_nonexistent_dynamicfield_ignore_exception_*']}]"),
"/error/msg=='copyField dest :\\'some_nonexistent_dynamicfield_ignore_exception_*\\' is not an explicit field and doesn\\'t match a dynamicField.'");
}
@Test
public void testPostCopyFields() throws Exception {
assertJPost("/schema/dynamicfields",
json( "[{'name':'dynamicfieldA_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldB_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldC_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldD_*','type':'text','stored':false},"
+ " {'name':'dynamicfieldE_*','type':'text','stored':false}]"),
"/responseHeader/status==0");
assertJPost("/schema/copyfields",
json( "[{'source':'dynamicfieldA_*', 'dest':'dynamicfieldB_*'},"
+ " {'source':'dynamicfieldD_*', 'dest':['dynamicfieldC_*', 'dynamicfieldE_*']}]"),
"/responseHeader/status==0");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldA_*",
"count(/response/arr[@name='copyFields']/lst) = 1");
assertQ("/schema/copyfields/?indent=on&wt=xml&source.fl=dynamicfieldD_*",
"count(/response/arr[@name='copyFields']/lst) = 2");
assertJPost("/schema/copyfields", // copyField glob sources are not required to match a dynamic field
json("[{'source':'some_glob_not_necessarily_matching_any_dynamicfield_*', 'dest':['dynamicfieldA_*']},"
+" {'source':'*', 'dest':['dynamicfieldD_*']}]"),
"/responseHeader/status==0");
assertJPost("/schema/copyfields",
json("[{'source':'dynamicfieldD_*', 'dest':['some_nonexistent_dynamicfield_ignore_exception_*']}]"),
"/error/msg=='copyField dest :\\'some_nonexistent_dynamicfield_ignore_exception_*\\' is not an explicit field and doesn\\'t match a dynamicField.'");
}
}

281
solr/core/src/test/org/apache/solr/schema/TestCloudManagedSchemaConcurrent.java
View File

@ -38,6 +38,10 @@ import java.util.concurrent.TimeUnit;
public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(TestCloudManagedSchemaConcurrent.class);
private static final String SUCCESS_XPATH = "/response/lst[@name='responseHeader']/int[@name='status'][.='0']";
private static final String PUT_DYNAMIC_FIELDNAME = "newdynamicfieldPut";
private static final String POST_DYNAMIC_FIELDNAME = "newdynamicfieldPost";
private static final String PUT_FIELDNAME = "newfieldPut";
private static final String POST_FIELDNAME = "newfieldPost";
public TestCloudManagedSchemaConcurrent() {
super();
@ -81,7 +85,7 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
}
}
private void verifySuccess(String request, String response) throws Exception {
private static void verifySuccess(String request, String response) throws Exception {
String result = BaseTestHarness.validateXPath(response, SUCCESS_XPATH);
if (null != result) {
String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
@ -90,51 +94,83 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
}
}
private void addFieldPut(RestTestHarness publisher, String fieldName) throws Exception {
private static void addFieldPut(RestTestHarness publisher, String fieldName) throws Exception {
final String content = "{\"type\":\"text\",\"stored\":\"false\"}";
String request = "/schema/fields/" + fieldName + "?wt=xml";
String response = publisher.put(request, content);
verifySuccess(request, response);
}
private void addFieldPost(RestTestHarness publisher, String fieldName) throws Exception {
private static void addFieldPost(RestTestHarness publisher, String fieldName) throws Exception {
final String content = "[{\"name\":\""+fieldName+"\",\"type\":\"text\",\"stored\":\"false\"}]";
String request = "/schema/fields/?wt=xml";
String response = publisher.post(request, content);
verifySuccess(request, response);
}
private void copyField(RestTestHarness publisher, String source, String dest) throws Exception {
private static void addDynamicFieldPut(RestTestHarness publisher, String dynamicFieldPattern) throws Exception {
final String content = "{\"type\":\"text\",\"stored\":\"false\"}";
String request = "/schema/dynamicfields/" + dynamicFieldPattern + "?wt=xml";
String response = publisher.put(request, content);
verifySuccess(request, response);
}
private static void addDynamicFieldPost(RestTestHarness publisher, String dynamicFieldPattern) throws Exception {
final String content = "[{\"name\":\""+dynamicFieldPattern+"\",\"type\":\"text\",\"stored\":\"false\"}]";
String request = "/schema/dynamicfields/?wt=xml";
String response = publisher.post(request, content);
verifySuccess(request, response);
}
private static void copyField(RestTestHarness publisher, String source, String dest) throws Exception {
final String content = "[{\"source\":\""+source+"\",\"dest\":[\""+dest+"\"]}]";
String request = "/schema/copyfields/?wt=xml";
String response = publisher.post(request, content);
verifySuccess(request, response);
}
private String[] getExpectedFieldResponses(int numAddFieldPuts, String putFieldName,
int numAddFieldPosts, String postFieldName) {
String[] expectedAddFields = new String[1 + numAddFieldPuts + numAddFieldPosts];
private String[] getExpectedFieldResponses(Info info) {
String[] expectedAddFields = new String[1 + info.numAddFieldPuts + info.numAddFieldPosts];
expectedAddFields[0] = SUCCESS_XPATH;
for (int i = 0; i < numAddFieldPuts; ++i) {
String newFieldName = putFieldName + i;
for (int i = 0; i < info.numAddFieldPuts; ++i) {
String newFieldName = PUT_FIELDNAME + info.fieldNameSuffix + i;
expectedAddFields[1 + i]
= "/response/arr[@name='fields']/lst/str[@name='name'][.='" + newFieldName + "']";
}
for (int i = 0; i < numAddFieldPosts; ++i) {
String newFieldName = postFieldName + i;
expectedAddFields[1 + numAddFieldPuts + i]
for (int i = 0; i < info.numAddFieldPosts; ++i) {
String newFieldName = POST_FIELDNAME + info.fieldNameSuffix + i;
expectedAddFields[1 + info.numAddFieldPuts + i]
= "/response/arr[@name='fields']/lst/str[@name='name'][.='" + newFieldName + "']";
}
return expectedAddFields;
}
private String[] getExpectedCopyFieldResponses(List<CopyFieldInfo> copyFields) {
private String[] getExpectedDynamicFieldResponses(Info info) {
String[] expectedAddDynamicFields = new String[1 + info.numAddDynamicFieldPuts + info.numAddDynamicFieldPosts];
expectedAddDynamicFields[0] = SUCCESS_XPATH;
for (int i = 0; i < info.numAddDynamicFieldPuts; ++i) {
String newDynamicFieldPattern = PUT_DYNAMIC_FIELDNAME + info.fieldNameSuffix + i + "_*";
expectedAddDynamicFields[1 + i]
= "/response/arr[@name='dynamicFields']/lst/str[@name='name'][.='" + newDynamicFieldPattern + "']";
}
for (int i = 0; i < info.numAddDynamicFieldPosts; ++i) {
String newDynamicFieldPattern = POST_DYNAMIC_FIELDNAME + info.fieldNameSuffix + i + "_*";
expectedAddDynamicFields[1 + info.numAddDynamicFieldPuts + i]
= "/response/arr[@name='dynamicFields']/lst/str[@name='name'][.='" + newDynamicFieldPattern + "']";
}
return expectedAddDynamicFields;
}
private String[] getExpectedCopyFieldResponses(Info info) {
ArrayList<String> expectedCopyFields = new ArrayList<>();
expectedCopyFields.add(SUCCESS_XPATH);
for (CopyFieldInfo cpi : copyFields) {
for (CopyFieldInfo cpi : info.copyFields) {
String expectedSourceName = cpi.getSourceField();
expectedCopyFields.add
("/response/arr[@name='copyFields']/lst/str[@name='source'][.='" + expectedSourceName + "']");
@ -151,31 +187,46 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
setupHarnesses();
concurrentOperationsTest();
schemaLockTest();
}
}
private void concurrentOperationsTest() throws Exception {
// First, add a bunch of fields via PUT and POST, as well as copyFields,
// but do it fast enough and verify shards' schemas after all of them are added
int numFields = 100;
private class Info {
int numAddFieldPuts = 0;
int numAddFieldPosts = 0;
int numAddDynamicFieldPuts = 0;
int numAddDynamicFieldPosts = 0;
public String fieldNameSuffix;
List<CopyFieldInfo> copyFields = new ArrayList<>();
final String putFieldName = "newfieldPut";
final String postFieldName = "newfieldPost";
for (int i = 0; i <= numFields ; ++i) {
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
public Info(String fieldNameSuffix) {
this.fieldNameSuffix = fieldNameSuffix;
}
}
int type = random().nextInt(3);
if (type == 0) { // send an add field via PUT
addFieldPut(publisher, putFieldName + numAddFieldPuts++);
private enum Operation {
PUT_AddField {
@Override public void execute(RestTestHarness publisher, int fieldNum, Info info) throws Exception {
String fieldname = PUT_FIELDNAME + info.numAddFieldPuts++;
addFieldPut(publisher, fieldname);
}
else if (type == 1) { // send an add field via POST
addFieldPost(publisher, postFieldName + numAddFieldPosts++);
},
POST_AddField {
@Override public void execute(RestTestHarness publisher, int fieldNum, Info info) throws Exception {
String fieldname = POST_FIELDNAME + info.numAddFieldPosts++;
addFieldPost(publisher, fieldname);
}
else if (type == 2) { // send a copy field
},
PUT_AddDynamicField {
@Override public void execute(RestTestHarness publisher, int fieldNum, Info info) throws Exception {
addDynamicFieldPut(publisher, PUT_DYNAMIC_FIELDNAME + info.numAddDynamicFieldPuts++ + "_*");
}
},
POST_AddDynamicField {
@Override public void execute(RestTestHarness publisher, int fieldNum, Info info) throws Exception {
addDynamicFieldPost(publisher, POST_DYNAMIC_FIELDNAME + info.numAddDynamicFieldPosts++ + "_*");
}
},
POST_AddCopyField {
@Override public void execute(RestTestHarness publisher, int fieldNum, Info info) throws Exception {
String sourceField = null;
String destField = null;
@ -183,31 +234,51 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
if (sourceType == 0) { // existing
sourceField = "name";
} else if (sourceType == 1) { // newly created
sourceField = "copySource" + i;
sourceField = "copySource" + fieldNum;
addFieldPut(publisher, sourceField);
} else { // dynamic
sourceField = "*_dynamicSource" + i + "_t";
sourceField = "*_dynamicSource" + fieldNum + "_t";
// * only supported if both src and dst use it
destField = "*_dynamicDest" + i + "_t";
destField = "*_dynamicDest" + fieldNum + "_t";
}
if (destField == null) {
int destType = random().nextInt(2);
if (destType == 0) { // existing
destField = "title";
} else { // newly created
destField = "copyDest" + i;
destField = "copyDest" + fieldNum;
addFieldPut(publisher, destField);
}
}
copyField(publisher, sourceField, destField);
copyFields.add(new CopyFieldInfo(sourceField, destField));
info.copyFields.add(new CopyFieldInfo(sourceField, destField));
}
};
public abstract void execute(RestTestHarness publisher, int fieldNum, Info info) throws Exception;
private static final Operation[] VALUES = values();
public static Operation randomOperation() {
return VALUES[r.nextInt(VALUES.length)];
}
}
private void concurrentOperationsTest() throws Exception {
// First, add a bunch of fields and dynamic fields via PUT and POST, as well as copyFields,
// but do it fast enough and verify shards' schemas after all of them are added
int numFields = 100;
Info info = new Info("");
for (int fieldNum = 0; fieldNum <= numFields ; ++fieldNum) {
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
Operation.randomOperation().execute(publisher, fieldNum, info);
}
String[] expectedAddFields = getExpectedFieldResponses(numAddFieldPuts, putFieldName,
numAddFieldPosts, postFieldName);
String[] expectedCopyFields = getExpectedCopyFieldResponses(copyFields);
String[] expectedAddFields = getExpectedFieldResponses(info);
String[] expectedAddDynamicFields = getExpectedDynamicFieldResponses(info);
String[] expectedCopyFields = getExpectedCopyFieldResponses(info);
boolean success = false;
long maxTimeoutMillis = 100000;
@ -229,6 +300,14 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
break;
}
// verify addDynamicFieldPuts and addDynamicFieldPosts
request = "/schema/dynamicfields?wt=xml";
response = client.query(request);
result = BaseTestHarness.validateXPath(response, expectedAddDynamicFields);
if (result != null) {
break;
}
// verify copyFields
request = "/schema/copyfields?wt=xml";
response = client.query(request);
@ -246,23 +325,72 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
}
}
private class PutPostThread extends Thread {
private abstract class PutPostThread extends Thread {
RestTestHarness harness;
String fieldName;
boolean isPut;
public PutPostThread(RestTestHarness harness, String fieldName, boolean isPut) {
Info info;
public String fieldName;
public PutPostThread(RestTestHarness harness, Info info) {
this.harness = harness;
this.fieldName = fieldName;
this.isPut = isPut;
this.info = info;
}
public abstract void run();
}
private class PutFieldThread extends PutPostThread {
public PutFieldThread(RestTestHarness harness, Info info) {
super(harness, info);
fieldName = PUT_FIELDNAME + "Thread" + info.numAddFieldPuts++;
}
public void run() {
try {
if (isPut) {
addFieldPut(harness, fieldName);
} else {
addFieldPost(harness, fieldName);
}
addFieldPut(harness, fieldName);
} catch (Exception e) {
// log.error("###ACTUAL FAILURE!");
throw new RuntimeException(e);
}
}
}
private class PostFieldThread extends PutPostThread {
public PostFieldThread(RestTestHarness harness, Info info) {
super(harness, info);
fieldName = POST_FIELDNAME + "Thread" + info.numAddFieldPosts++;
}
public void run() {
try {
addFieldPost(harness, fieldName);
} catch (Exception e) {
// log.error("###ACTUAL FAILURE!");
throw new RuntimeException(e);
}
}
}
private class PutDynamicFieldThread extends PutPostThread {
public PutDynamicFieldThread(RestTestHarness harness, Info info) {
super(harness, info);
fieldName = PUT_FIELDNAME + "Thread" + info.numAddFieldPuts++;
}
public void run() {
try {
addFieldPut(harness, fieldName);
} catch (Exception e) {
// log.error("###ACTUAL FAILURE!");
throw new RuntimeException(e);
}
}
}
private class PostDynamicFieldThread extends PutPostThread {
public PostDynamicFieldThread(RestTestHarness harness, Info info) {
super(harness, info);
fieldName = POST_FIELDNAME + "Thread" + info.numAddFieldPosts++;
}
public void run() {
try {
addFieldPost(harness, fieldName);
} catch (Exception e) {
// log.error("###ACTUAL FAILURE!");
throw new RuntimeException(e);
@ -275,28 +403,33 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
// First, add a bunch of fields via PUT and POST, as well as copyFields,
// but do it fast enough and verify shards' schemas after all of them are added
int numFields = 25;
int numAddFieldPuts = 0;
int numAddFieldPosts = 0;
final String putFieldName = "newfieldPutThread";
final String postFieldName = "newfieldPostThread";
Info info = new Info("Thread");
for (int i = 0; i <= numFields ; ++i) {
// System.err.println("###ITERATION: " + i);
int postHarness = r.nextInt(restTestHarnesses.size());
RestTestHarness publisher = restTestHarnesses.get(postHarness);
PutPostThread postThread = new PutPostThread(publisher, postFieldName + numAddFieldPosts++, false);
postThread.start();
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
PostFieldThread postFieldThread = new PostFieldThread(publisher, info);
postFieldThread.start();
int putHarness = r.nextInt(restTestHarnesses.size());
publisher = restTestHarnesses.get(putHarness);
PutPostThread putThread = new PutPostThread(publisher, putFieldName + numAddFieldPuts++, true);
putThread.start();
postThread.join();
putThread.join();
publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
PutFieldThread putFieldThread = new PutFieldThread(publisher, info);
putFieldThread.start();
String[] expectedAddFields = getExpectedFieldResponses(numAddFieldPuts, putFieldName,
numAddFieldPosts, postFieldName);
publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
PostDynamicFieldThread postDynamicFieldThread = new PostDynamicFieldThread(publisher, info);
postDynamicFieldThread.start();
publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
PutDynamicFieldThread putDynamicFieldThread = new PutDynamicFieldThread(publisher, info);
putDynamicFieldThread.start();
postFieldThread.join();
putFieldThread.join();
postDynamicFieldThread.join();
putDynamicFieldThread.join();
String[] expectedAddFields = getExpectedFieldResponses(info);
String[] expectedAddDynamicFields = getExpectedDynamicFieldResponses(info);
boolean success = false;
long maxTimeoutMillis = 100000;
@ -318,6 +451,18 @@ public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestB
response = client.query(request);
//System.err.println("###RESPONSE: " + response);
result = BaseTestHarness.validateXPath(response, expectedAddFields);
if (result != null) {
// System.err.println("###FAILURE!");
break;
}
// verify addDynamicFieldPuts and addDynamicFieldPosts
request = "/schema/dynamicfields?wt=xml";
response = client.query(request);
//System.err.println("###RESPONSE: " + response);
result = BaseTestHarness.validateXPath(response, expectedAddDynamicFields);
if (result != null) {
// System.err.println("###FAILURE!");
break;