Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-13795: Managed schema should do a core reload in standalone mode. 

Fixes #902

(cherry picked from commit 22e96697de)
This commit is contained in:
Thomas Wöckinger 2019-10-04 13:05:01 -04:00 committed by David Smiley
parent e734b40375
commit 95e54196fc
6 changed files with 97 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
solr/CHANGES.txt
View File

@ -135,6 +135,9 @@ Improvements
* SOLR-13771: Add -v and -m to ulimit section of reference guide and bin/solr checks (Erick Erickson) * SOLR-13771: Add -v and -m to ulimit section of reference guide and bin/solr checks (Erick Erickson)
* SOLR-13795: Managed schema operations should do a core reload in Solr standalone mode.
(Thomas Wöckinger via David Smiley)
Bug Fixes Bug Fixes
---------------------- ----------------------

108
solr/core/src/java/org/apache/solr/schema/IndexSchema.java
View File

@ -138,7 +138,7 @@ public class IndexSchema {
protected List<SchemaField> fieldsWithDefaultValue = new ArrayList<>(); protected List<SchemaField> fieldsWithDefaultValue = new ArrayList<>();
protected Collection<SchemaField> requiredFields = new HashSet<>(); protected Collection<SchemaField> requiredFields = new HashSet<>();
protected volatile DynamicField[] dynamicFields; protected DynamicField[] dynamicFields = new DynamicField[] {};
public DynamicField[] getDynamicFields() { return dynamicFields; } public DynamicField[] getDynamicFields() { return dynamicFields; }
protected Map<String, SchemaField> dynamicFieldCache = new ConcurrentHashMap<>(); protected Map<String, SchemaField> dynamicFieldCache = new ConcurrentHashMap<>();
@ -151,7 +151,7 @@ public class IndexSchema {
protected Map<String, List<CopyField>> copyFieldsMap = new HashMap<>(); protected Map<String, List<CopyField>> copyFieldsMap = new HashMap<>();
public Map<String,List<CopyField>> getCopyFieldsMap() { return Collections.unmodifiableMap(copyFieldsMap); } public Map<String,List<CopyField>> getCopyFieldsMap() { return Collections.unmodifiableMap(copyFieldsMap); }
protected DynamicCopy[] dynamicCopyFields; protected DynamicCopy[] dynamicCopyFields = new DynamicCopy[] {};
public DynamicCopy[] getDynamicCopyFields() { return dynamicCopyFields; } public DynamicCopy[] getDynamicCopyFields() { return dynamicCopyFields; }
private Map<FieldType, PayloadDecoder> decoders = new HashMap<>(); // cache to avoid scanning token filters repeatedly, unnecessarily private Map<FieldType, PayloadDecoder> decoders = new HashMap<>(); // cache to avoid scanning token filters repeatedly, unnecessarily
@ -962,18 +962,12 @@ public class IndexSchema {
private void incrementCopyFieldTargetCount(SchemaField dest) { private void incrementCopyFieldTargetCount(SchemaField dest) {
copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1); copyFieldTargetCounts.put(dest, copyFieldTargetCounts.containsKey(dest) ? copyFieldTargetCounts.get(dest) + 1 : 1);
} }
private void registerDynamicCopyField( DynamicCopy dcopy ) { private void registerDynamicCopyField(DynamicCopy dcopy) {
if( dynamicCopyFields == null ) { DynamicCopy[] temp = new DynamicCopy[dynamicCopyFields.length + 1];
dynamicCopyFields = new DynamicCopy[] {dcopy}; System.arraycopy(dynamicCopyFields, 0, temp, 0, dynamicCopyFields.length);
} temp[temp.length - 1] = dcopy;
else { dynamicCopyFields = temp;
DynamicCopy[] temp = new DynamicCopy[dynamicCopyFields.length+1];
System.arraycopy(dynamicCopyFields,0,temp,0,dynamicCopyFields.length);
temp[temp.length -1] = dcopy;
dynamicCopyFields = temp;
}
log.trace("Dynamic Copy Field:" + dcopy);
} }
static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) { static SimilarityFactory readSimilarity(SolrResourceLoader loader, Node node) {
@ -1337,11 +1331,9 @@ public class IndexSchema {
} }
} }
} }
if (null != dynamicCopyFields) { for (DynamicCopy dynamicCopy : dynamicCopyFields) {
for (DynamicCopy dynamicCopy : dynamicCopyFields) { if (dynamicCopy.getDestFieldName().equals(destField)) {
if (dynamicCopy.getDestFieldName().equals(destField)) { fieldNames.add(dynamicCopy.getRegex());
fieldNames.add(dynamicCopy.getRegex());
}
} }
} }
return fieldNames; return fieldNames;
@ -1356,11 +1348,9 @@ public class IndexSchema {
// This is useful when we need the maxSize param of each CopyField // This is useful when we need the maxSize param of each CopyField
public List<CopyField> getCopyFieldsList(final String sourceField){ public List<CopyField> getCopyFieldsList(final String sourceField){
final List<CopyField> result = new ArrayList<>(); final List<CopyField> result = new ArrayList<>();
if (null != dynamicCopyFields) { for (DynamicCopy dynamicCopy : dynamicCopyFields) {
for (DynamicCopy dynamicCopy : dynamicCopyFields) { if (dynamicCopy.matches(sourceField)) {
if (dynamicCopy.matches(sourceField)) { result.add(new CopyField(getField(sourceField), dynamicCopy.getTargetField(sourceField), dynamicCopy.maxChars));
result.add(new CopyField(getField(sourceField), dynamicCopy.getTargetField(sourceField), dynamicCopy.maxChars));
}
} }
} }
List<CopyField> fixedCopyFields = copyFieldsMap.get(sourceField); List<CopyField> fixedCopyFields = copyFieldsMap.get(sourceField);
@ -1556,48 +1546,46 @@ public class IndexSchema {
} }
} }
} }
if (null != dynamicCopyFields) { for (IndexSchema.DynamicCopy dynamicCopy : dynamicCopyFields) {
for (IndexSchema.DynamicCopy dynamicCopy : dynamicCopyFields) { final String source = dynamicCopy.getRegex();
final String source = dynamicCopy.getRegex(); final String destination = dynamicCopy.getDestFieldName();
final String destination = dynamicCopy.getDestFieldName(); if ((null == requestedSourceFields || requestedSourceFields.contains(source))
if ((null == requestedSourceFields || requestedSourceFields.contains(source)) && (null == requestedDestinationFields || requestedDestinationFields.contains(destination))) {
&& (null == requestedDestinationFields || requestedDestinationFields.contains(destination))) { SimpleOrderedMap<Object> dynamicCopyProps = new SimpleOrderedMap<>();
SimpleOrderedMap<Object> dynamicCopyProps = new SimpleOrderedMap<>();
dynamicCopyProps.add(SOURCE, dynamicCopy.getRegex()); dynamicCopyProps.add(SOURCE, dynamicCopy.getRegex());
if (showDetails) { if (showDetails) {
IndexSchema.DynamicField sourceDynamicBase = dynamicCopy.getSourceDynamicBase(); IndexSchema.DynamicField sourceDynamicBase = dynamicCopy.getSourceDynamicBase();
if (null != sourceDynamicBase) { if (null != sourceDynamicBase) {
dynamicCopyProps.add(SOURCE_DYNAMIC_BASE, sourceDynamicBase.getRegex()); dynamicCopyProps.add(SOURCE_DYNAMIC_BASE, sourceDynamicBase.getRegex());
} else if (source.contains("*")) { } else if (source.contains("*")) {
List<String> sourceExplicitFields = new ArrayList<>(); List<String> sourceExplicitFields = new ArrayList<>();
Pattern pattern = Pattern.compile(source.replace("*", ".*")); // glob->regex Pattern pattern = Pattern.compile(source.replace("*", ".*")); // glob->regex
for (String field : fields.keySet()) { for (String field : fields.keySet()) {
if (pattern.matcher(field).matches()) { if (pattern.matcher(field).matches()) {
sourceExplicitFields.add(field); sourceExplicitFields.add(field);
}
}
if (sourceExplicitFields.size() > 0) {
Collections.sort(sourceExplicitFields);
dynamicCopyProps.add(SOURCE_EXPLICIT_FIELDS, sourceExplicitFields);
} }
} }
} if (sourceExplicitFields.size() > 0) {
Collections.sort(sourceExplicitFields);
dynamicCopyProps.add(DESTINATION, dynamicCopy.getDestFieldName()); dynamicCopyProps.add(SOURCE_EXPLICIT_FIELDS, sourceExplicitFields);
if (showDetails) {
IndexSchema.DynamicField destDynamicBase = dynamicCopy.getDestDynamicBase();
if (null != destDynamicBase) {
dynamicCopyProps.add(DESTINATION_DYNAMIC_BASE, destDynamicBase.getRegex());
} }
} }
if (0 != dynamicCopy.getMaxChars()) {
dynamicCopyProps.add(MAX_CHARS, dynamicCopy.getMaxChars());
}
copyFieldProperties.add(dynamicCopyProps);
} }
dynamicCopyProps.add(DESTINATION, dynamicCopy.getDestFieldName());
if (showDetails) {
IndexSchema.DynamicField destDynamicBase = dynamicCopy.getDestDynamicBase();
if (null != destDynamicBase) {
dynamicCopyProps.add(DESTINATION_DYNAMIC_BASE, destDynamicBase.getRegex());
}
}
if (0 != dynamicCopy.getMaxChars()) {
dynamicCopyProps.add(MAX_CHARS, dynamicCopy.getMaxChars());
}
copyFieldProperties.add(dynamicCopyProps);
} }
} }
return copyFieldProperties; return copyFieldProperties;

40
solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
View File

@ -81,7 +81,7 @@ import org.xml.sax.InputSource;
/** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */ /** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */
public final class ManagedIndexSchema extends IndexSchema { public final class ManagedIndexSchema extends IndexSchema {
private boolean isMutable = false; private final boolean isMutable;
@Override public boolean isMutable() { return isMutable; } @Override public boolean isMutable() { return isMutable; }
@ -654,7 +654,7 @@ public final class ManagedIndexSchema extends IndexSchema {
System.arraycopy(newSchema.dynamicFields, dfPos + 1, temp, dfPos, newSchema.dynamicFields.length - dfPos - 1); System.arraycopy(newSchema.dynamicFields, dfPos + 1, temp, dfPos, newSchema.dynamicFields.length - dfPos - 1);
newSchema.dynamicFields = temp; newSchema.dynamicFields = temp;
} else { } else {
newSchema.dynamicFields = new DynamicField[0]; newSchema.dynamicFields = new DynamicField[] {};
} }
} }
// After removing all dynamic fields, rebuild affected dynamic copy fields. // After removing all dynamic fields, rebuild affected dynamic copy fields.
@ -840,26 +840,24 @@ public final class ManagedIndexSchema extends IndexSchema {
boolean found = false; boolean found = false;
if (null == destSchemaField || null == sourceSchemaField) { // Must be dynamic copy field if (null == destSchemaField || null == sourceSchemaField) { // Must be dynamic copy field
if (dynamicCopyFields != null) { for (int i = 0; i < dynamicCopyFields.length; ++i) {
for (int i = 0 ; i < dynamicCopyFields.length ; ++i) { DynamicCopy dynamicCopy = dynamicCopyFields[i];
DynamicCopy dynamicCopy = dynamicCopyFields[i]; if (source.equals(dynamicCopy.getRegex()) && dest.equals(dynamicCopy.getDestFieldName())) {
if (source.equals(dynamicCopy.getRegex()) && dest.equals(dynamicCopy.getDestFieldName())) { found = true;
found = true; SchemaField destinationPrototype = dynamicCopy.getDestination().getPrototype();
SchemaField destinationPrototype = dynamicCopy.getDestination().getPrototype(); if (copyFieldTargetCounts.containsKey(destinationPrototype)) {
if (copyFieldTargetCounts.containsKey(destinationPrototype)) { decrementCopyFieldTargetCount(destinationPrototype);
decrementCopyFieldTargetCount(destinationPrototype);
}
if (dynamicCopyFields.length > 1) {
DynamicCopy[] temp = new DynamicCopy[dynamicCopyFields.length - 1];
System.arraycopy(dynamicCopyFields, 0, temp, 0, i);
// skip over the dynamic copy field to be deleted
System.arraycopy(dynamicCopyFields, i + 1, temp, i, dynamicCopyFields.length - i - 1);
dynamicCopyFields = temp;
} else {
dynamicCopyFields = null;
}
break;
} }
if (dynamicCopyFields.length > 1) {
DynamicCopy[] temp = new DynamicCopy[dynamicCopyFields.length - 1];
System.arraycopy(dynamicCopyFields, 0, temp, 0, i);
// skip over the dynamic copy field to be deleted
System.arraycopy(dynamicCopyFields, i + 1, temp, i, dynamicCopyFields.length - i - 1);
dynamicCopyFields = temp;
} else {
dynamicCopyFields = new DynamicCopy[] {};
}
break;
} }
} }
} else { // non-dynamic copy field directive } else { // non-dynamic copy field directive

1
solr/core/src/java/org/apache/solr/schema/SchemaManager.java
View File

@ -138,6 +138,7 @@ public class SchemaManager {
//only for non cloud stuff //only for non cloud stuff
managedIndexSchema.persistManagedSchema(false); managedIndexSchema.persistManagedSchema(false);
core.setLatestSchema(managedIndexSchema); core.setLatestSchema(managedIndexSchema);
core.getCoreContainer().reload(core.getName());
} catch (SolrException e) { } catch (SolrException e) {
log.warn(errorMsg); log.warn(errorMsg);
errors = singletonList(errorMsg + e.getMessage()); errors = singletonList(errorMsg + e.getMessage());

24
solr/core/src/test/org/apache/solr/rest/schema/TestBulkSchemaAPI.java
View File

@ -20,6 +20,7 @@ import java.io.File;
import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodHandles;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -32,6 +33,8 @@ import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.search.similarities.DFISimilarity; import org.apache.lucene.search.similarities.DFISimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper; import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrCore;
@ -43,6 +46,7 @@ import org.apache.solr.util.RestTestBase;
import org.apache.solr.util.RestTestHarness; import org.apache.solr.util.RestTestHarness;
import org.junit.After; import org.junit.After;
import org.junit.Before; import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -184,8 +188,6 @@ public class TestBulkSchemaAPI extends RestTestBase {
response = restTestHarness.post("/schema", json(addFieldTypeAnalyzerWithClass + suffix)); response = restTestHarness.post("/schema", json(addFieldTypeAnalyzerWithClass + suffix));
map = (Map) fromJSONString(response); map = (Map) fromJSONString(response);
assertNull(response, map.get("error")); assertNull(response, map.get("error"));
restTestHarness.checkAdminResponseStatus("/admin/cores?wt=xml&action=RELOAD&core=" + coreName, "0");
map = getObj(restTestHarness, "myNewTextFieldWithAnalyzerClass", "fieldTypes"); map = getObj(restTestHarness, "myNewTextFieldWithAnalyzerClass", "fieldTypes");
assertNotNull(map); assertNotNull(map);
@ -901,6 +903,24 @@ public class TestBulkSchemaAPI extends RestTestBase {
} }
@Test
public void testAddNewFieldAndQuery() throws Exception {
getSolrClient().add(Arrays.asList(
sdoc("id", "1", "term_s", "tux")));
getSolrClient().commit(true, true);
Map<String,Object> attrs = new HashMap<>();
attrs.put("name", "newstringtestfield");
attrs.put("type", "string");
new SchemaRequest.AddField(attrs).process(getSolrClient());
SolrQuery query = new SolrQuery("*:*");
query.addFacetField("newstringtestfield");
int size = getSolrClient().query(query).getResults().size();
assertEquals(1, size);
}
public void testSimilarityParser() throws Exception { public void testSimilarityParser() throws Exception {
RestTestHarness harness = restTestHarness; RestTestHarness harness = restTestHarness;

10
solr/solrj/src/test/org/apache/solr/client/solrj/request/SchemaTest.java
View File

@ -16,6 +16,10 @@
*/ */
package org.apache.solr.client.solrj.request; package org.apache.solr.client.solrj.request;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
@ -47,10 +51,6 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.restlet.ext.servlet.ServerServlet; import org.restlet.ext.servlet.ServerServlet;
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
/** /**
* Test the functionality (accuracy and failure) of the methods exposed by the classes * Test the functionality (accuracy and failure) of the methods exposed by the classes
* {@link SchemaRequest} and {@link SchemaResponse}. * {@link SchemaRequest} and {@link SchemaResponse}.
@ -622,8 +622,6 @@ public class SchemaTest extends RestTestBase {
SchemaResponse.UpdateResponse addFieldTypeResponse = addFieldTypeRequest.process(getSolrClient()); SchemaResponse.UpdateResponse addFieldTypeResponse = addFieldTypeRequest.process(getSolrClient());
assertValidSchemaResponse(addFieldTypeResponse); assertValidSchemaResponse(addFieldTypeResponse);
restTestHarness.reload();
SchemaRequest.FieldType fieldTypeRequest = new SchemaRequest.FieldType(fieldTypeName); SchemaRequest.FieldType fieldTypeRequest = new SchemaRequest.FieldType(fieldTypeName);
SchemaResponse.FieldTypeResponse newFieldTypeResponse = fieldTypeRequest.process(getSolrClient()); SchemaResponse.FieldTypeResponse newFieldTypeResponse = fieldTypeRequest.process(getSolrClient());
assertValidSchemaResponse(newFieldTypeResponse); assertValidSchemaResponse(newFieldTypeResponse);