mirror of https://github.com/apache/lucene.git
SOLR-6164: Copy Fields Schema additions are not distributed to other nodes
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1603300 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Steven Rowe
parent
726607a6fc
commit
bdd350df8f
|
|
@ -197,6 +197,9 @@ Bug Fixes
|
|||
|
||||
* SOLR-6129: DateFormatTransformer doesn't resolve dateTimeFormat. (Aaron LaBella via shalin)
|
||||
|
||||
* SOLR-6164: Copy Fields Schema additions are not distributed to other nodes.
|
||||
(Gregory Chanan via Steve Rowe)
|
||||
|
||||
Other Changes
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -591,44 +591,7 @@ public class IndexSchema {
|
|||
// expression = "/schema/copyField";
|
||||
|
||||
dynamicCopyFields = new DynamicCopy[] {};
|
||||
expression = "//" + COPY_FIELD;
|
||||
nodes = (NodeList) xpath.evaluate(expression, document, XPathConstants.NODESET);
|
||||
|
||||
for (int i=0; i<nodes.getLength(); i++) {
|
||||
node = nodes.item(i);
|
||||
NamedNodeMap attrs = node.getAttributes();
|
||||
|
||||
String source = DOMUtil.getAttr(attrs, SOURCE, COPY_FIELD + " definition");
|
||||
String dest = DOMUtil.getAttr(attrs, DESTINATION, COPY_FIELD + " definition");
|
||||
String maxChars = DOMUtil.getAttr(attrs, MAX_CHARS);
|
||||
int maxCharsInt = CopyField.UNLIMITED;
|
||||
if (maxChars != null) {
|
||||
try {
|
||||
maxCharsInt = Integer.parseInt(maxChars);
|
||||
} catch (NumberFormatException e) {
|
||||
log.warn("Couldn't parse " + MAX_CHARS + " attribute for " + COPY_FIELD + " from "
|
||||
+ source + " to " + dest + " as integer. The whole field will be copied.");
|
||||
}
|
||||
}
|
||||
|
||||
if (dest.equals(uniqueKeyFieldName)) {
|
||||
String msg = UNIQUE_KEY + " field ("+uniqueKeyFieldName+
|
||||
") can not be the " + DESTINATION + " of a " + COPY_FIELD + "(" + SOURCE + "=" +source+")";
|
||||
log.error(msg);
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
|
||||
|
||||
}
|
||||
|
||||
registerCopyField(source, dest, maxCharsInt);
|
||||
}
|
||||
|
||||
for (Map.Entry<SchemaField, Integer> entry : copyFieldTargetCounts.entrySet()) {
|
||||
if (entry.getValue() > 1 && !entry.getKey().multiValued()) {
|
||||
log.warn("Field " + entry.getKey().name + " is not multivalued "+
|
||||
"and destination for multiple " + COPY_FIELDS + " ("+
|
||||
entry.getValue()+")");
|
||||
}
|
||||
}
|
||||
loadCopyFields(document, xpath);
|
||||
|
||||
//Run the callbacks on SchemaAware now that everything else is done
|
||||
for (SchemaAware aware : schemaAware) {
|
||||
|
|
@ -746,6 +709,50 @@ public class IndexSchema {
|
|||
return explicitRequiredProp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the copy fields
|
||||
*/
|
||||
protected synchronized void loadCopyFields(Document document, XPath xpath) throws XPathExpressionException {
|
||||
String expression = "//" + COPY_FIELD;
|
||||
NodeList nodes = (NodeList)xpath.evaluate(expression, document, XPathConstants.NODESET);
|
||||
|
||||
for (int i=0; i<nodes.getLength(); i++) {
|
||||
Node node = nodes.item(i);
|
||||
NamedNodeMap attrs = node.getAttributes();
|
||||
|
||||
String source = DOMUtil.getAttr(attrs, SOURCE, COPY_FIELD + " definition");
|
||||
String dest = DOMUtil.getAttr(attrs, DESTINATION, COPY_FIELD + " definition");
|
||||
String maxChars = DOMUtil.getAttr(attrs, MAX_CHARS);
|
||||
|
||||
int maxCharsInt = CopyField.UNLIMITED;
|
||||
if (maxChars != null) {
|
||||
try {
|
||||
maxCharsInt = Integer.parseInt(maxChars);
|
||||
} catch (NumberFormatException e) {
|
||||
log.warn("Couldn't parse " + MAX_CHARS + " attribute for " + COPY_FIELD + " from "
|
||||
+ source + " to " + dest + " as integer. The whole field will be copied.");
|
||||
}
|
||||
}
|
||||
|
||||
if (dest.equals(uniqueKeyFieldName)) {
|
||||
String msg = UNIQUE_KEY + " field ("+uniqueKeyFieldName+
|
||||
") can not be the " + DESTINATION + " of a " + COPY_FIELD + "(" + SOURCE + "=" +source+")";
|
||||
log.error(msg);
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, msg);
|
||||
}
|
||||
|
||||
registerCopyField(source, dest, maxCharsInt);
|
||||
}
|
||||
|
||||
for (Map.Entry<SchemaField, Integer> entry : copyFieldTargetCounts.entrySet()) {
|
||||
if (entry.getValue() > 1 && !entry.getKey().multiValued()) {
|
||||
log.warn("Field " + entry.getKey().name + " is not multivalued "+
|
||||
"and destination for multiple " + COPY_FIELDS + " ("+
|
||||
entry.getValue()+")");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a sequence of path steps into a rooted path, by inserting slashes in front of each step.
|
||||
* @param steps The steps to join with slashes to form a path
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import java.nio.charset.StandardCharsets;
|
|||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
/** Solr-managed schema - non-user-editable, but can be mutable via internal and external REST API requests. */
|
||||
|
|
@ -338,6 +339,13 @@ public final class ManagedIndexSchema extends IndexSchema {
|
|||
Document document = schemaConf.getDocument();
|
||||
final XPath xpath = schemaConf.getXPath();
|
||||
newSchema.loadFields(document, xpath);
|
||||
// let's completely rebuild the copy fields from the schema in ZK.
|
||||
// create new copyField-related objects so we don't affect the
|
||||
// old schema
|
||||
newSchema.copyFieldsMap = new HashMap<>();
|
||||
newSchema.dynamicCopyFields = null;
|
||||
newSchema.copyFieldTargetCounts = new HashMap<>();
|
||||
newSchema.loadCopyFields(document, xpath);
|
||||
if (null != uniqueKeyField) {
|
||||
newSchema.requiredFields.add(uniqueKeyField);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,123 +0,0 @@
|
|||
package org.apache.solr.schema;
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
|
||||
import org.apache.solr.util.BaseTestHarness;
|
||||
import org.apache.solr.util.RESTfulServerProvider;
|
||||
import org.apache.solr.util.RestTestHarness;
|
||||
import org.eclipse.jetty.servlet.ServletHolder;
|
||||
import org.restlet.ext.servlet.ServerServlet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
public class TestCloudManagedSchemaAddField extends AbstractFullDistribZkTestBase {
|
||||
private static final Logger log = LoggerFactory.getLogger(TestCloudManagedSchemaAddField.class);
|
||||
|
||||
public TestCloudManagedSchemaAddField() {
|
||||
super();
|
||||
fixShardCount = true;
|
||||
|
||||
sliceCount = 4;
|
||||
shardCount = 8;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void initSysProperties() {
|
||||
System.setProperty("managed.schema.mutable", "true");
|
||||
System.setProperty("enable.update.log", "true");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getCloudSolrConfig() {
|
||||
return "solrconfig-managed-schema.xml";
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<ServletHolder,String> getExtraServlets() {
|
||||
final SortedMap<ServletHolder,String> extraServlets = new TreeMap<>();
|
||||
final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
|
||||
solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
|
||||
extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...'
|
||||
return extraServlets;
|
||||
}
|
||||
|
||||
private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
|
||||
|
||||
private void setupHarnesses() {
|
||||
for (int i = 0 ; i < clients.size() ; ++i) {
|
||||
final HttpSolrServer client = (HttpSolrServer)clients.get(i);
|
||||
RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
|
||||
@Override
|
||||
public String getBaseURL() {
|
||||
return client.getBaseURL();
|
||||
}
|
||||
});
|
||||
restTestHarnesses.add(harness);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doTest() throws Exception {
|
||||
setupHarnesses();
|
||||
|
||||
// First. add a bunch of fields, but do it fast enough
|
||||
// and verify shards' schemas after all of them are added
|
||||
int numFields = 25;
|
||||
for (int i = 1 ; i <= numFields ; ++i) {
|
||||
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
|
||||
String newFieldName = "newfield" + i;
|
||||
final String content = "{\"type\":\"text\",\"stored\":\"false\"}";
|
||||
String request = "/schema/fields/" + newFieldName + "?wt=xml";
|
||||
String response = publisher.put(request, content);
|
||||
String result = publisher.validateXPath
|
||||
(response, "/response/lst[@name='responseHeader']/int[@name='status'][.='0']");
|
||||
if (null != result) {
|
||||
fail("PUT REQUEST FAILED: xpath=" + result + " request=" + request
|
||||
+ " content=" + content + " response=" + response);
|
||||
}
|
||||
}
|
||||
|
||||
Thread.sleep(100000);
|
||||
|
||||
for (int i = 1 ; i <= numFields ; ++i) {
|
||||
String newFieldName = "newfield" + i;
|
||||
for (RestTestHarness client : restTestHarnesses) {
|
||||
String request = "/schema/fields/" + newFieldName + "?wt=xml";
|
||||
String response = client.query(request);
|
||||
String result = client.validateXPath(response,
|
||||
"/response/lst[@name='responseHeader']/int[@name='status'][.='0']",
|
||||
"/response/lst[@name='field']/str[@name='name'][.='" + newFieldName + "']");
|
||||
if (null != result) {
|
||||
if (response.contains("Field '" + newFieldName + "' not found.")) {
|
||||
String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
|
||||
log.error(msg);
|
||||
fail(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
package org.apache.solr.schema;
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
|
||||
import org.apache.solr.util.BaseTestHarness;
|
||||
import org.apache.solr.util.RESTfulServerProvider;
|
||||
import org.apache.solr.util.RestTestHarness;
|
||||
import org.eclipse.jetty.servlet.ServletHolder;
|
||||
import org.restlet.ext.servlet.ServerServlet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
public class TestCloudManagedSchemaAddFields extends AbstractFullDistribZkTestBase {
|
||||
private static final Logger log = LoggerFactory.getLogger(TestCloudManagedSchemaAddField.class);
|
||||
|
||||
public TestCloudManagedSchemaAddFields() {
|
||||
super();
|
||||
fixShardCount = true;
|
||||
|
||||
sliceCount = 4;
|
||||
shardCount = 8;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void initSysProperties() {
|
||||
System.setProperty("managed.schema.mutable", "true");
|
||||
System.setProperty("enable.update.log", "true");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getCloudSolrConfig() {
|
||||
return "solrconfig-managed-schema.xml";
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<ServletHolder,String> getExtraServlets() {
|
||||
final SortedMap<ServletHolder,String> extraServlets = new TreeMap<>();
|
||||
final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
|
||||
solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
|
||||
extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...'
|
||||
return extraServlets;
|
||||
}
|
||||
|
||||
private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
|
||||
|
||||
private void setupHarnesses() {
|
||||
for (int i = 0 ; i < clients.size() ; ++i) {
|
||||
final HttpSolrServer client = (HttpSolrServer)clients.get(i);
|
||||
RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
|
||||
@Override
|
||||
public String getBaseURL() {
|
||||
return client.getBaseURL();
|
||||
}
|
||||
});
|
||||
restTestHarnesses.add(harness);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doTest() throws Exception {
|
||||
setupHarnesses();
|
||||
|
||||
// First. add a bunch of fields, but do it fast enough
|
||||
// and verify shards' schemas after all of them are added
|
||||
int numFields = 200;
|
||||
for (int i = 1 ; i <= numFields ; ++i) {
|
||||
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
|
||||
String newFieldName = "newfield" + i;
|
||||
final String content = "[{\"name\":\""+newFieldName+"\",\"type\":\"text\",\"stored\":\"false\"}]";
|
||||
String request = "/schema/fields/?wt=xml";
|
||||
String response = publisher.post(request, content);
|
||||
String result = publisher.validateXPath
|
||||
(response, "/response/lst[@name='responseHeader']/int[@name='status'][.='0']");
|
||||
if (null != result) {
|
||||
fail("POST REQUEST FAILED: xpath=" + result + " request=" + request
|
||||
+ " content=" + content + " response=" + response);
|
||||
}
|
||||
}
|
||||
|
||||
Thread.sleep(100000);
|
||||
|
||||
for (int i = 1 ; i <= numFields ; ++i) {
|
||||
String newFieldName = "newfield" + i;
|
||||
for (RestTestHarness client : restTestHarnesses) {
|
||||
String request = "/schema/fields/" + newFieldName + "?wt=xml";
|
||||
String response = client.query(request);
|
||||
String result = client.validateXPath(response,
|
||||
"/response/lst[@name='responseHeader']/int[@name='status'][.='0']",
|
||||
"/response/lst[@name='field']/str[@name='name'][.='" + newFieldName + "']");
|
||||
if (null != result) {
|
||||
if (response.contains("Field '" + newFieldName + "' not found.")) {
|
||||
String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
|
||||
log.error(msg);
|
||||
fail(msg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
package org.apache.solr.schema;
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.solr.client.solrj.SolrServer;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
|
||||
import org.apache.solr.util.BaseTestHarness;
|
||||
import org.apache.solr.util.RESTfulServerProvider;
|
||||
import org.apache.solr.util.RestTestHarness;
|
||||
import org.eclipse.jetty.servlet.ServletHolder;
|
||||
import org.restlet.ext.servlet.ServerServlet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class TestCloudManagedSchemaConcurrent extends AbstractFullDistribZkTestBase {
|
||||
private static final Logger log = LoggerFactory.getLogger(TestCloudManagedSchemaConcurrent.class);
|
||||
private static final String SUCCESS_XPATH = "/response/lst[@name='responseHeader']/int[@name='status'][.='0']";
|
||||
|
||||
public TestCloudManagedSchemaConcurrent() {
|
||||
super();
|
||||
fixShardCount = true;
|
||||
|
||||
sliceCount = 4;
|
||||
shardCount = 8;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void initSysProperties() {
|
||||
System.setProperty("managed.schema.mutable", "true");
|
||||
System.setProperty("enable.update.log", "true");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getCloudSolrConfig() {
|
||||
return "solrconfig-managed-schema.xml";
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<ServletHolder,String> getExtraServlets() {
|
||||
final SortedMap<ServletHolder,String> extraServlets = new TreeMap<>();
|
||||
final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
|
||||
solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
|
||||
extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...'
|
||||
return extraServlets;
|
||||
}
|
||||
|
||||
private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
|
||||
|
||||
private void setupHarnesses() {
|
||||
for (final SolrServer client : clients) {
|
||||
RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
|
||||
@Override
|
||||
public String getBaseURL() {
|
||||
return ((HttpSolrServer)client).getBaseURL();
|
||||
}
|
||||
});
|
||||
restTestHarnesses.add(harness);
|
||||
}
|
||||
}
|
||||
|
||||
private void verifySuccess(String request, String response) throws Exception {
|
||||
String result = BaseTestHarness.validateXPath(response, SUCCESS_XPATH);
|
||||
if (null != result) {
|
||||
String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
|
||||
log.error(msg);
|
||||
fail(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private void addFieldPut(RestTestHarness publisher, String fieldName) throws Exception {
|
||||
final String content = "{\"type\":\"text\",\"stored\":\"false\"}";
|
||||
String request = "/schema/fields/" + fieldName + "?wt=xml";
|
||||
String response = publisher.put(request, content);
|
||||
verifySuccess(request, response);
|
||||
}
|
||||
|
||||
private void addFieldPost(RestTestHarness publisher, String fieldName) throws Exception {
|
||||
final String content = "[{\"name\":\""+fieldName+"\",\"type\":\"text\",\"stored\":\"false\"}]";
|
||||
String request = "/schema/fields/?wt=xml";
|
||||
String response = publisher.post(request, content);
|
||||
verifySuccess(request, response);
|
||||
}
|
||||
|
||||
private void copyField(RestTestHarness publisher, String source, String dest) throws Exception {
|
||||
final String content = "[{\"source\":\""+source+"\",\"dest\":[\""+dest+"\"]}]";
|
||||
String request = "/schema/copyfields/?wt=xml";
|
||||
String response = publisher.post(request, content);
|
||||
verifySuccess(request, response);
|
||||
}
|
||||
|
||||
private String[] getExpectedFieldResponses(int numAddFieldPuts, int numAddFieldPosts) {
|
||||
String[] expectedAddFields = new String[1 + numAddFieldPuts + numAddFieldPosts];
|
||||
expectedAddFields[0] = SUCCESS_XPATH;
|
||||
|
||||
for (int i = 0; i < numAddFieldPuts; ++i) {
|
||||
String newFieldName = "newfieldPut" + i;
|
||||
expectedAddFields[1 + i]
|
||||
= "/response/arr[@name='fields']/lst/str[@name='name'][.='" + newFieldName + "']";
|
||||
}
|
||||
|
||||
for (int i = 0; i < numAddFieldPosts; ++i) {
|
||||
String newFieldName = "newfieldPost" + i;
|
||||
expectedAddFields[1 + numAddFieldPuts + i]
|
||||
= "/response/arr[@name='fields']/lst/str[@name='name'][.='" + newFieldName + "']";
|
||||
}
|
||||
|
||||
return expectedAddFields;
|
||||
}
|
||||
|
||||
private String[] getExpectedCopyFieldResponses(List<CopyFieldInfo> copyFields) {
|
||||
ArrayList<String> expectedCopyFields = new ArrayList<>();
|
||||
expectedCopyFields.add(SUCCESS_XPATH);
|
||||
for (CopyFieldInfo cpi : copyFields) {
|
||||
String expectedSourceName = cpi.getSourceField();
|
||||
expectedCopyFields.add
|
||||
("/response/arr[@name='copyFields']/lst/str[@name='source'][.='" + expectedSourceName + "']");
|
||||
String expectedDestName = cpi.getDestField();
|
||||
expectedCopyFields.add
|
||||
("/response/arr[@name='copyFields']/lst/str[@name='dest'][.='" + expectedDestName + "']");
|
||||
}
|
||||
|
||||
return expectedCopyFields.toArray(new String[expectedCopyFields.size()]);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doTest() throws Exception {
|
||||
setupHarnesses();
|
||||
|
||||
// First, add a bunch of fields via PUT and POST, as well as copyFields,
|
||||
// but do it fast enough and verify shards' schemas after all of them are added
|
||||
int numFields = 100;
|
||||
int numAddFieldPuts = 0;
|
||||
int numAddFieldPosts = 0;
|
||||
List<CopyFieldInfo> copyFields = new ArrayList<>();
|
||||
|
||||
for (int i = 0; i <= numFields ; ++i) {
|
||||
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
|
||||
|
||||
int type = random().nextInt(3);
|
||||
if (type == 0) { // send an add field via PUT
|
||||
addFieldPut(publisher, "newfieldPut" + numAddFieldPuts++);
|
||||
}
|
||||
else if (type == 1) { // send an add field via POST
|
||||
addFieldPost(publisher, "newfieldPost" + numAddFieldPosts++);
|
||||
}
|
||||
else if (type == 2) { // send a copy field
|
||||
String sourceField = null;
|
||||
String destField = null;
|
||||
|
||||
int sourceType = random().nextInt(3);
|
||||
if (sourceType == 0) { // existing
|
||||
sourceField = "name";
|
||||
} else if (sourceType == 1) { // newly created
|
||||
sourceField = "copySource" + i;
|
||||
addFieldPut(publisher, sourceField);
|
||||
} else { // dynamic
|
||||
sourceField = "*_dynamicSource" + i + "_t";
|
||||
// * only supported if both src and dst use it
|
||||
destField = "*_dynamicDest" + i + "_t";
|
||||
}
|
||||
|
||||
if (destField == null) {
|
||||
int destType = random().nextInt(2);
|
||||
if (destType == 0) { // existing
|
||||
destField = "title";
|
||||
} else { // newly created
|
||||
destField = "copyDest" + i;
|
||||
addFieldPut(publisher, destField);
|
||||
}
|
||||
}
|
||||
copyField(publisher, sourceField, destField);
|
||||
copyFields.add(new CopyFieldInfo(sourceField, destField));
|
||||
}
|
||||
}
|
||||
|
||||
String[] expectedAddFields = getExpectedFieldResponses(numAddFieldPuts, numAddFieldPosts);
|
||||
String[] expectedCopyFields = getExpectedCopyFieldResponses(copyFields);
|
||||
|
||||
boolean success = false;
|
||||
long maxTimeoutMillis = 100000;
|
||||
long startTime = System.nanoTime();
|
||||
String request = null;
|
||||
String response = null;
|
||||
String result = null;
|
||||
|
||||
while ( ! success
|
||||
&& TimeUnit.MILLISECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS) < maxTimeoutMillis) {
|
||||
Thread.sleep(100);
|
||||
|
||||
for (RestTestHarness client : restTestHarnesses) {
|
||||
// verify addFieldPuts and addFieldPosts
|
||||
request = "/schema/fields?wt=xml";
|
||||
response = client.query(request);
|
||||
result = BaseTestHarness.validateXPath(response, expectedAddFields);
|
||||
if (result != null) {
|
||||
break;
|
||||
}
|
||||
|
||||
// verify copyFields
|
||||
request = "/schema/copyfields?wt=xml";
|
||||
response = client.query(request);
|
||||
result = BaseTestHarness.validateXPath(response, expectedCopyFields);
|
||||
if (result != null) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
success = (result == null);
|
||||
}
|
||||
if ( ! success) {
|
||||
String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
|
||||
log.error(msg);
|
||||
fail(msg);
|
||||
}
|
||||
}
|
||||
|
||||
private static class CopyFieldInfo {
|
||||
private String sourceField;
|
||||
private String destField;
|
||||
|
||||
public CopyFieldInfo(String sourceField, String destField) {
|
||||
this.sourceField = sourceField;
|
||||
this.destField = destField;
|
||||
}
|
||||
|
||||
public String getSourceField() { return sourceField; }
|
||||
public String getDestField() { return destField; }
|
||||
}
|
||||
}
|
||||
|
|
@ -1,114 +0,0 @@
|
|||
package org.apache.solr.schema;
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrServer;
|
||||
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
|
||||
import org.apache.solr.util.RESTfulServerProvider;
|
||||
import org.apache.solr.util.RestTestHarness;
|
||||
import org.eclipse.jetty.servlet.ServletHolder;
|
||||
import org.restlet.ext.servlet.ServerServlet;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.SortedMap;
|
||||
import java.util.TreeMap;
|
||||
|
||||
public class TestCloudManagedSchemaCopyFields extends AbstractFullDistribZkTestBase {
|
||||
private static final Logger log = LoggerFactory.getLogger(TestCloudManagedSchemaAddField.class);
|
||||
|
||||
public TestCloudManagedSchemaCopyFields() {
|
||||
super();
|
||||
fixShardCount = true;
|
||||
|
||||
sliceCount = 4;
|
||||
shardCount = 8;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void initSysProperties() {
|
||||
System.setProperty("managed.schema.mutable", "true");
|
||||
System.setProperty("enable.update.log", "true");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected String getCloudSolrConfig() {
|
||||
return "solrconfig-managed-schema.xml";
|
||||
}
|
||||
|
||||
@Override
|
||||
public SortedMap<ServletHolder,String> getExtraServlets() {
|
||||
final SortedMap<ServletHolder,String> extraServlets = new TreeMap<>();
|
||||
final ServletHolder solrRestApi = new ServletHolder("SolrSchemaRestApi", ServerServlet.class);
|
||||
solrRestApi.setInitParameter("org.restlet.application", "org.apache.solr.rest.SolrSchemaRestApi");
|
||||
extraServlets.put(solrRestApi, "/schema/*"); // '/schema/*' matches '/schema', '/schema/', and '/schema/whatever...'
|
||||
return extraServlets;
|
||||
}
|
||||
|
||||
private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
|
||||
|
||||
private void setupHarnesses() {
|
||||
for (int i = 0 ; i < clients.size() ; ++i) {
|
||||
final HttpSolrServer client = (HttpSolrServer)clients.get(i);
|
||||
RestTestHarness harness = new RestTestHarness(new RESTfulServerProvider() {
|
||||
@Override
|
||||
public String getBaseURL() {
|
||||
return client.getBaseURL();
|
||||
}
|
||||
});
|
||||
restTestHarnesses.add(harness);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doTest() throws Exception {
|
||||
setupHarnesses();
|
||||
|
||||
// First, add the same copy field directive a bunch of times.
|
||||
// Then verify each shard's schema has it.
|
||||
int numFields = 200;
|
||||
for (int i = 1 ; i <= numFields ; ++i) {
|
||||
RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
|
||||
final String content = "[{\"source\":\""+"sku1"+"\",\"dest\":[\"sku2\"]}]";
|
||||
String request = "/schema/copyfields/?wt=xml";
|
||||
String response = publisher.post(request, content);
|
||||
String result = publisher.validateXPath
|
||||
(response, "/response/lst[@name='responseHeader']/int[@name='status'][.='0']");
|
||||
if (null != result) {
|
||||
fail("POST REQUEST FAILED: xpath=" + result + " request=" + request
|
||||
+ " content=" + content + " response=" + response);
|
||||
}
|
||||
}
|
||||
|
||||
Thread.sleep(100000);
|
||||
|
||||
String request = "/schema/copyfields/?wt=xml&indent=on&source.fl=sku1";
|
||||
for (RestTestHarness client : restTestHarnesses) {
|
||||
String response = client.query(request);
|
||||
String result = client.validateXPath(response,
|
||||
"/response/lst[@name='responseHeader']/int[@name='status'][.='0']",
|
||||
"/response/arr[@name='copyFields']/lst/str[@name='dest'][.='sku2']");
|
||||
if (null != result) {
|
||||
fail("QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue