From 8989c774783a80cab6902e45f111cfe60ed15d49 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Mon, 14 Nov 2016 19:17:57 -0500 Subject: [PATCH] SOLR-9751: PreAnalyzedField can cause managed schema corruption --- solr/CHANGES.txt | 2 + .../org/apache/solr/schema/FieldType.java | 20 +++-- .../solr/schema/FieldTypePluginLoader.java | 61 ++++++++++------ .../solr/schema/HasImplicitIndexAnalyzer.java | 25 +++++++ .../apache/solr/schema/PreAnalyzedField.java | 2 +- .../conf/managed-schema | 41 +++++++++++ .../conf/solrconfig.xml | 51 +++++++++++++ ...reAnalyzedFieldManagedSchemaCloudTest.java | 73 +++++++++++++++++++ 8 files changed, 244 insertions(+), 31 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/schema/HasImplicitIndexAnalyzer.java create mode 100644 solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema create mode 100644 solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/solrconfig.xml create mode 100644 solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldManagedSchemaCloudTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 2ee215e7a6e..26beef7ed38 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -93,6 +93,8 @@ Bug Fixes * SOLR-9740: A bug in macro expansion of multi-valued parameters caused non-expanded values after the first expanded value in the same multi-valued parameter to be dropped. (Erik Hatcher, yonik) + +* SOLR-9751: PreAnalyzedField can cause managed schema corruption. (Steve Rowe) Other Changes diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 6556ddb77f6..f3fdcba146c 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -879,13 +879,19 @@ public abstract class FieldType extends FieldProperties { namedPropertyValues.add(SIMILARITY, getSimilarityFactory().getNamedPropertyValues()); } - if (isExplicitAnalyzer()) { - String analyzerProperty = isExplicitQueryAnalyzer() ? INDEX_ANALYZER : ANALYZER; - namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getIndexAnalyzer())); - } - if (isExplicitQueryAnalyzer()) { - String analyzerProperty = isExplicitAnalyzer() ? QUERY_ANALYZER : ANALYZER; - namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getQueryAnalyzer())); + if (this instanceof HasImplicitIndexAnalyzer) { + if (isExplicitQueryAnalyzer()) { + namedPropertyValues.add(QUERY_ANALYZER, getAnalyzerProperties(getQueryAnalyzer())); + } + } else { + if (isExplicitAnalyzer()) { + String analyzerProperty = isExplicitQueryAnalyzer() ? INDEX_ANALYZER : ANALYZER; + namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getIndexAnalyzer())); + } + if (isExplicitQueryAnalyzer()) { + String analyzerProperty = isExplicitAnalyzer() ? QUERY_ANALYZER : ANALYZER; + namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getQueryAnalyzer())); + } } if (this instanceof TextField) { if (((TextField)this).isExplicitMultiTermAnalyzer()) { diff --git a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java index f7e9b0e602f..f332934abde 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java @@ -110,31 +110,46 @@ public final class FieldTypePluginLoader if (null != simFactory) { ft.setSimilarity(simFactory); } - - if (null == queryAnalyzer) { - queryAnalyzer = analyzer; - ft.setIsExplicitQueryAnalyzer(false); - } else { - ft.setIsExplicitQueryAnalyzer(true); - } - if (null == analyzer) { - analyzer = queryAnalyzer; - ft.setIsExplicitAnalyzer(false); - } else { - ft.setIsExplicitAnalyzer(true); - } - if (null != analyzer) { - ft.setIndexAnalyzer(analyzer); - ft.setQueryAnalyzer(queryAnalyzer); - if (ft instanceof TextField) { - if (null == multiAnalyzer) { - multiAnalyzer = constructMultiTermAnalyzer(queryAnalyzer); - ((TextField)ft).setIsExplicitMultiTermAnalyzer(false); - } else { - ((TextField)ft).setIsExplicitMultiTermAnalyzer(true); + if (ft instanceof HasImplicitIndexAnalyzer) { + ft.setIsExplicitAnalyzer(false); + if (null != queryAnalyzer && null != analyzer) { + if (log.isWarnEnabled()) { + log.warn("Ignoring index-time analyzer for field: " + name); + } + } else if (null == queryAnalyzer) { // Accept non-query-time analyzer as a query-time analyzer + queryAnalyzer = analyzer; + } + if (null != queryAnalyzer) { + ft.setIsExplicitQueryAnalyzer(true); + ft.setQueryAnalyzer(queryAnalyzer); + } + } else { + if (null == queryAnalyzer) { + queryAnalyzer = analyzer; + ft.setIsExplicitQueryAnalyzer(false); + } else { + ft.setIsExplicitQueryAnalyzer(true); + } + if (null == analyzer) { + analyzer = queryAnalyzer; + ft.setIsExplicitAnalyzer(false); + } else { + ft.setIsExplicitAnalyzer(true); + } + + if (null != analyzer) { + ft.setIndexAnalyzer(analyzer); + ft.setQueryAnalyzer(queryAnalyzer); + if (ft instanceof TextField) { + if (null == multiAnalyzer) { + multiAnalyzer = constructMultiTermAnalyzer(queryAnalyzer); + ((TextField)ft).setIsExplicitMultiTermAnalyzer(false); + } else { + ((TextField)ft).setIsExplicitMultiTermAnalyzer(true); + } + ((TextField)ft).setMultiTermAnalyzer(multiAnalyzer); } - ((TextField)ft).setMultiTermAnalyzer(multiAnalyzer); } } if (ft instanceof SchemaAware){ diff --git a/solr/core/src/java/org/apache/solr/schema/HasImplicitIndexAnalyzer.java b/solr/core/src/java/org/apache/solr/schema/HasImplicitIndexAnalyzer.java new file mode 100644 index 00000000000..9722852f6cf --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/HasImplicitIndexAnalyzer.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +/** + * Marker interface for field types that have an implicit (non-user-configurable) + * index-time schema. + */ +public interface HasImplicitIndexAnalyzer { +} diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java index 09d3590a21f..6fdb457c176 100644 --- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java +++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java @@ -50,7 +50,7 @@ import static org.apache.solr.common.params.CommonParams.JSON; * Pre-analyzed field type provides a way to index a serialized token stream, * optionally with an independent stored value of a field. */ -public class PreAnalyzedField extends TextField { +public class PreAnalyzedField extends TextField implements HasImplicitIndexAnalyzer { private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** Init argument name. Value is a fully-qualified class name of the parser diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema b/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema new file mode 100644 index 00000000000..e70e02b36f8 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + id + diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/solrconfig.xml new file mode 100644 index 00000000000..1beaf76877f --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/solrconfig.xml @@ -0,0 +1,51 @@ + + + + + + + + + ${solr.data.dir:} + + + + + ${managed.schema.mutable:true} + managed-schema + + + ${tests.luceneMatchVersion:LATEST} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + explicit + true + text + + + + diff --git a/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldManagedSchemaCloudTest.java b/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldManagedSchemaCloudTest.java new file mode 100644 index 00000000000..04e1be0d04b --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldManagedSchemaCloudTest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; +import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldResponse; +import org.apache.solr.client.solrj.response.schema.SchemaResponse.UpdateResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.cloud.DocCollection; +import org.junit.BeforeClass; +import org.junit.Test; + +public class PreAnalyzedFieldManagedSchemaCloudTest extends SolrCloudTestCase { + + private static final String COLLECTION = "managed-preanalyzed"; + private static final String CONFIG = "cloud-managed-preanalyzed"; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(2).addConfig(CONFIG, configset(CONFIG)).configure(); + CollectionAdminRequest.createCollection(COLLECTION, CONFIG, 2, 1) + .setMaxShardsPerNode(1) + .process(cluster.getSolrClient()); + cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS, + (n, c) -> DocCollection.isFullyActive(n, c, 2, 1)); + } + + @Test + public void testAdd2Fields() throws Exception { + addField(keyValueArrayToMap("name", "field1", "type", "string")); + addField(keyValueArrayToMap("name", "field2", "type", "string")); + } + + private void addField(Map field) throws Exception { + CloudSolrClient client = cluster.getSolrClient(); + UpdateResponse addFieldResponse = new SchemaRequest.AddField(field).process(client, COLLECTION); + assertNotNull(addFieldResponse); + assertEquals(0, addFieldResponse.getStatus()); + assertNull(addFieldResponse.getResponse().get("errors")); + FieldResponse fieldResponse = new SchemaRequest.Field(field.get("name").toString()).process(client, COLLECTION); + assertNotNull(fieldResponse); + assertEquals(0, fieldResponse.getStatus()); + } + + private Map keyValueArrayToMap(String... alternatingKeysAndValues) { + Map map = new HashMap<>(); + for (int i = 0 ; i < alternatingKeysAndValues.length ; i += 2) + map.put(alternatingKeysAndValues[i], alternatingKeysAndValues[i + 1]); + return map; + } +} +