SOLR-9751: PreAnalyzedField can cause managed schema corruption

solr/CHANGES.txt

@@ -93,6 +93,8 @@ Bug Fixes
 * SOLR-9740: A bug in macro expansion of multi-valued parameters caused non-expanded values
   after the first expanded value in the same multi-valued parameter to be dropped.
   (Erik Hatcher, yonik)
+  
+* SOLR-9751: PreAnalyzedField can cause managed schema corruption. (Steve Rowe)
 
 
 Other Changes

solr/core/src/java/org/apache/solr/schema/FieldType.java

@@ -879,13 +879,19 @@ public abstract class FieldType extends FieldProperties {
       namedPropertyValues.add(SIMILARITY, getSimilarityFactory().getNamedPropertyValues());
     }
     
-    if (isExplicitAnalyzer()) {
-      String analyzerProperty = isExplicitQueryAnalyzer() ? INDEX_ANALYZER : ANALYZER;
-      namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getIndexAnalyzer()));
-    } 
-    if (isExplicitQueryAnalyzer()) {
-      String analyzerProperty = isExplicitAnalyzer() ? QUERY_ANALYZER : ANALYZER;
-      namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getQueryAnalyzer()));
+    if (this instanceof HasImplicitIndexAnalyzer) {
+      if (isExplicitQueryAnalyzer()) {
+        namedPropertyValues.add(QUERY_ANALYZER, getAnalyzerProperties(getQueryAnalyzer()));
+      }
+    } else {
+      if (isExplicitAnalyzer()) {
+        String analyzerProperty = isExplicitQueryAnalyzer() ? INDEX_ANALYZER : ANALYZER;
+        namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getIndexAnalyzer()));
+      }
+      if (isExplicitQueryAnalyzer()) {
+        String analyzerProperty = isExplicitAnalyzer() ? QUERY_ANALYZER : ANALYZER;
+        namedPropertyValues.add(analyzerProperty, getAnalyzerProperties(getQueryAnalyzer()));
+      }
     }
     if (this instanceof TextField) {
       if (((TextField)this).isExplicitMultiTermAnalyzer()) {

solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java

@@ -110,31 +110,46 @@ public final class FieldTypePluginLoader
     if (null != simFactory) {
       ft.setSimilarity(simFactory);
     }
-    
-    if (null == queryAnalyzer) {
-      queryAnalyzer = analyzer;
-      ft.setIsExplicitQueryAnalyzer(false);
-    } else {
-      ft.setIsExplicitQueryAnalyzer(true);
-    }
-    if (null == analyzer) {
-      analyzer = queryAnalyzer;
-      ft.setIsExplicitAnalyzer(false);
-    } else {
-      ft.setIsExplicitAnalyzer(true);
-    }
 
-    if (null != analyzer) {
-      ft.setIndexAnalyzer(analyzer);
-      ft.setQueryAnalyzer(queryAnalyzer);
-      if (ft instanceof TextField) {
-        if (null == multiAnalyzer) {
-          multiAnalyzer = constructMultiTermAnalyzer(queryAnalyzer);
-          ((TextField)ft).setIsExplicitMultiTermAnalyzer(false);
-        } else {
-          ((TextField)ft).setIsExplicitMultiTermAnalyzer(true);
+    if (ft instanceof HasImplicitIndexAnalyzer) {
+      ft.setIsExplicitAnalyzer(false);
+      if (null != queryAnalyzer && null != analyzer) {
+        if (log.isWarnEnabled()) {
+          log.warn("Ignoring index-time analyzer for field: " + name);
+        }
+      } else if (null == queryAnalyzer) { // Accept non-query-time analyzer as a query-time analyzer 
+        queryAnalyzer = analyzer;
+      }
+      if (null != queryAnalyzer) {
+        ft.setIsExplicitQueryAnalyzer(true);
+        ft.setQueryAnalyzer(queryAnalyzer);
+      }
+    } else {
+      if (null == queryAnalyzer) {
+        queryAnalyzer = analyzer;
+        ft.setIsExplicitQueryAnalyzer(false);
+      } else {
+        ft.setIsExplicitQueryAnalyzer(true);
+      }
+      if (null == analyzer) {
+        analyzer = queryAnalyzer;
+        ft.setIsExplicitAnalyzer(false);
+      } else {
+        ft.setIsExplicitAnalyzer(true);
+      }
+  
+      if (null != analyzer) {
+        ft.setIndexAnalyzer(analyzer);
+        ft.setQueryAnalyzer(queryAnalyzer);
+        if (ft instanceof TextField) {
+          if (null == multiAnalyzer) {
+            multiAnalyzer = constructMultiTermAnalyzer(queryAnalyzer);
+            ((TextField)ft).setIsExplicitMultiTermAnalyzer(false);
+          } else {
+            ((TextField)ft).setIsExplicitMultiTermAnalyzer(true);
+          }
+          ((TextField)ft).setMultiTermAnalyzer(multiAnalyzer);
         }
-        ((TextField)ft).setMultiTermAnalyzer(multiAnalyzer);
       }
     }
     if (ft instanceof SchemaAware){

solr/core/src/java/org/apache/solr/schema/HasImplicitIndexAnalyzer.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.schema;
+
+/**
+ * Marker interface for field types that have an implicit (non-user-configurable)
+ * index-time schema.
+ */
+public interface HasImplicitIndexAnalyzer {
+}

solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java

@@ -50,7 +50,7 @@ import static org.apache.solr.common.params.CommonParams.JSON;
  * Pre-analyzed field type provides a way to index a serialized token stream,
  * optionally with an independent stored value of a field.
  */
-public class PreAnalyzedField extends TextField {
+public class PreAnalyzedField extends TextField implements HasImplicitIndexAnalyzer {
   private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   /** Init argument name. Value is a fully-qualified class name of the parser

solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/managed-schema

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="managed-preanalyzed" version="1.6">
+  <fieldType name="string" class="solr.StrField"/>
+  <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+  <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+
+  <fieldType name="preanalyzed-no-analyzer" class="solr.PreAnalyzedField" parserImpl="json"/>
+  <fieldType name="preanalyzed-with-analyzer" class="solr.PreAnalyzedField">
+    <analyzer>
+      <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+    </analyzer>
+  </fieldType>
+  <fieldType name="preanalyzed-with-query-analyzer" class="solr.PreAnalyzedField">
+    <analyzer type="query">
+      <tokenizer class="solr.StandardTokenizerFactory"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+    </analyzer>
+  </fieldType>
+
+  <!-- for versioning -->
+  <field name="_version_" type="long" indexed="true" stored="true"/>
+  <field name="_root_" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
+  <field name="id" type="string" indexed="true" stored="true"/>
+  <uniqueKey>id</uniqueKey>
+</schema>

solr/core/src/test-files/solr/configsets/cloud-managed-preanalyzed/conf/solrconfig.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- Minimal solrconfig.xml with /select, /admin and /update only -->
+
+<config>
+
+  <dataDir>${solr.data.dir:}</dataDir>
+
+  <directoryFactory name="DirectoryFactory"
+                    class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
+
+  <schemaFactory class="ManagedIndexSchemaFactory">
+    <bool name="mutable">${managed.schema.mutable:true}</bool>
+    <str name="managedSchemaResourceName">managed-schema</str>
+  </schemaFactory>
+
+  <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+  <updateHandler class="solr.DirectUpdateHandler2">
+    <commitWithin>
+      <softCommit>${solr.commitwithin.softcommit:true}</softCommit>
+    </commitWithin>
+    <updateLog></updateLog>
+  </updateHandler>
+
+  <requestHandler name="/select" class="solr.SearchHandler">
+    <lst name="defaults">
+      <str name="echoParams">explicit</str>
+      <str name="indent">true</str>
+      <str name="df">text</str>
+    </lst>
+
+  </requestHandler>
+</config>

solr/core/src/test/org/apache/solr/schema/PreAnalyzedFieldManagedSchemaCloudTest.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.schema;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldResponse;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse.UpdateResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.DocCollection;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class PreAnalyzedFieldManagedSchemaCloudTest extends SolrCloudTestCase {
+
+  private static final String COLLECTION = "managed-preanalyzed";
+  private static final String CONFIG = "cloud-managed-preanalyzed";
+
+  @BeforeClass
+  public static void setupCluster() throws Exception {
+    configureCluster(2).addConfig(CONFIG, configset(CONFIG)).configure();
+    CollectionAdminRequest.createCollection(COLLECTION, CONFIG, 2, 1)
+        .setMaxShardsPerNode(1)
+        .process(cluster.getSolrClient());
+    cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
+        (n, c) -> DocCollection.isFullyActive(n, c, 2, 1));
+  }
+
+  @Test
+  public void testAdd2Fields() throws Exception {
+    addField(keyValueArrayToMap("name", "field1", "type", "string"));
+    addField(keyValueArrayToMap("name", "field2", "type", "string"));
+  }
+
+  private void addField(Map<String,Object> field) throws Exception {
+    CloudSolrClient client = cluster.getSolrClient();
+    UpdateResponse addFieldResponse = new SchemaRequest.AddField(field).process(client, COLLECTION);
+    assertNotNull(addFieldResponse);
+    assertEquals(0, addFieldResponse.getStatus());
+    assertNull(addFieldResponse.getResponse().get("errors"));
+    FieldResponse fieldResponse = new SchemaRequest.Field(field.get("name").toString()).process(client, COLLECTION);
+    assertNotNull(fieldResponse);
+    assertEquals(0, fieldResponse.getStatus());
+  }
+
+  private Map<String,Object> keyValueArrayToMap(String... alternatingKeysAndValues) {
+    Map<String,Object> map = new HashMap<>();
+    for (int i = 0 ; i < alternatingKeysAndValues.length ; i += 2)
+      map.put(alternatingKeysAndValues[i], alternatingKeysAndValues[i + 1]);
+    return map;
+  }
+}
+