diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index 7a10f8964d0..532f1dd3896 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -16,8 +16,12 @@ */ package org.apache.solr.core; -import com.google.common.annotations.VisibleForTesting; -import java.io.*; +import java.io.Closeable; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; import java.net.MalformedURLException; @@ -30,14 +34,26 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; import org.apache.lucene.analysis.WordlistLoader; -import org.apache.lucene.analysis.util.*; +import org.apache.lucene.analysis.util.CharFilterFactory; +import org.apache.lucene.analysis.util.ResourceLoader; +import org.apache.lucene.analysis.util.ResourceLoaderAware; +import org.apache.lucene.analysis.util.TokenFilterFactory; +import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; @@ -83,27 +99,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL private CoreContainer coreContainer; private PackageListeningClassLoader schemaLoader ; - private PackageListeningClassLoader createSchemaLoader() { - CoreContainer cc = getCoreContainer(); - if (cc == null) { - //corecontainer not available . can't load from packages - return null; - } - return new PackageListeningClassLoader(cc, this, pkg -> { - if (getSolrConfig() == null) return null; - return getSolrConfig().maxPackageVersion(pkg); - }, () -> { - if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return; - try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) { - if (c != null) { - c.fetchLatestSchema(); - } - } - }); - } - - - private final List waitingForCore = Collections.synchronizedList(new ArrayList()); private final List infoMBeans = Collections.synchronizedList(new ArrayList()); private final List waitingForResources = Collections.synchronizedList(new ArrayList()); @@ -476,8 +471,9 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL } } } - Class clazz = null; + clazz = getPackageClass(cname, expectedType); + if(clazz != null) return clazz; try { // first try legacy analysis patterns, now replaced by Lucene's Analysis package: final Matcher m = legacyAnalysisPattern.matcher(cname); @@ -540,6 +536,24 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL } } + private Class getPackageClass(String cname, Class expectedType) { + PluginInfo.ClassName cName = PluginInfo.parseClassName(cname); + if (cName.pkg == null) return null; + ResourceLoaderAware aware = CURRENT_AWARE.get(); + if (aware != null) { + //this is invoked from a component + //let's check if it's a schema component + @SuppressWarnings("rawtypes") + Class type = assertAwareCompatibility(ResourceLoaderAware.class, aware); + if (schemaResourceLoaderComponents.contains(type)) { + //this is a schema component + //lets use schema classloader + return getSchemaLoader().findClass(cname, expectedType); + } + } + return null; + } + static final String[] empty = new String[0]; @Override @@ -679,7 +693,13 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL } for (ResourceLoaderAware aware : arr) { - aware.inform(loader); + CURRENT_AWARE.set(aware); + try{ + aware.inform(loader); + } finally { + CURRENT_AWARE.remove(); + } + } } } @@ -749,6 +769,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL ResourceLoaderAware.class, new Class[]{ // DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY // VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311 + // evaluate if this must go into schemaResourceLoaderComponents CharFilterFactory.class, TokenFilterFactory.class, TokenizerFactory.class, @@ -758,11 +779,21 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL ); } + /**If these components are trying to load classes, use schema classloader + * + */ + @SuppressWarnings("rawtypes") + private static final ImmutableSet schemaResourceLoaderComponents = ImmutableSet.of( + CharFilterFactory.class, + TokenFilterFactory.class, + TokenizerFactory.class, + FieldType.class); + /** * Utility function to throw an exception if the class is invalid */ @SuppressWarnings({"rawtypes"}) - public static void assertAwareCompatibility(Class aware, Object obj) { + public static Class assertAwareCompatibility(Class aware, Object obj) { Class[] valid = awareCompatibility.get(aware); if (valid == null) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, @@ -770,7 +801,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL } for (Class v : valid) { if (v.isInstance(obj)) { - return; + return v; } } StringBuilder builder = new StringBuilder(); @@ -800,6 +831,25 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL return Collections.unmodifiableList(infoMBeans); } + private PackageListeningClassLoader createSchemaLoader() { + CoreContainer cc = getCoreContainer(); + if (cc == null) { + //corecontainer not available . can't load from packages + return null; + } + return new PackageListeningClassLoader(cc, this, pkg -> { + if (getSolrConfig() == null) return null; + return getSolrConfig().maxPackageVersion(pkg); + }, () -> { + if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return; + try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) { + if (c != null) { + c.fetchLatestSchema(); + } + } + }); + } + public static void persistConfLocally(SolrResourceLoader loader, String resourceName, byte[] content) { // Persist locally @@ -831,4 +881,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL } } + //This is to verify if this requires to use the schema classloader for classes loaded from packages + public static final ThreadLocal CURRENT_AWARE = new ThreadLocal<>(); + } diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java index 9fd52431a44..f8c0c4fd1b4 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java @@ -66,10 +66,10 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.rest.schema.FieldTypeXmlAdapter; -import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.util.FileUtils; import org.apache.solr.util.RTimer; import org.apache.zookeeper.CreateMode; @@ -1343,10 +1343,13 @@ public final class ManagedIndexSchema extends IndexSchema { TokenFilterFactory[] filters = chain.getTokenFilterFactories(); for (TokenFilterFactory next : filters) { if (next instanceof ResourceLoaderAware) { + SolrResourceLoader.CURRENT_AWARE.set((ResourceLoaderAware) next); try { ((ResourceLoaderAware) next).inform(loader); } catch (IOException e) { throw new SolrException(ErrorCode.SERVER_ERROR, e); + } finally { + SolrResourceLoader.CURRENT_AWARE.remove(); } } } diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java index 3731a176275..c04c9c36c77 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaManager.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaManager.java @@ -31,13 +31,13 @@ import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkSolrResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.CommandOperation; import org.apache.solr.common.util.TimeSource; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.rest.BaseSolrResource; -import org.apache.solr.common.util.CommandOperation; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -189,6 +189,7 @@ public class SchemaManager { mgr.managedIndexSchema = mgr.managedIndexSchema.addFieldTypes(singletonList(fieldType), false); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -223,6 +224,7 @@ public class SchemaManager { mgr.managedIndexSchema = mgr.managedIndexSchema.addCopyFields(src, dests, maxChars); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -240,6 +242,7 @@ public class SchemaManager { = mgr.managedIndexSchema.addFields(singletonList(field), Collections.emptyMap(), false); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -257,6 +260,7 @@ public class SchemaManager { = mgr.managedIndexSchema.addDynamicFields(singletonList(field), Collections.emptyMap(), false); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -275,6 +279,7 @@ public class SchemaManager { mgr.managedIndexSchema = mgr.managedIndexSchema.deleteFieldTypes(singleton(name)); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -331,6 +336,7 @@ public class SchemaManager { mgr.managedIndexSchema = mgr.managedIndexSchema.deleteDynamicFields(singleton(name)); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -346,6 +352,7 @@ public class SchemaManager { mgr.managedIndexSchema = mgr.managedIndexSchema.replaceFieldType(name, className, op.getDataMap()); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } @@ -366,6 +373,7 @@ public class SchemaManager { mgr.managedIndexSchema = mgr.managedIndexSchema.replaceField(name, ft, op.getValuesExcluding(NAME, TYPE)); return true; } catch (Exception e) { + log.error("err", e); op.addError(getErrorStr(e)); return false; } diff --git a/solr/core/src/test-files/runtimecode/payload-component.jar.bin b/solr/core/src/test-files/runtimecode/payload-component.jar.bin new file mode 100644 index 00000000000..47a4305f543 Binary files /dev/null and b/solr/core/src/test-files/runtimecode/payload-component.jar.bin differ diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 8d8583c11a4..9473d4140e4 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -19,13 +19,20 @@ package org.apache.solr.pkg; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.Callable; import org.apache.commons.codec.digest.DigestUtils; import org.apache.lucene.analysis.util.ResourceLoader; import org.apache.lucene.analysis.util.ResourceLoaderAware; -import org.apache.solr.client.solrj.*; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; import org.apache.solr.client.solrj.impl.HttpSolrClient; @@ -69,9 +76,9 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; import static org.apache.solr.core.TestSolrConfigHandler.getFileContent; +import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile; import static org.apache.solr.filestore.TestDistribPackageStore.readFile; import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey; -import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile; import static org.hamcrest.CoreMatchers.containsString; @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") @@ -655,10 +662,14 @@ public class TestPackages extends SolrCloudTestCase { postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1, "iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A=="); + String FILE2 = "/schemapkg/payload-component.jar"; + postFileAndWait(cluster, "runtimecode/payload-component.jar.bin", FILE2, + "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); + Package.AddVersion add = new Package.AddVersion(); add.version = "1.0"; add.pkg = "schemapkg"; - add.files = Arrays.asList(new String[]{FILE1}); + add.files = Arrays.asList(FILE1,FILE2); V2Request req = new V2Request.Builder("/cluster/package") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) @@ -697,7 +708,7 @@ public class TestPackages extends SolrCloudTestCase { String tokenizer = " 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n"; String filters = - " 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n"; + " 'filters' : [{ 'class':'solr.DelimitedPayloadTokenFilterFactory', 'encoder' : 'schemapkg:com.o19s.payloads.Base64Encoder'}]\n"; String suffix = " }\n" + "}}"; cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {