Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-14151: fixed the classloading issue

This commit is contained in:
noblepaul 2020-09-17 19:10:04 +10:00
parent 4f344cb0d4
commit 515608a087
5 changed files with 111 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
View File

@ -16,8 +16,12 @@
*/
package org.apache.solr.core;
import com.google.common.annotations.VisibleForTesting;
import java.io.*;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.Constructor;
import java.net.MalformedURLException;
@ -30,14 +34,26 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.util.*;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
@ -83,27 +99,6 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
private CoreContainer coreContainer;
private PackageListeningClassLoader schemaLoader ;
private PackageListeningClassLoader createSchemaLoader() {
CoreContainer cc = getCoreContainer();
if (cc == null) {
//corecontainer not available . can't load from packages
return null;
}
return new PackageListeningClassLoader(cc, this, pkg -> {
if (getSolrConfig() == null) return null;
return getSolrConfig().maxPackageVersion(pkg);
}, () -> {
if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
if (c != null) {
c.fetchLatestSchema();
}
}
});
}
private final List<SolrCoreAware> waitingForCore = Collections.synchronizedList(new ArrayList<SolrCoreAware>());
private final List<SolrInfoBean> infoMBeans = Collections.synchronizedList(new ArrayList<SolrInfoBean>());
private final List<ResourceLoaderAware> waitingForResources = Collections.synchronizedList(new ArrayList<ResourceLoaderAware>());
@ -476,8 +471,9 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
}
Class<? extends T> clazz = null;
clazz = getPackageClass(cname, expectedType);
if(clazz != null) return clazz;
try {
// first try legacy analysis patterns, now replaced by Lucene's Analysis package:
final Matcher m = legacyAnalysisPattern.matcher(cname);
@ -540,6 +536,24 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
private <T> Class<? extends T> getPackageClass(String cname, Class<T> expectedType) {
PluginInfo.ClassName cName = PluginInfo.parseClassName(cname);
if (cName.pkg == null) return null;
ResourceLoaderAware aware = CURRENT_AWARE.get();
if (aware != null) {
//this is invoked from a component
//let's check if it's a schema component
@SuppressWarnings("rawtypes")
Class type = assertAwareCompatibility(ResourceLoaderAware.class, aware);
if (schemaResourceLoaderComponents.contains(type)) {
//this is a schema component
//lets use schema classloader
return getSchemaLoader().findClass(cname, expectedType);
}
}
return null;
}
static final String[] empty = new String[0];
@Override
@ -679,7 +693,13 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
for (ResourceLoaderAware aware : arr) {
CURRENT_AWARE.set(aware);
try{
aware.inform(loader);
} finally {
CURRENT_AWARE.remove();
}
}
}
}
@ -749,6 +769,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
ResourceLoaderAware.class, new Class<?>[]{
// DO NOT ADD THINGS TO THIS LIST -- ESPECIALLY THINGS THAT CAN BE CREATED DYNAMICALLY
// VIA RUNTIME APIS -- UNTILL CAREFULLY CONSIDERING THE ISSUES MENTIONED IN SOLR-8311
// evaluate if this must go into schemaResourceLoaderComponents
CharFilterFactory.class,
TokenFilterFactory.class,
TokenizerFactory.class,
@ -758,11 +779,21 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
);
}
/**If these components are trying to load classes, use schema classloader
*
*/
@SuppressWarnings("rawtypes")
private static final ImmutableSet<Class> schemaResourceLoaderComponents = ImmutableSet.of(
CharFilterFactory.class,
TokenFilterFactory.class,
TokenizerFactory.class,
FieldType.class);
/**
* Utility function to throw an exception if the class is invalid
*/
@SuppressWarnings({"rawtypes"})
public static void assertAwareCompatibility(Class aware, Object obj) {
public static Class assertAwareCompatibility(Class aware, Object obj) {
Class[] valid = awareCompatibility.get(aware);
if (valid == null) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@ -770,7 +801,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
for (Class v : valid) {
if (v.isInstance(obj)) {
return;
return v;
}
}
StringBuilder builder = new StringBuilder();
@ -800,6 +831,25 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
return Collections.unmodifiableList(infoMBeans);
}
private PackageListeningClassLoader createSchemaLoader() {
CoreContainer cc = getCoreContainer();
if (cc == null) {
//corecontainer not available . can't load from packages
return null;
}
return new PackageListeningClassLoader(cc, this, pkg -> {
if (getSolrConfig() == null) return null;
return getSolrConfig().maxPackageVersion(pkg);
}, () -> {
if(getCoreContainer() == null || config == null || coreName == null || coreId==null) return;
try (SolrCore c = getCoreContainer().getCore(coreName, coreId)) {
if (c != null) {
c.fetchLatestSchema();
}
}
});
}
public static void persistConfLocally(SolrResourceLoader loader, String resourceName, byte[] content) {
// Persist locally
@ -831,4 +881,7 @@ public class SolrResourceLoader implements ResourceLoader, Closeable, SolrClassL
}
}
//This is to verify if this requires to use the schema classloader for classes loaded from packages
public static final ThreadLocal<ResourceLoaderAware> CURRENT_AWARE = new ThreadLocal<>();
}

5
solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
View File

@ -66,10 +66,10 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.rest.schema.FieldTypeXmlAdapter;
import org.apache.solr.common.util.SolrNamedThreadFactory;
import org.apache.solr.util.FileUtils;
import org.apache.solr.util.RTimer;
import org.apache.zookeeper.CreateMode;
@ -1343,10 +1343,13 @@ public final class ManagedIndexSchema extends IndexSchema {
TokenFilterFactory[] filters = chain.getTokenFilterFactories();
for (TokenFilterFactory next : filters) {
if (next instanceof ResourceLoaderAware) {
SolrResourceLoader.CURRENT_AWARE.set((ResourceLoaderAware) next);
try {
((ResourceLoaderAware) next).inform(loader);
} catch (IOException e) {
throw new SolrException(ErrorCode.SERVER_ERROR, e);
} finally {
SolrResourceLoader.CURRENT_AWARE.remove();
}
}
}

10
solr/core/src/java/org/apache/solr/schema/SchemaManager.java
View File

@ -31,13 +31,13 @@ import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.rest.BaseSolrResource;
import org.apache.solr.common.util.CommandOperation;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@ -189,6 +189,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.addFieldTypes(singletonList(fieldType), false);
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -223,6 +224,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.addCopyFields(src, dests, maxChars);
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -240,6 +242,7 @@ public class SchemaManager {
= mgr.managedIndexSchema.addFields(singletonList(field), Collections.emptyMap(), false);
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -257,6 +260,7 @@ public class SchemaManager {
= mgr.managedIndexSchema.addDynamicFields(singletonList(field), Collections.emptyMap(), false);
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -275,6 +279,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.deleteFieldTypes(singleton(name));
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -331,6 +336,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.deleteDynamicFields(singleton(name));
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -346,6 +352,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.replaceFieldType(name, className, op.getDataMap());
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}
@ -366,6 +373,7 @@ public class SchemaManager {
mgr.managedIndexSchema = mgr.managedIndexSchema.replaceField(name, ft, op.getValuesExcluding(NAME, TYPE));
return true;
} catch (Exception e) {
log.error("err", e);
op.addError(getErrorStr(e));
return false;
}

BIN
solr/core/src/test-files/runtimecode/payload-component.jar.bin Normal file
View File

Binary file not shown.

21
solr/core/src/test/org/apache/solr/pkg/TestPackages.java
View File

@ -19,13 +19,20 @@ package org.apache.solr.pkg;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.Callable;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.analysis.util.ResourceLoaderAware;
import org.apache.solr.client.solrj.*;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
@ -69,9 +76,9 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH;
import static org.apache.solr.common.params.CommonParams.JAVABIN;
import static org.apache.solr.common.params.CommonParams.WT;
import static org.apache.solr.core.TestSolrConfigHandler.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
import static org.apache.solr.filestore.TestDistribPackageStore.checkAllNodesForFile;
import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
@ -655,10 +662,14 @@ public class TestPackages extends SolrCloudTestCase {
postFileAndWait(cluster, "runtimecode/schema-plugins.jar.bin", FILE1,
"iSRhrogDyt9P1htmSf/krh1kx9oty3TYyWm4GKHQGlb8a+X4tKCe9kKk+3tGs+bU9zq5JBZ5txNXsn96aZem5A==");
String FILE2 = "/schemapkg/payload-component.jar";
postFileAndWait(cluster, "runtimecode/payload-component.jar.bin", FILE2,
"gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q==");
Package.AddVersion add = new Package.AddVersion();
add.version = "1.0";
add.pkg = "schemapkg";
add.files = Arrays.asList(new String[]{FILE1});
add.files = Arrays.asList(FILE1,FILE2);
V2Request req = new V2Request.Builder("/cluster/package")
.forceV2(true)
.withMethod(SolrRequest.METHOD.POST)
@ -697,7 +708,7 @@ public class TestPackages extends SolrCloudTestCase {
String tokenizer =
" 'tokenizer' : { 'class':'schemapkg:my.pkg.MyWhitespaceTokenizerFactory' },\n";
String filters =
" 'filters' : [{ 'class':'solr.ASCIIFoldingFilterFactory' }]\n";
" 'filters' : [{ 'class':'solr.DelimitedPayloadTokenFilterFactory', 'encoder' : 'schemapkg:com.o19s.payloads.Base64Encoder'}]\n";
String suffix = " }\n" +
"}}";
cluster.getSolrClient().request(new SolrRequest(SolrRequest.METHOD.POST, "/schema") {