Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-6801 more tests , SOLR-6770 refactored code around 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1648689 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Noble Paul 2014-12-31 12:25:08 +00:00
parent e17de210f4
commit 5444b1f45a
9 changed files with 277 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
solr/core/src/java/org/apache/solr/core/CoreContainer.java
View File

@ -63,7 +63,7 @@ public class CoreContainer {
protected static final Logger log = LoggerFactory.getLogger(CoreContainer.class);
private final SolrCores solrCores = new SolrCores(this);
final SolrCores solrCores = new SolrCores(this);
public static class CoreLoadFailure {

14
solr/core/src/java/org/apache/solr/core/RequestParams.java
View File

@ -95,13 +95,13 @@ public class RequestParams implements MapSerializable{
p.remove(name);
} else {
Map old = (Map) p.get(name);
int version = 0;
long version = 0;
Map meta = null;
if(old != null){
meta = (Map) old.get("");
if(meta!=null) {
Integer oldVersion = (Integer) old.get("v");
if(oldVersion != null) version = oldVersion.intValue()+1;
Long oldVersion = (Long) old.get("v");
if(oldVersion != null) version = oldVersion.longValue()+1;
}
meta = new LinkedHashMap<>(meta);
} else {
@ -197,9 +197,13 @@ public class RequestParams implements MapSerializable{
super(map);
this.meta = meta;
}
public Map getRawMap(){
return meta;
}
public Integer getVersion() {
return meta == null? 0 : (Integer)meta.get("v");
public Long getVersion() {
return meta == null? 0l : (Long)meta.get("v");
}
}
}

1
solr/core/src/java/org/apache/solr/core/SolrConfig.java
View File

@ -179,6 +179,7 @@ public class SolrConfig extends Config implements MapSerializable{
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, is, "/config/");
getOverlay();//just in case it is not initialized
getRequestParams();
initLibs();
luceneMatchVersion = getLuceneVersion("luceneMatchVersion");
String indexConfigPrefix;

122
solr/core/src/java/org/apache/solr/core/SolrCore.java
View File

@ -29,9 +29,11 @@ import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.CommonParams.EchoParamStyle;
import org.apache.solr.common.params.SolrParams;
@ -74,6 +76,7 @@ import org.apache.solr.rest.RestManager;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.apache.solr.schema.ManagedIndexSchema;
import org.apache.solr.schema.SimilarityFactory;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.SolrFieldCacheMBean;
@ -100,6 +103,8 @@ import org.apache.solr.util.RefCounted;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
import org.apache.solr.util.plugin.PluginInfoInitialized;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
@ -135,6 +140,7 @@ import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@ -182,6 +188,8 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
private DirectoryFactory directoryFactory;
private IndexReaderFactory indexReaderFactory;
private final Codec codec;
private final List<Runnable> confListeners = new CopyOnWriteArrayList<>();
private final ReentrantLock ruleExpiryLock;
@ -937,6 +945,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
// openHandles.put(this, new RuntimeException("unclosed core - name:" + getName() + " refs: " + refCount.get()));
ruleExpiryLock = new ReentrantLock();
registerConfListener();
}
private Codec initCodec(SolrConfig solrConfig, final IndexSchema schema) {
@ -2128,7 +2137,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
@Override
public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response) throws IOException {
RawWriter rawWriter = (RawWriter) response.getValues().get(ReplicationHandler.FILE_STREAM);
rawWriter.write(out);
if(rawWriter!=null) rawWriter.write(out);
}
@Override
@ -2618,6 +2627,117 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
return getWrappedWriter().getContentType(request, response);
}
}
/**Register to notify for any file change in the conf directory.
* If the file change results in a core reload , then the listener
* is not fired
*/
public void addConfListener(Runnable runnable) {
confListeners.add(runnable);
}
/**Remove a listener
* */
public boolean removeConfListener(Runnable runnable) {
return confListeners.remove(runnable);
}
/**This registers one listener for the entire conf directory. In zookeeper
* there is no event fired when children are modified. So , we expect everyone
* to 'touch' the /conf directory by setting some data so that events are triggered.
*/
private void registerConfListener() {
if( ! (resourceLoader instanceof ZkSolrResourceLoader)) return;
final ZkSolrResourceLoader zkSolrResourceLoader = (ZkSolrResourceLoader) resourceLoader;
if(zkSolrResourceLoader != null)
zkSolrResourceLoader.getZkController().registerConfListenerForCore(
zkSolrResourceLoader.getConfigSetZkPath(),
this,
getListener(this, zkSolrResourceLoader));
}
private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
final String coreName = core.getName();
final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
final String solrConfigPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
String schemaRes = null;
if (core.getLatestSchema().isMutable() && core.getLatestSchema() instanceof ManagedIndexSchema) {
ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema();
schemaRes = mis.getResourceName();
}
final String managedSchmaResourcePath = schemaRes == null ? null : zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes;
return new Runnable() {
@Override
public void run() {
log.info("config update listener called for core {}", coreName);
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion, overlayVersion, managedSchemaVersion = 0;
SolrConfig cfg = null;
try (SolrCore core = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (core == null || core.isClosed()) return;
cfg = core.getSolrConfig();
solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion();
overlayVersion = core.getSolrConfig().getZnodeVersion();
if (managedSchmaResourcePath != null) {
managedSchemaVersion = ((ManagedIndexSchema) core.getLatestSchema()).getSchemaZkVersion();
}
}
if (cfg != null) {
cfg.refreshRequestParams() ;
}
if (checkStale(zkClient, overlayPath, solrConfigversion) ||
checkStale(zkClient, solrConfigPath, overlayVersion) ||
checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) {
log.info("core reload {}", coreName);
cc.reload(coreName);
return;
}
//some files in conf directoy has changed other than schema.xml,
// solrconfig.xml. so fire event listeners
try (SolrCore core = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (core == null || core.isClosed()) return;
for (Runnable listener : core.confListeners) {
try {
listener.run();
} catch (Exception e) {
log.error("Error in listener ", e);
}
}
}
}
};
}
private static boolean checkStale(SolrZkClient zkClient, String zkPath, int currentVersion) {
if(zkPath == null) return false;
try {
Stat stat = zkClient.exists(zkPath, null, true);
if(stat == null){
if(currentVersion > -1) return true;
return false;
}
if (stat.getVersion() > currentVersion) {
log.info(zkPath+" is stale will need an update from {} to {}", currentVersion,stat.getVersion());
return true;
}
return false;
} catch (KeeperException.NoNodeException nne){
//no problem
} catch (KeeperException e) {
log.error("error refreshing solrconfig ", e);
} catch (InterruptedException e) {
Thread.currentThread().isInterrupted();
}
return false;
}
}

2
solr/core/src/java/org/apache/solr/core/SolrCores.java
View File

@ -245,7 +245,7 @@ class SolrCores {
}
/* If you don't increment the reference count, someone could close the core before you use it. */
protected SolrCore getCoreFromAnyList(String name, boolean incRefCount) {
SolrCore getCoreFromAnyList(String name, boolean incRefCount) {
synchronized (modifyLock) {
SolrCore core = cores.get(name);

81
solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
View File

@ -64,7 +64,7 @@ import static org.apache.solr.common.params.CoreAdminParams.NAME;
import static org.apache.solr.core.ConfigOverlay.NOT_EDITABLE;
import static org.apache.solr.schema.FieldType.CLASS_NAME;
public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAware{
public class SolrConfigHandler extends RequestHandlerBase {
public static final Logger log = LoggerFactory.getLogger(SolrConfigHandler.class);
public static final boolean configEditing_disabled = Boolean.getBoolean("disable.configEdit");
@ -83,85 +83,6 @@ public class SolrConfigHandler extends RequestHandlerBase implements SolrCoreAwa
}
@Override
public void inform(final SolrCore core) {
if( ! (core.getResourceLoader() instanceof ZkSolrResourceLoader)) return;
final ZkSolrResourceLoader zkSolrResourceLoader = (ZkSolrResourceLoader) core.getResourceLoader();
if(zkSolrResourceLoader != null)
zkSolrResourceLoader.getZkController().registerConfListenerForCore(
zkSolrResourceLoader.getConfigSetZkPath(),
core,
getListener(core, zkSolrResourceLoader));
}
private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
final String coreName = core.getName();
final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
final String solrConfigPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName();
String schemaRes = null;
if(core.getLatestSchema().isMutable() && core.getLatestSchema() instanceof ManagedIndexSchema){
ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema();
schemaRes = mis.getResourceName();
}
final String managedSchmaResourcePath = schemaRes ==null ? null: zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes;
return new Runnable() {
@Override
public void run() {
log.info("config update listener called for core {}", coreName);
SolrZkClient zkClient = cc.getZkController().getZkClient();
int solrConfigversion,overlayVersion, managedSchemaVersion=0;
SolrConfig cfg = null;
try (SolrCore core = cc.getCore(coreName)) {
if (core.isClosed()) return;
cfg = core.getSolrConfig();
solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion();
overlayVersion = core.getSolrConfig().getZnodeVersion();
if(managedSchmaResourcePath != null){
managedSchemaVersion = ((ManagedIndexSchema)core.getLatestSchema()).getSchemaZkVersion();
}
}
if(cfg != null){
cfg.refreshRequestParams();
}
if (checkStale(zkClient, overlayPath, solrConfigversion) ||
checkStale(zkClient, solrConfigPath, overlayVersion) ||
checkStale(zkClient, managedSchmaResourcePath,managedSchemaVersion)) {
log.info("core reload {}",coreName);
cc.reload(coreName);
}
}
};
}
private static boolean checkStale(SolrZkClient zkClient, String zkPath, int currentVersion) {
if(zkPath == null) return false;
try {
Stat stat = zkClient.exists(zkPath, null, true);
if(stat == null){
if(currentVersion > -1) return true;
return false;
}
if (stat.getVersion() > currentVersion) {
log.info(zkPath+" is stale will need an update from {} to {}", currentVersion,stat.getVersion());
return true;
}
return false;
} catch (KeeperException.NoNodeException nne){
//no problem
} catch (KeeperException e) {
log.error("error refreshing solrconfig ", e);
} catch (InterruptedException e) {
Thread.currentThread().isInterrupted();
}
return false;
}
private static class Command{
private final SolrQueryRequest req;
private final SolrQueryResponse resp;

2
solr/core/src/test/org/apache/solr/core/BlobStoreTestRequestHandler.java
View File

@ -28,6 +28,6 @@ public class BlobStoreTestRequestHandler extends DumpRequestHandler{
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
super.handleRequestBody(req, rsp);
rsp.add("class", BlobStoreTestRequestHandler.class.getName());
rsp.add("class", this.getClass().getName());
}
}

68
solr/core/src/test/org/apache/solr/core/BlobStoreTestRequestHandlerV2.java Normal file
View File

@ -0,0 +1,68 @@
package org.apache.solr.core;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.plugin.SolrCoreAware;
public class BlobStoreTestRequestHandlerV2 extends BlobStoreTestRequestHandler implements Runnable, SolrCoreAware{
private SolrCore core;
private long version = 1;
private String watchedVal = null;
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
super.handleRequestBody(req, rsp);
rsp.add("class", this.getClass().getName());
rsp.add("x",watchedVal);
/* try {
Class.forName("org.apache.solr.core.BlobStoreTestRequestHandler");
} catch (ClassNotFoundException e) {
rsp.add("e", ClassNotFoundException.class.getSimpleName());
}*/
}
@Override
public void run() {
RequestParams p = core.getSolrConfig().getRequestParams();
RequestParams.VersionedParams v = p.getParams("watched");
if(v== null){
watchedVal = null;
version=-1;
return;
}
if(v.getVersion() != version){
watchedVal = v.getMap().get("x");
}
}
@Override
public void inform(SolrCore core) {
this.core = core;
core.addConfListener(this);
run();
}
}

74
solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
View File

@ -107,6 +107,80 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
}
assertTrue(new String( ZkStateReader.toJSON(map) , StandardCharsets.UTF_8), success );
jar = generateZip( TestDynamicLoading.class,BlobStoreTestRequestHandlerV2.class);
TestBlobHandler.postAndCheck(cloudClient, baseURL, jar,2);
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val',\n" +
" 'y': 'Y val'}\n" +
" }\n" +
" }";
TestSolrConfigHandler.runConfigCommand(client,"/config/params?wt=json",payload);
TestSolrConfigHandler.testForResponseElement(
client,
null,
"/config/params?wt=json",
cloudClient,
Arrays.asList("response", "params", "watched", "x"),
"X val",
10);
payload = "{\n" +
"'update-requesthandler' : { 'name' : '/test1', 'class': 'org.apache.solr.core.BlobStoreTestRequestHandlerV2' , 'lib':'test','version':'2'}\n" +
"}";
client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
TestSolrConfigHandler.runConfigCommand(client,"/config?wt=json",payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
"/config/overlay?wt=json",
null,
Arrays.asList("overlay", "requestHandler", "/test1", "version"),
"2",10);
success= false;
for(int i=0;i<50;i++) {
map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
if(BlobStoreTestRequestHandlerV2.class.getName().equals(map.get("class"))) {
success = true;
break;
}
Thread.sleep(100);
}
assertTrue("New version of class is not loaded " + new String(ZkStateReader.toJSON(map), StandardCharsets.UTF_8), success);
for(int i=0;i<50;i++) {
map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
if("X val".equals(map.get("x"))){
success = true;
break;
}
Thread.sleep(100);
}
payload = " {\n" +
" 'set' : {'watched': {" +
" 'x':'X val changed',\n" +
" 'y': 'Y val'}\n" +
" }\n" +
" }";
TestSolrConfigHandler.runConfigCommand(client,"/config/params?wt=json",payload);
for(int i=0;i<50;i++) {
map = TestSolrConfigHandler.getRespMap("/test1?wt=json", client);
if("X val changed".equals(map.get("x"))){
success = true;
break;
}
Thread.sleep(100);
}
assertTrue("listener did not get triggered" + new String(ZkStateReader.toJSON(map), StandardCharsets.UTF_8), success);
}