From 211d106cc23adb1206479d2f7b455d4ded1da4fc Mon Sep 17 00:00:00 2001 From: Anshum Gupta Date: Tue, 1 Aug 2017 12:17:26 -0700 Subject: [PATCH] SOLR-11126: Node-level health check handler, with SolrJ support --- solr/CHANGES.txt | 2 + .../org/apache/solr/core/CoreContainer.java | 8 +- .../java/org/apache/solr/core/NodeConfig.java | 18 ++- .../org/apache/solr/core/SolrXmlConfig.java | 3 + .../handler/admin/HealthCheckHandler.java | 117 ++++++++++++++++++ .../solr/cloud/HealthCheckHandlerTest.java | 89 +++++++++++++ .../solrj/request/HealthCheckRequest.java | 61 +++++++++ .../solrj/response/HealthCheckResponse.java | 39 ++++++ .../solr/common/params/CommonParams.java | 7 ++ 9 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java create mode 100644 solr/core/src/test/org/apache/solr/cloud/HealthCheckHandlerTest.java create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java create mode 100644 solr/solrj/src/java/org/apache/solr/client/solrj/response/HealthCheckResponse.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index bd56c6a2c18..fd8a1e8e238 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -63,6 +63,8 @@ New Features * SOLR-10858: Make UUIDUpdateProcessorFactory as Runtime URP (Amit Sarkar, noble) +* SOLR-11126: Node level health check handler (Anshum Gupta) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 53a3bb31653..0b789f8fba4 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -22,6 +22,7 @@ import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH; import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.CONFIGSETS_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.CORES_HANDLER_PATH; +import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.METRICS_PATH; import static org.apache.solr.common.params.CommonParams.ZK_PATH; @@ -80,6 +81,7 @@ import org.apache.solr.handler.SnapShooter; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.handler.admin.HealthCheckHandler; import org.apache.solr.handler.admin.InfoHandler; import org.apache.solr.handler.admin.MetricsCollectorHandler; import org.apache.solr.handler.admin.MetricsHandler; @@ -135,6 +137,7 @@ public class CoreContainer { protected CoreAdminHandler coreAdminHandler = null; protected CollectionsHandler collectionsHandler = null; + protected HealthCheckHandler healthCheckHandler = null; private InfoHandler infoHandler; protected ConfigSetsHandler configSetsHandler = null; @@ -523,6 +526,7 @@ public class CoreContainer { createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class); collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class); + healthCheckHandler = createHandler(HEALTH_CHECK_HANDLER_PATH, cfg.getHealthCheckHandlerClass(), HealthCheckHandler.class); infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class); coreAdminHandler = createHandler(CORES_HANDLER_PATH, cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class); configSetsHandler = createHandler(CONFIGSETS_HANDLER_PATH, cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class); @@ -640,7 +644,7 @@ public class CoreContainer { } finally { if (asyncSolrCoreLoad && futures != null) { - coreContainerWorkExecutor.submit((Runnable) () -> { + coreContainerWorkExecutor.submit(() -> { try { for (Future future : futures) { try { @@ -1470,6 +1474,8 @@ public class CoreContainer { return collectionsHandler; } + public HealthCheckHandler getHealthCheckHandler() { return healthCheckHandler; } + public InfoHandler getInfoHandler() { return infoHandler; } diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java index 5b4debe5e0f..ce7fe2792cd 100644 --- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java +++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java @@ -47,6 +47,8 @@ public class NodeConfig { private final String collectionsAdminHandlerClass; + private final String healthCheckHandlerClass; + private final String infoHandlerClass; private final String configSetsHandlerClass; @@ -74,7 +76,7 @@ public class NodeConfig { private NodeConfig(String nodeName, Path coreRootDirectory, Path solrDataHome, Path configSetBaseDirectory, String sharedLibDirectory, PluginInfo shardHandlerFactoryConfig, UpdateShardHandlerConfig updateShardHandlerConfig, String coreAdminHandlerClass, String collectionsAdminHandlerClass, - String infoHandlerClass, String configSetsHandlerClass, + String healthCheckHandlerClass, String infoHandlerClass, String configSetsHandlerClass, LogWatcherConfig logWatcherConfig, CloudConfig cloudConfig, Integer coreLoadThreads, int transientCacheSize, boolean useSchemaCache, String managementPath, SolrResourceLoader loader, Properties solrProperties, PluginInfo[] backupRepositoryPlugins, @@ -88,6 +90,7 @@ public class NodeConfig { this.updateShardHandlerConfig = updateShardHandlerConfig; this.coreAdminHandlerClass = coreAdminHandlerClass; this.collectionsAdminHandlerClass = collectionsAdminHandlerClass; + this.healthCheckHandlerClass = healthCheckHandlerClass; this.infoHandlerClass = infoHandlerClass; this.configSetsHandlerClass = configSetsHandlerClass; this.logWatcherConfig = logWatcherConfig; @@ -146,6 +149,10 @@ public class NodeConfig { return collectionsAdminHandlerClass; } + public String getHealthCheckHandlerClass() { + return healthCheckHandlerClass; + } + public String getInfoHandlerClass() { return infoHandlerClass; } @@ -209,6 +216,7 @@ public class NodeConfig { private UpdateShardHandlerConfig updateShardHandlerConfig = UpdateShardHandlerConfig.DEFAULT; private String coreAdminHandlerClass = DEFAULT_ADMINHANDLERCLASS; private String collectionsAdminHandlerClass = DEFAULT_COLLECTIONSHANDLERCLASS; + private String healthCheckHandlerClass = DEFAULT_HEALTHCHECKHANDLERCLASS; private String infoHandlerClass = DEFAULT_INFOHANDLERCLASS; private String configSetsHandlerClass = DEFAULT_CONFIGSETSHANDLERCLASS; private LogWatcherConfig logWatcherConfig = new LogWatcherConfig(true, null, null, 50); @@ -236,6 +244,7 @@ public class NodeConfig { private static final String DEFAULT_ADMINHANDLERCLASS = "org.apache.solr.handler.admin.CoreAdminHandler"; private static final String DEFAULT_INFOHANDLERCLASS = "org.apache.solr.handler.admin.InfoHandler"; private static final String DEFAULT_COLLECTIONSHANDLERCLASS = "org.apache.solr.handler.admin.CollectionsHandler"; + private static final String DEFAULT_HEALTHCHECKHANDLERCLASS = "org.apache.solr.handler.admin.HealthCheckHandler"; private static final String DEFAULT_CONFIGSETSHANDLERCLASS = "org.apache.solr.handler.admin.ConfigSetsHandler"; public static final Set DEFAULT_HIDDEN_SYS_PROPS = new HashSet<>(Arrays.asList( @@ -302,6 +311,11 @@ public class NodeConfig { return this; } + public NodeConfigBuilder setHealthCheckHandlerClass(String healthCheckHandlerClass) { + this.healthCheckHandlerClass = healthCheckHandlerClass; + return this; + } + public NodeConfigBuilder setInfoHandlerClass(String infoHandlerClass) { this.infoHandlerClass = infoHandlerClass; return this; @@ -366,7 +380,7 @@ public class NodeConfig { public NodeConfig build() { return new NodeConfig(nodeName, coreRootDirectory, solrDataHome, configSetBaseDirectory, sharedLibDirectory, shardHandlerFactoryConfig, - updateShardHandlerConfig, coreAdminHandlerClass, collectionsAdminHandlerClass, infoHandlerClass, configSetsHandlerClass, + updateShardHandlerConfig, coreAdminHandlerClass, collectionsAdminHandlerClass, healthCheckHandlerClass, infoHandlerClass, configSetsHandlerClass, logWatcherConfig, cloudConfig, coreLoadThreads, transientCacheSize, useSchemaCache, managementPath, loader, solrProperties, backupRepositoryPlugins, metricsConfig, transientCacheConfig); } diff --git a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java index 8cdf94773e9..99f0b517637 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java @@ -241,6 +241,9 @@ public class SolrXmlConfig { case "collectionsHandler": builder.setCollectionsAdminHandlerClass(value); break; + case "healthCheckHandler": + builder.setHealthCheckHandlerClass(value); + break; case "infoHandler": builder.setInfoHandlerClass(value); break; diff --git a/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java new file mode 100644 index 00000000000..03c7bd47d1c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/HealthCheckHandler.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin; + +import java.lang.invoke.MethodHandles; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.params.CommonParams.FAILURE; +import static org.apache.solr.common.params.CommonParams.OK; +import static org.apache.solr.common.params.CommonParams.STATUS; + +/* + * Health Check Handler for reporting the health of a specific node. + * + * This checks if the node is: + * 1. Connected to zookeeper + * 2. listed in 'live_nodes'. + */ +public class HealthCheckHandler extends RequestHandlerBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + CoreContainer coreContainer; + + public HealthCheckHandler(final CoreContainer coreContainer) { + super(); + this.coreContainer = coreContainer; + } + + @Override + final public void init(NamedList args) { + + } + + public CoreContainer getCoreContainer() { + return this.coreContainer; + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + + log.info("Invoked HealthCheckHandler on [{}]", coreContainer.getZkController().getNodeName()); + CoreContainer cores = getCoreContainer(); + + if(cores == null) { + rsp.setException(new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Core container not initialized")); + return; + } + if(!cores.isZooKeeperAware()) { + rsp.setException(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Health check is only available when running in SolrCloud mode")); + return; + } + ZkStateReader zkStateReader = cores.getZkController().getZkStateReader(); + ClusterState clusterState = zkStateReader.getClusterState(); + // Check for isConnected and isClosed + if(zkStateReader.getZkClient().isClosed() || !zkStateReader.getZkClient().isConnected()) { + rsp.add(STATUS, FAILURE); + rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not connected to zk")); + return; + } + + try { + zkStateReader.updateLiveNodes(); + + // Set status to true if this node is in live_nodes + if (clusterState.getLiveNodes().contains(cores.getZkController().getNodeName())) { + rsp.add(STATUS, OK); + } else { + rsp.add(STATUS, FAILURE); + rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not in live nodes as per zk")); + } + } catch (KeeperException e) { + rsp.add(STATUS, FAILURE); + rsp.setException(new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Host Unavailable: Not connected to zk")); + } + + rsp.setHttpCaching(false); + + return; + } + + @Override + public String getDescription() { + return "Health check handler for SolrCloud node"; + } + + @Override + public Category getCategory() { + return Category.ADMIN; + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/HealthCheckHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/HealthCheckHandlerTest.java new file mode 100644 index 00000000000..3baa1811ebf --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/HealthCheckHandlerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.io.IOException; +import java.util.Set; + +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrResponse; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.HealthCheckRequest; +import org.apache.solr.client.solrj.response.HealthCheckResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.zookeeper.KeeperException; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH; + +public class HealthCheckHandlerTest extends SolrCloudTestCase { + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Test + public void testHealthCheckHandler() throws IOException, SolrServerException, InterruptedException, KeeperException { + SolrRequest req = new GenericSolrRequest(SolrRequest.METHOD.GET, HEALTH_CHECK_HANDLER_PATH, new ModifiableSolrParams()); + try (HttpSolrClient httpSolrClient = getHttpSolrClient(cluster.getJettySolrRunner(0).getBaseUrl().toString())) { + SolrResponse response = req.process(cluster.getSolrClient()); + assertEquals(CommonParams.OK, response.getResponse().get(CommonParams.STATUS)); + + JettySolrRunner jetty = cluster.getJettySolrRunner(0); + cluster.expireZkSession(jetty); + Set live_nodes = cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes(); + + int counter = 0; + while (live_nodes.size() == 1 && counter++ < 100) { + Thread.sleep(100); + live_nodes = cluster.getSolrClient().getZkStateReader().getClusterState().getLiveNodes(); + } + + try { + req.process(httpSolrClient); + } catch (HttpSolrClient.RemoteSolrException e) { + assertTrue(e.getMessage(), e.getMessage().contains("Host Unavailable")); + assertEquals(SolrException.ErrorCode.SERVICE_UNAVAILABLE.code, e.code()); + } + } + } + + @Test + public void testHealthCheckHandlerSolrJ() throws IOException, SolrServerException { + HealthCheckRequest req = new HealthCheckRequest(); + try (HttpSolrClient httpSolrClient = getHttpSolrClient(cluster.getJettySolrRunner(0).getBaseUrl().toString())) { + HealthCheckResponse rsp = req.process(httpSolrClient); + assertEquals(CommonParams.OK, rsp.getNodeStatus()); + } + } + + @Test (expected = AssertionError.class) + public void testHealthCheckHandlerWithCloudClient() throws IOException, SolrServerException { + HealthCheckRequest req = new HealthCheckRequest(); + req.process(cluster.getSolrClient()); + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java new file mode 100644 index 00000000000..7073167bd54 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/HealthCheckRequest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.request; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.HealthCheckResponse; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.ContentStream; + +import static org.apache.solr.common.params.CommonParams.HEALTH_CHECK_HANDLER_PATH; + +public class HealthCheckRequest extends SolrRequest { + + public HealthCheckRequest() { + this(METHOD.GET, HEALTH_CHECK_HANDLER_PATH); + } + + private HealthCheckRequest(METHOD m, String path) { + super(m, path); + } + + @Override + public SolrParams getParams() { + return null; + } + + @Override + public Collection getContentStreams() throws IOException { + return null; + } + + @Override + protected HealthCheckResponse createResponse(SolrClient client) { + // TODO: Accept requests w/ CloudSolrClient while ensuring that the request doesn't get routed to + // an unintended recepient. + assert client instanceof HttpSolrClient; + return new HealthCheckResponse(); + } + + +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/response/HealthCheckResponse.java b/solr/solrj/src/java/org/apache/solr/client/solrj/response/HealthCheckResponse.java new file mode 100644 index 00000000000..b6fc36bb6b7 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/response/HealthCheckResponse.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.response; + +import org.apache.solr.common.util.NamedList; + +public class HealthCheckResponse extends SolrResponseBase { + + public HealthCheckResponse() { + } + + public NamedList getErrorMessages() { + return (NamedList) getResponse().get( "errors" ); + } + + public String getMessage() { + return (String) getResponse().get("message"); + } + + public String getNodeStatus() { + return (String) getResponse().get("status"); + } + +} diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index 3b24f224cd5..a206c49f6ef 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -176,6 +176,7 @@ public interface CommonParams { String OMIT_HEADER = "omitHeader"; String CORES_HANDLER_PATH = "/admin/cores"; String COLLECTIONS_HANDLER_PATH = "/admin/collections"; + String HEALTH_CHECK_HANDLER_PATH = "/admin/health"; String INFO_HANDLER_PATH = "/admin/info"; String CONFIGSETS_HANDLER_PATH = "/admin/configs"; String AUTHZ_PATH = "/admin/authorization"; @@ -185,9 +186,15 @@ public interface CommonParams { String AUTOSCALING_PATH = "/admin/autoscaling"; String AUTOSCALING_DIAGNOSTICS_PATH = "/admin/autoscaling/diagnostics"; + String STATUS = "status"; + + String OK = "OK"; + String FAILURE = "FAILURE"; + Set ADMIN_PATHS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( CORES_HANDLER_PATH, COLLECTIONS_HANDLER_PATH, + HEALTH_CHECK_HANDLER_PATH, CONFIGSETS_HANDLER_PATH, AUTHC_PATH, AUTHZ_PATH,