SOLR-7742: Support for Immutable ConfigSets

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1690828 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Gregory Chanan 2015-07-13 22:01:50 +00:00
parent b922713826
commit 8b06b59efa
14 changed files with 394 additions and 20 deletions

View File

@ -159,6 +159,8 @@ New Features
facet.range={!tag=r1}price&facet.query={!tag=q1}somequery&facet.pivot={!range=r1 query=q1}category,manufacturer
(Steve Molloy, hossman, shalin)
* SOLR-7742: Support for Immutable ConfigSets (Gregory Chanan)
Bug Fixes
----------------------

View File

@ -22,6 +22,7 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.core.SolrResourceNotFoundException;
import org.apache.solr.schema.ZkIndexSchemaReader;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@ -93,7 +94,7 @@ public class ZkSolrResourceLoader extends SolrResourceLoader {
throw new IOException("Error opening " + resource, e);
}
if (is == null) {
throw new IOException("Can't find resource '" + resource
throw new SolrResourceNotFoundException("Can't find resource '" + resource
+ "' in classpath or '" + configSetZkPath + "', cwd="
+ System.getProperty("user.dir"));
}

View File

@ -17,6 +17,7 @@
package org.apache.solr.core;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.IndexSchema;
/**
@ -30,10 +31,13 @@ public class ConfigSet {
private final IndexSchema indexSchema;
public ConfigSet(String name, SolrConfig solrConfig, IndexSchema indexSchema) {
private final NamedList properties;
public ConfigSet(String name, SolrConfig solrConfig, IndexSchema indexSchema, NamedList properties) {
this.name = name;
this.solrconfig = solrConfig;
this.indexSchema = indexSchema;
this.properties = properties;
}
public String getName() {
@ -47,4 +51,8 @@ public class ConfigSet {
public IndexSchema getIndexSchema() {
return indexSchema;
}
public NamedList getProperties() {
return properties;
}
}

View File

@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.noggit.JSONParser;
import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ConfigSetProperties {
private static final Logger log = LoggerFactory.getLogger(ConfigSetProperties.class);
/**
* Return the properties associated with the ConfigSet (e.g. immutable)
*
* @param loader the resource loader
* @param name the name of the config set properties file
* @return the properties in a NamedList
*/
public static NamedList readFromResourceLoader(SolrResourceLoader loader, String name) {
InputStreamReader reader;
try {
reader = new InputStreamReader(loader.openResource(name), StandardCharsets.UTF_8);
} catch (SolrResourceNotFoundException ex) {
log.info("Did not find ConfigSet properties", ex);
return null;
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to load reader for ConfigSet properties: " + name, ex);
}
try {
JSONParser jsonParser = new JSONParser(reader);
Object object = ObjectBuilder.getVal(jsonParser);
if (!(object instanceof Map)) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid JSON type " + object.getClass().getName() + ", expected Map");
}
return new NamedList((Map)object);
} catch (Exception ex) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to load ConfigSet properties", ex);
} finally {
IOUtils.closeQuietly(reader);
}
}
}

View File

@ -22,6 +22,7 @@ import com.google.common.cache.CacheBuilder;
import org.apache.solr.cloud.CloudConfigSetService;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
import org.joda.time.format.DateTimeFormat;
@ -72,7 +73,8 @@ public abstract class ConfigSetService {
try {
SolrConfig solrConfig = createSolrConfig(dcore, coreLoader);
IndexSchema schema = createIndexSchema(dcore, solrConfig);
return new ConfigSet(configName(dcore), solrConfig, schema);
NamedList properties = createConfigSetProperties(dcore, coreLoader);
return new ConfigSet(configName(dcore), solrConfig, schema, properties);
}
catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
@ -102,6 +104,16 @@ public abstract class ConfigSetService {
return IndexSchemaFactory.buildIndexSchema(cd.getSchemaName(), solrConfig);
}
/**
* Return the ConfigSet properties
* @param cd the core's CoreDescriptor
* @param loader the core's resource loader
* @return the ConfigSet properties
*/
protected NamedList createConfigSetProperties(CoreDescriptor cd, SolrResourceLoader loader) {
return ConfigSetProperties.readFromResourceLoader(loader, cd.getConfigSetPropertiesName());
}
/**
* Create a SolrResourceLoader for a core
* @param cd the core's CoreDescriptor

View File

@ -60,6 +60,7 @@ public class CoreDescriptor {
public static final String CORE_TRANSIENT = "transient";
public static final String CORE_NODE_NAME = "coreNodeName";
public static final String CORE_CONFIGSET = "configSet";
public static final String CORE_CONFIGSET_PROPERTIES = "configSetProperties";
public static final String SOLR_CORE_PROP_PREFIX = "solr.core.";
public static final String DEFAULT_EXTERNAL_PROPERTIES_FILE = "conf" + File.separator + "solrcore.properties";
@ -80,13 +81,14 @@ public class CoreDescriptor {
return originalExtraProperties;
}
private static ImmutableMap<String, String> defaultProperties = ImmutableMap.of(
CORE_CONFIG, "solrconfig.xml",
CORE_SCHEMA, "schema.xml",
CORE_DATADIR, "data" + File.separator,
CORE_TRANSIENT, "false",
CORE_LOADONSTARTUP, "true"
);
private static ImmutableMap<String, String> defaultProperties = new ImmutableMap.Builder<String, String>()
.put(CORE_CONFIG, "solrconfig.xml")
.put(CORE_SCHEMA, "schema.xml")
.put(CORE_CONFIGSET_PROPERTIES, "configsetprops.json")
.put(CORE_DATADIR, "data" + File.separator)
.put(CORE_TRANSIENT, "false")
.put(CORE_LOADONSTARTUP, "true")
.build();
private static ImmutableList<String> requiredProperties = ImmutableList.of(
CORE_NAME, CORE_INSTDIR, CORE_ABS_INSTDIR
@ -100,6 +102,7 @@ public class CoreDescriptor {
CORE_ULOGDIR,
CORE_SCHEMA,
CORE_PROPERTIES,
CORE_CONFIGSET_PROPERTIES,
CORE_LOADONSTARTUP,
CORE_TRANSIENT,
CORE_CONFIGSET,
@ -409,4 +412,8 @@ public class CoreDescriptor {
public String getConfigSet() {
return coreProperties.getProperty(CORE_CONFIGSET);
}
public String getConfigSetPropertiesName() {
return coreProperties.getProperty(CORE_CONFIGSET_PROPERTIES);
}
}

View File

@ -62,9 +62,17 @@ public class ImplicitPlugins {
implicits.add(getReqHandlerInfo(UpdateRequestHandler.DOC_PATH, UpdateRequestHandler.class, makeMap("update.contentType", "application/json", "json.command", "false")));
//solrconfighandler
implicits.add(getReqHandlerInfo("/config", SolrConfigHandler.class, null));
PluginInfo config = getReqHandlerInfo("/config", SolrConfigHandler.class, null);
if (solrCore.getConfigSetProperties() != null) {
config.initArgs.addAll(solrCore.getConfigSetProperties());
}
implicits.add(config);
//schemahandler
implicits.add(getReqHandlerInfo("/schema", SchemaHandler.class, null));
PluginInfo schema = getReqHandlerInfo("/schema", SchemaHandler.class, null);
if (solrCore.getConfigSetProperties() != null) {
schema.initArgs.addAll(solrCore.getConfigSetProperties());
}
implicits.add(schema);
//register replicationhandler always for SolrCloud
implicits.add(getReqHandlerInfo("/replication", ReplicationHandler.class,null));

View File

@ -172,6 +172,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
private final SolrConfig solrConfig;
private final SolrResourceLoader resourceLoader;
private volatile IndexSchema schema;
private final NamedList configSetProperties;
private final String dataDir;
private final String ulogDir;
private final UpdateHandler updateHandler;
@ -256,6 +257,10 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
schema = replacementSchema;
}
public NamedList getConfigSetProperties() {
return configSetProperties;
}
public String getDataDir() {
return dataDir;
}
@ -454,7 +459,8 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
SolrCore core = null;
try {
core = new SolrCore(getName(), getDataDir(), coreConfig.getSolrConfig(),
coreConfig.getIndexSchema(), coreDescriptor, updateHandler, solrDelPolicy, currentCore);
coreConfig.getIndexSchema(), coreConfig.getProperties(),
coreDescriptor, updateHandler, solrDelPolicy, currentCore);
// we open a new IndexWriter to pick up the latest config
core.getUpdateHandler().getSolrCoreState().newIndexWriter(core, false);
@ -646,11 +652,12 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
* @deprecated will be removed in the next release
*/
public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd) {
this(name, dataDir, config, schema, cd, null, null, null);
this(name, dataDir, config, schema, null, cd, null, null, null);
}
public SolrCore(CoreDescriptor cd, ConfigSet coreConfig) {
this(cd.getName(), null, coreConfig.getSolrConfig(), coreConfig.getIndexSchema(), cd, null, null, null);
this(cd.getName(), null, coreConfig.getSolrConfig(), coreConfig.getIndexSchema(), coreConfig.getProperties(),
cd, null, null, null);
}
/**
@ -666,6 +673,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
this.dataDir = null;
this.ulogDir = null;
this.solrConfig = null;
this.configSetProperties = null;
this.startTime = System.currentTimeMillis();
this.maxWarmingSearchers = 2; // we don't have a config yet, just pick a number.
this.slowQueryThresholdMillis = -1;
@ -698,7 +706,8 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
* @since solr 1.3
*/
public SolrCore(String name, String dataDir, SolrConfig config,
IndexSchema schema, CoreDescriptor coreDescriptor, UpdateHandler updateHandler,
IndexSchema schema, NamedList configSetProperties,
CoreDescriptor coreDescriptor, UpdateHandler updateHandler,
IndexDeletionPolicyWrapper delPolicy, SolrCore prev) {
checkNotNull(coreDescriptor, "coreDescriptor cannot be null");
@ -708,6 +717,7 @@ public final class SolrCore implements SolrInfoMBean, Closeable {
resourceLoader = config.getResourceLoader();
this.solrConfig = config;
this.configSetProperties = configSetProperties;
if (updateHandler == null) {
directoryFactory = initDirectoryFactory();

View File

@ -360,7 +360,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
throw new IOException("Error opening " + resource, e);
}
if (is==null) {
throw new IOException("Can't find resource '" + resource + "' in classpath or '" + new File(getConfigDir()).getAbsolutePath() + "'");
throw new SolrResourceNotFoundException("Can't find resource '" + resource + "' in classpath or '" + new File(getConfigDir()).getAbsolutePath() + "'");
}
return is;
}

View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.core;
import java.io.IOException;
public class SolrResourceNotFoundException extends IOException {
public SolrResourceNotFoundException() {
super();
}
public SolrResourceNotFoundException(String message) {
super(message);
}
public SolrResourceNotFoundException(String message, Throwable cause) {
super(message, cause);
}
public SolrResourceNotFoundException(Throwable cause) {
super(cause);
}
}

View File

@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
@ -44,11 +45,25 @@ import static org.apache.solr.common.params.CommonParams.JSON;
public class SchemaHandler extends RequestHandlerBase {
private static final Logger log = LoggerFactory.getLogger(SchemaHandler.class);
public static final String IMMUTABLE_CONFIGSET_ARG = "immutable";
private boolean isImmutableConfigSet = false;
@Override
public void init(NamedList args) {
super.init(args);
Object immutable = args.get(IMMUTABLE_CONFIGSET_ARG);
isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrConfigHandler.setWt(req, JSON);
String httpMethod = (String) req.getContext().get("httpMethod");
if ("POST".equals(httpMethod)) {
if (isImmutableConfigSet) {
rsp.add("errors", "ConfigSet is immutable");
return;
}
if (req.getContentStreams() == null) {
rsp.add("errors", "no stream");
return;

View File

@ -89,9 +89,12 @@ import static org.apache.solr.schema.FieldType.CLASS_NAME;
public class SolrConfigHandler extends RequestHandlerBase {
public static final Logger log = LoggerFactory.getLogger(SolrConfigHandler.class);
public static final boolean configEditing_disabled = Boolean.getBoolean("disable.configEdit");
public static final String CONFIGSET_EDITING_DISABLED_ARG = "disable.configEdit";
public static final boolean configEditing_disabled = Boolean.getBoolean(CONFIGSET_EDITING_DISABLED_ARG);
public static final String IMMUTABLE_CONFIGSET_ARG = "immutable";
private static final Map<String, SolrConfig.SolrPluginInfo> namedPlugins;
private Lock reloadLock = new ReentrantLock(true);
private boolean isImmutableConfigSet = false;
static {
Map<String, SolrConfig.SolrPluginInfo> map = new HashMap<>();
@ -103,6 +106,12 @@ public class SolrConfigHandler extends RequestHandlerBase {
namedPlugins = Collections.unmodifiableMap(map);
}
@Override
public void init(NamedList args) {
super.init(args);
Object immutable = args.get(IMMUTABLE_CONFIGSET_ARG);
isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
@ -111,8 +120,10 @@ public class SolrConfigHandler extends RequestHandlerBase {
String httpMethod = (String) req.getContext().get("httpMethod");
Command command = new Command(req, rsp, httpMethod);
if ("POST".equals(httpMethod)) {
if (configEditing_disabled)
throw new SolrException(SolrException.ErrorCode.FORBIDDEN, " solrconfig editing is not enabled");
if (configEditing_disabled || isImmutableConfigSet) {
final String reason = configEditing_disabled ? "due to " + CONFIGSET_EDITING_DISABLED_ARG : "because ConfigSet is immutable";
throw new SolrException(SolrException.ErrorCode.FORBIDDEN, " solrconfig editing is not enabled " + reason);
}
try {
command.handlePOST();
} finally {

View File

@ -0,0 +1,98 @@
package org.apache.solr.core;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.io.StringReader;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.util.RestTestBase;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.noggit.JSONParser;
import org.noggit.ObjectBuilder;
/**
* Test that a ConfigSet marked as immutable cannot be modified via
* the known APIs, i.e. SolrConfigHandler and SchemaHandler.
*/
public class TestConfigSetImmutable extends RestTestBase {
private static final String collection = "collection1";
private static final String confDir = collection + "/conf";
@Before
public void before() throws Exception {
File tmpSolrHome = createTempDir().toFile();
File tmpConfDir = new File(tmpSolrHome, confDir);
FileUtils.copyDirectory(new File(TEST_HOME()), tmpSolrHome.getAbsoluteFile());
// make the ConfigSet immutable
FileUtils.write(new File(tmpConfDir, "configsetprops.json"), new StringBuilder("{\"immutable\":\"true\"}"));
System.setProperty("managed.schema.mutable", "true");
System.setProperty("enable.update.log", "false");
createJettyAndHarness(tmpSolrHome.getAbsolutePath(), "solrconfig-managed-schema.xml", "schema-rest.xml",
"/solr", true, null);
}
@After
public void after() throws Exception {
if (jetty != null) {
jetty.stop();
jetty = null;
}
client = null;
if (restTestHarness != null) {
restTestHarness.close();
}
restTestHarness = null;
}
@Test
public void testSolrConfigHandlerImmutable() throws Exception {
String payload = "{\n" +
"'create-requesthandler' : { 'name' : '/x', 'class': 'org.apache.solr.handler.DumpRequestHandler' , 'startup' : 'lazy'}\n" +
"}";
String uri = "/config?wt=json";
String response = restTestHarness.post(uri, SolrTestCaseJ4.json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
assertNotNull(map.get("error"));
assertTrue(map.get("error").toString().contains("immutable"));
}
@Test
public void testSchemaHandlerImmutable() throws Exception {
String payload = "{\n" +
" 'add-field' : {\n" +
" 'name':'a1',\n" +
" 'type': 'string',\n" +
" 'stored':true,\n" +
" 'indexed':false\n" +
" },\n" +
" }";
String response = restTestHarness.post("/schema?wt=json", json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
assertNotNull(map.get("errors"));
assertTrue(map.get("errors").toString().contains("immutable"));
}
}

View File

@ -0,0 +1,93 @@
package org.apache.solr.core;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import com.google.common.collect.ImmutableMap;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
import org.noggit.JSONUtil;
import java.io.File;
public class TestConfigSetProperties extends SolrTestCaseJ4 {
@Rule
public TestRule testRule = RuleChain.outerRule(new SystemPropertiesRestoreRule());
@Test
public void testNoConfigSetPropertiesFile() throws Exception {
assertNull(createConfigSetProps(null));
}
@Test
public void testEmptyConfigSetProperties() throws Exception {
try {
createConfigSetProps("");
fail("Excepted SolrException");
} catch (SolrException ex) {
assertEquals(ErrorCode.SERVER_ERROR.code, ex.code());
}
}
@Test
public void testConfigSetPropertiesNotMap() throws Exception {
try {
createConfigSetProps(JSONUtil.toJSON(new String[] {"test"}));
fail("Expected SolrException");
} catch (SolrException ex) {
assertEquals(ErrorCode.SERVER_ERROR.code, ex.code());
}
}
@Test
public void testEmptyMap() throws Exception {
NamedList list = createConfigSetProps(JSONUtil.toJSON(ImmutableMap.of()));
assertEquals(0, list.size());
}
@Test
public void testMultipleProps() throws Exception {
Map map = ImmutableMap.of("immutable", "true", "someOtherProp", "true");
NamedList list = createConfigSetProps(JSONUtil.toJSON(map));
assertEquals(2, list.size());
assertEquals("true", list.get("immutable"));
assertEquals("true", list.get("someOtherProp"));
}
private NamedList createConfigSetProps(String props) throws Exception {
File testDirectory = createTempDir().toFile();
String filename = "configsetprops.json";
if (props != null) {
File confDir = new File(testDirectory, "conf");
FileUtils.forceMkdir(confDir);
FileUtils.write(new File(confDir, filename), new StringBuilder(props));
}
SolrResourceLoader loader = new SolrResourceLoader(testDirectory.getAbsolutePath());
return ConfigSetProperties.readFromResourceLoader(loader, filename);
}
}