From 8e6f865deed2db3c7014350f8b41e24bc1cf520b Mon Sep 17 00:00:00 2001 From: Erick Erickson Date: Tue, 3 Dec 2013 00:55:25 +0000 Subject: [PATCH] SOLR-5518: Move editing files to a separte request handler git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1547251 13f79535-47bb-0310-9956-ffa450edef68 --- solr/CHANGES.txt | 3 + .../handler/admin/EditFileRequestHandler.java | 357 ++++++++++++ .../handler/admin/ShowFileRequestHandler.java | 509 +++++------------- .../solr/collection1/conf/solrconfig-tlog.xml | 5 + .../solr/collection1/conf/solrconfig.xml | 6 + .../solr/cloud/TestModifyConfFiles.java | 29 +- .../solr/schema/ModifyConfFileTest.java | 33 +- .../solr/collection1/conf/solrconfig.xml | 13 + 8 files changed, 578 insertions(+), 377 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1e5f40d38a9..5a5c1bca2ca 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -117,6 +117,9 @@ New Features * SOLR-5506: Support docValues in CollationField and ICUCollationField. (Robert Muir) + + * SOLR-5518: Added EditFileRequestHandler to deal with security issues around modifying + solr configuration files. * SOLR-5023: Add support for deleteInstanceDir to be passed from SolrJ for Core Unload action. (Lyubov Romanchuk, shalin) diff --git a/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java new file mode 100644 index 00000000000..fbc4c1b3873 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/EditFileRequestHandler.java @@ -0,0 +1,357 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.solr.cloud.ZkController; +import org.apache.solr.cloud.ZkSolrResourceLoader; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.Config; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.CoreDescriptor; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.RawResponseWriter; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.InputSource; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Locale; +import java.util.Set; + +/** + * This handler uses the RawResponseWriter to give client access to + * files inside ${solr.home}/conf + *

+ * If you want to selectively restrict access some configuration files, you can list + * these files in the hidden invariants. For example to hide + * synonyms.txt and anotherfile.txt, you would register: + *

+ * 

+ * <requestHandler name="/admin/fileupdate" class="org.apache.solr.handler.admin.EditFileRequestHandler" >
+ *   <lst name="defaults">
+ *    <str name="echoParams">explicit</str>
+ *   </lst>
+ *   <lst name="invariants">
+ *    <str name="hidden">synonyms.txt</str>
+ *    <str name="hidden">anotherfile.txt</str>
+ *    <str name="hidden">*</str>
+ *   </lst>
+ * </requestHandler>
+ *
+ *

+ * At present, there is only explicit file names (including path) or the glob '*' are supported. Variants like '*.xml' + * are NOT supported.ere + *

+ *

+ * The EditFileRequestHandler uses the {@link RawResponseWriter} (wt=raw) to return + * file contents. If you need to use a different writer, you will need to change + * the registered invariant param for wt. + *

+ * If you want to override the contentType header returned for a given file, you can + * set it directly using: CONTENT_TYPE. For example, to get a plain text + * version of schema.xml, try: + * 

+ *   http://localhost:8983/solr/admin/fileedit?file=schema.xml&contentType=text/plain
+ *
+ * + * @since solr 4.7 + *

+ *

+ * You can use this handler to modify any files in the conf directory, e.g. solrconfig.xml + * or schema.xml, or even in sub-directories (e.g. velocity/error.vm) by POSTing a file. Here's an example cURL command + * 

+ *                                            curl -X POST --form "fileupload=@schema.new" 'http://localhost:8983/solr/collection1/admin/fileedit?op=write&file=schema.xml'
+ *
+ * + * or + * 
+ *                                            curl -X POST --form "fileupload=@error.new" 'http://localhost:8983/solr/collection1/admin/file?op=write&file=velocity/error.vm'
+ *
+ * + * For the first iteration, this is probably going to be used from the Solr admin screen. + * + * NOTE: Specifying a directory or simply leaving the any "file=XXX" parameters will list the contents of a directory. + * + * NOTE: You must reload the core/collection for any changes made via this handler to take effect! + * + * NOTE: If the core does not load (say schema.xml is not well formed for instance) you may be unable to replace + * the files with this interface. + * + * NOTE: Leaving this handler enabled is a security risk! This handler should be disabled in all but trusted + * (probably development only) environments! + * + * Configuration files in ZooKeeper are supported. + */ +public class EditFileRequestHandler extends RequestHandlerBase { + + protected static final Logger log = LoggerFactory.getLogger(EditFileRequestHandler.class); + + private final static String OP_PARAM = "op"; + private final static String OP_WRITE = "write"; + private final static String OP_TEST = "test"; + + ContentStream stream; + private byte[] data = null; + Set hiddenFiles; + + public EditFileRequestHandler() { + super(); + } + + @Override + public void init(NamedList args) { + super.init(args); + hiddenFiles = ShowFileRequestHandler.initHidden(invariants); + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) + throws InterruptedException, KeeperException, IOException { + + CoreContainer coreContainer = req.getCore().getCoreDescriptor().getCoreContainer(); + String op = req.getParams().get(OP_PARAM); + if (OP_WRITE.equalsIgnoreCase(op) || OP_TEST.equalsIgnoreCase(op)) { + String fname = req.getParams().get("file", null); + if (fname == null) { + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "No file name specified for write operation.")); + } else { + fname = fname.replace('\\', '/'); + stream = getOneInputStream(req, rsp); + if (stream == null) { + return; // Error already in rsp. + } + + data = IOUtils.toByteArray(new InputStreamReader(stream.getStream(), "UTF-8"), "UTF-8"); + + // If it's "solrconfig.xml", try parsing it as that object. Otherwise, if it ends in '.xml', + // see if it at least parses. + if ("solrconfig.xml".equals(fname)) { + try { + new SolrConfig("unused", new InputSource(new ByteArrayInputStream(data))); + } catch (Exception e) { + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "Invalid solr config file: " + e.getMessage())); + return; + } + } else if (fname.endsWith(".xml")) { // At least do a rudimentary test, see if the thing parses. + try { + new Config(null, null, new InputSource(new ByteArrayInputStream(data)), null, false); + } catch (Exception e) { + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "Invalid XML file: " + e.getMessage())); + return; + } + } + if (ShowFileRequestHandler.isHiddenFile(req, rsp, fname, true, hiddenFiles) == false) { + if (coreContainer.isZooKeeperAware()) { + writeToZooKeeper(req, rsp); + } else { + writeToFileSystem(req, rsp); + } + } + } + } + } + + // write the file contained in the parameter "file=XXX" to ZooKeeper. The file may be a path, e.g. + // file=velocity/error.vm or file=schema.xml + // + // Important: Assumes that the file already exists in ZK, so far we aren't creating files there. + private void writeToZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp) + throws KeeperException, InterruptedException, IOException { + + CoreContainer coreContainer = req.getCore().getCoreDescriptor().getCoreContainer(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + + String adminFile = ShowFileRequestHandler.getAdminFileFromZooKeeper(req, rsp, zkClient, hiddenFiles); + String fname = req.getParams().get("file", null); + if (OP_TEST.equals(req.getParams().get(OP_PARAM))) { + testReloadSuccess(req, rsp); + return; + } + // Persist the managed schema + try { + // Assumption: the path exists + zkClient.setData(adminFile, data, true); + log.info("Saved " + fname + " to ZooKeeper successfully."); + } catch (KeeperException.BadVersionException e) { + log.error("Cannot save file: " + fname + " to Zookeeper, " + + "ZooKeeper error: " + e.getMessage()); + rsp.setException(new SolrException(ErrorCode.SERVER_ERROR, "Cannot save file: " + fname + " to Zookeeper, " + + "ZooKeeper error: " + e.getMessage())); + } + } + + // Used when POSTing the configuration files to Solr (either ZooKeeper or locally). + // + // It takes some effort to insure that there is one (and only one) stream provided, there's no provision for + // more than one stream at present. + private ContentStream getOneInputStream(SolrQueryRequest req, SolrQueryResponse rsp) { + String file = req.getParams().get("file"); + if (file == null) { + log.error("You must specify a file for the write operation."); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "You must specify a file for the write operation.")); + return null; + } + + // Now, this is truly clumsy + Iterable streams = req.getContentStreams(); + if (streams == null) { + log.error("Input stream list was null for admin file write operation."); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "Input stream list was null for admin file write operation.")); + return null; + } + Iterator iter = streams.iterator(); + if (!iter.hasNext()) { + log.error("No input streams were in the list for admin file write operation."); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "No input streams were in the list for admin file write operation.")); + return null; + } + ContentStream stream = iter.next(); + if (iter.hasNext()) { + log.error("More than one input stream was found for admin file write operation."); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "More than one input stream was found for admin file write operation.")); + return null; + } + return stream; + } + + // Write the data passed in from the stream to the file indicated by the file=XXX parameter on the local file system + private void writeToFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + + File adminFile = ShowFileRequestHandler.getAdminFileFromFileSystem(req, rsp, hiddenFiles); + if (adminFile == null || adminFile.isDirectory()) { + String fname = req.getParams().get("file", null); + + if (adminFile == null) { + log.error("File " + fname + " was not found."); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "File " + fname + " was not found.")); + return; + } + log.error("File " + fname + " is a directory."); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "File " + fname + " is a directory.")); + return; + } + if (OP_TEST.equals(req.getParams().get(OP_PARAM))) { + testReloadSuccess(req, rsp); + return; + } + + FileUtils.copyInputStreamToFile(stream.getStream(), adminFile); + log.info("Successfully saved file " + adminFile.getAbsolutePath() + " locally"); + } + + private boolean testReloadSuccess(SolrQueryRequest req, SolrQueryResponse rsp) { + // Try writing the config to a temporary core and reloading to see that we don't allow people to shoot themselves + // in the foot. + File home = null; + try { + home = new File(FileUtils.getTempDirectory(), "SOLR_5459"); // Unlikely to name a core or collection this! + FileUtils.writeStringToFile(new File(home, "solr.xml"), "", "UTF-8"); // Use auto-discovery + File coll = new File(home, "SOLR_5459"); + + SolrCore core = req.getCore(); + CoreDescriptor desc = core.getCoreDescriptor(); + CoreContainer coreContainer = desc.getCoreContainer(); + + if (coreContainer.isZooKeeperAware()) { + try { + String confPath = ((ZkSolrResourceLoader) core.getResourceLoader()).getCollectionZkPath(); + + ZkController.downloadConfigDir(coreContainer.getZkController().getZkClient(), confPath, + new File(coll, "conf")); + } catch (Exception ex) { + log.error("Error when attempting to download conf from ZooKeeper: " + ex.getMessage()); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, + "Error when attempting to download conf from ZooKeeper" + ex.getMessage())); + return false; + } + } else { + FileUtils.copyDirectory(new File(desc.getInstanceDir(), "conf"), + new File(coll, "conf")); + } + + FileUtils.writeStringToFile(new File(coll, "core.properties"), "name=SOLR_5459", "UTF-8"); + + FileUtils.writeByteArrayToFile(new File(new File(coll, "conf"), req.getParams().get("file", null)), data); + + return tryReloading(rsp, home); + + } catch (IOException ex) { + log.warn("Caught IO exception when trying to verify configs. " + ex.getMessage()); + rsp.setException(new SolrException(ErrorCode.SERVER_ERROR, + "Caught IO exception when trying to verify configs. " + ex.getMessage())); + return false; + } finally { + if (home != null) { + try { + FileUtils.deleteDirectory(home); + } catch (IOException e) { + log.warn("Caught IO exception trying to delete temporary directory " + home + e.getMessage()); + return true; // Don't fail for this reason! + } + } + } + } + + private boolean tryReloading(SolrQueryResponse rsp, File home) { + CoreContainer cc = null; + try { + cc = CoreContainer.createAndLoad(home.getAbsolutePath(), new File(home, "solr.xml")); + if (cc.getCoreInitFailures().size() > 0) { + for (Exception ex : cc.getCoreInitFailures().values()) { + log.error("Error when attempting to reload core: " + ex.getMessage()); + rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, + "Error when attempting to reload core after writing config" + ex.getMessage())); + } + return false; + } + return true; + } finally { + if (cc != null) { + cc.shutdown(); + } + } + } + + //////////////////////// SolrInfoMBeans methods ////////////////////// + + @Override + public String getDescription() { + return "Admin Config File -- update config files directly"; + } + + @Override + public String getSource() { + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java $"; + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java index 743deb5b917..d8314258be2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java @@ -17,21 +17,17 @@ package org.apache.solr.handler.admin; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.solr.cloud.ZkController; import org.apache.solr.cloud.ZkSolrResourceLoader; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.ContentStream; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.handler.RequestHandlerBase; @@ -46,12 +42,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Set; @@ -94,47 +88,17 @@ import java.util.Set; * * * @since solr 1.3 - * - * - * As of Solr 4.7, you can use this handler to modify any files in the conf directory, e.g. solrconfig.xml - * or schema.xml, or even in sub-directories (e.g. velocity/error.vm) by POSTing a file. Here's an example cURL command - * 
- *   curl -X POST --form "fileupload=@schema.new" 'http://localhost:8983/solr/collection1/admin/file?op=write&file=schema.xml'
- *
- * - * or - * 
- * curl -X POST --form "fileupload=@error.new" 'http://localhost:8983/solr/collection1/admin/file?op=write&file=velocity/error.vm'
- *
- * - * For the first iteration, this is probably going to be used from the Solr admin screen. - * - * NOTE: Specifying a directory or simply leaving the any "file=XXX" parameters will list the contents of a directory. - * - * NOTE: You must reload the core/collection for any changes made via this handler to take effect! - * - * NOTE: If the core does not load (say schema.xml is not well formed for instance) you may be unable to replace - * the files with this interface. - * - * Configuration files in ZooKeeper are supported. - * - * Writing files out, @since solr 4.7 */ public class ShowFileRequestHandler extends RequestHandlerBase { + public static final String HIDDEN = "hidden"; + public static final String USE_CONTENT_TYPE = "contentType"; + + protected Set hiddenFiles; protected static final Logger log = LoggerFactory .getLogger(ShowFileRequestHandler.class); - public static final String HIDDEN = "hidden"; - public static final String USE_CONTENT_TYPE = "contentType"; - - protected Set hiddenFiles; - - private final static String OP_PARAM = "op"; - private final static String OP_WRITE = "write"; - private final static String OP_TEST = "test"; - public ShowFileRequestHandler() { @@ -144,161 +108,33 @@ public class ShowFileRequestHandler extends RequestHandlerBase @Override public void init(NamedList args) { super.init( args ); + hiddenFiles = initHidden(invariants); + } + public static Set initHidden(SolrParams invariants) { + + Set hiddenRet = new HashSet(); // Build a list of hidden files - hiddenFiles = new HashSet(); - if( invariants != null ) { - String[] hidden = invariants.getParams( HIDDEN ); - if( hidden != null ) { - for( String s : hidden ) { - hiddenFiles.add( s.toUpperCase(Locale.ROOT) ); + if (invariants != null) { + String[] hidden = invariants.getParams(HIDDEN); + if (hidden != null) { + for (String s : hidden) { + hiddenRet.add(s.toUpperCase(Locale.ROOT)); } } } + return hiddenRet; } - - public Set getHiddenFiles() - { - return hiddenFiles; - } - + @Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws InterruptedException, KeeperException, IOException { CoreContainer coreContainer = req.getCore().getCoreDescriptor().getCoreContainer(); - String op = req.getParams().get(OP_PARAM); - if (op == null) { - if (coreContainer.isZooKeeperAware()) { - showFromZooKeeper(req, rsp, coreContainer); - } else { - showFromFileSystem(req, rsp); - } - } else if (OP_WRITE.equalsIgnoreCase(op) || OP_TEST.equalsIgnoreCase(op)) { - String fname = req.getParams().get("file", null); - if (fname == null) { - rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "No file name specified for write operation.")); - } else { - fname = fname.replace('\\', '/'); - if (isHiddenFile(req, rsp, fname, true) == false) { - if (coreContainer.isZooKeeperAware()) { - writeToZooKeeper(req, rsp); - } else { - writeToFileSystem(req, rsp); - } - } - } - } - } - - // See if we should deal with this file - - private boolean isHiddenFile(SolrQueryRequest req, SolrQueryResponse rsp, String fnameIn, boolean reportError) { - String fname = fnameIn.toUpperCase(Locale.ROOT); - if (hiddenFiles.contains(fname) || hiddenFiles.contains("*")) { - if (reportError) { - log.error("Cannot access " + fname); - rsp.setException(new SolrException(ErrorCode.FORBIDDEN, "Can not access: " + fnameIn)); - } - return true; - } - - // This is slightly off, a valid path is something like ./schema.xml. I don't think it's worth the effort though - // to fix it to handle all possibilities though. - if (fname.indexOf("..") >= 0 || fname.startsWith(".")) { - if (reportError) { - log.error("Invalid path: " + fname); - rsp.setException(new SolrException(ErrorCode.FORBIDDEN, "Invalid path: " + fnameIn)); - } - return true; - } - - // Make sure that if the schema is managed, we don't allow editing. Don't really want to put - // this in the init since we're not entirely sure when the managed schema will get initialized relative to this - // handler. - SolrCore core = req.getCore(); - IndexSchema schema = core.getLatestSchema(); - if (schema instanceof ManagedIndexSchema) { - String managed = schema.getResourceName(); - - if (fname.equalsIgnoreCase(managed)) { - return true; - } - } - return false; - } - - // Refactored to be usable from multiple methods. Gets the path of the requested file from ZK. - // Returns null if the file is not found. - // - // Assumes that the file is in a parameter called "file". - - private String getAdminFileFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp, SolrZkClient zkClient) - throws KeeperException, InterruptedException { - String adminFile = null; - SolrCore core = req.getCore(); - - final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core - .getResourceLoader(); - String confPath = loader.getCollectionZkPath(); - - String fname = req.getParams().get("file", null); - if (fname == null) { - adminFile = confPath; + if (coreContainer.isZooKeeperAware()) { + showFromZooKeeper(req, rsp, coreContainer); } else { - fname = fname.replace('\\', '/'); // normalize slashes - if (isHiddenFile(req, rsp, fname, true)) { - return null; - } - if (fname.startsWith("/")) { // Only files relative to conf are valid - fname = fname.substring(1); - } - adminFile = confPath + "/" + fname; - } - - // Make sure the file exists, is readable and is not a hidden file - if (!zkClient.exists(adminFile, true)) { - log.error("Can not find: " + adminFile); - rsp.setException(new SolrException(ErrorCode.NOT_FOUND, "Can not find: " - + adminFile)); - return null; - } - - return adminFile; - } - - // write the file contained in the parameter "file=XXX" to ZooKeeper. The file may be a path, e.g. - // file=velocity/error.vm or file=schema.xml - // - // Important: Assumes that the file already exists in ZK, so far we aren't creating files there. - private void writeToZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp) - throws KeeperException, InterruptedException, IOException { - - CoreContainer coreContainer = req.getCore().getCoreDescriptor().getCoreContainer(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - - String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient); - ContentStream stream = getOneInputStream(req, rsp); - if (stream == null) { - return; // Error already in rsp. - } - - byte[] data = IOUtils.toByteArray(new InputStreamReader(stream.getStream(), "UTF-8"), "UTF-8"); - String fname = req.getParams().get("file", null); - if (OP_TEST.equals(req.getParams().get(OP_PARAM))) { - testReloadSuccess(req, rsp, stream); - return; - } - // Persist the managed schema - try { - // Assumption: the path exists - zkClient.setData(adminFile, data, true); - log.info("Saved " + fname + " to ZooKeeper successfully."); - } catch (KeeperException.BadVersionException e) { - log.error("Cannot save file: " + fname + " to Zookeeper, " + - "ZooKeeper error: " + e.getMessage()); - rsp.setException(new SolrException(ErrorCode.SERVER_ERROR, "Cannot save file: " + fname + " to Zookeeper, " + - "ZooKeeper error: " + e.getMessage())); + showFromFileSystem(req, rsp); } } @@ -309,7 +145,7 @@ public class ShowFileRequestHandler extends RequestHandlerBase SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient); + String adminFile = getAdminFileFromZooKeeper(req, rsp, zkClient, hiddenFiles); if (adminFile == null) { return; @@ -321,7 +157,7 @@ public class ShowFileRequestHandler extends RequestHandlerBase NamedList> files = new SimpleOrderedMap>(); for (String f : children) { - if (isHiddenFile(req, rsp, f, false)) { + if (isHiddenFile(req, rsp, f, false, hiddenFiles)) { continue; } @@ -352,185 +188,9 @@ public class ShowFileRequestHandler extends RequestHandlerBase rsp.setHttpCaching(false); } - - // Used when POSTing the configuration files to Solr (either ZooKeeper or locally). - // - // It takes some effort to insure that there is one (and only one) stream provided, there's no provision for - // more than one stream at present. - private ContentStream getOneInputStream(SolrQueryRequest req, SolrQueryResponse rsp) { - String file = req.getParams().get("file"); - if (file == null) { - log.error("You must specify a file for the write operation."); - rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "You must specify a file for the write operation.")); - return null; - } - - // Now, this is truly clumsy - Iterable streams = req.getContentStreams(); - if (streams == null) { - log.error("Input stream list was null for admin file write operation."); - rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "Input stream list was null for admin file write operation.")); - return null; - } - Iterator iter = streams.iterator(); - if (!iter.hasNext()) { - log.error("No input streams were in the list for admin file write operation."); - rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "No input streams were in the list for admin file write operation.")); - return null; - } - ContentStream stream = iter.next(); - if (iter.hasNext()) { - log.error("More than one input stream was found for admin file write operation."); - rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, "More than one input stream was found for admin file write operation.")); - return null; - } - return stream; - } - - - // Write the data passed in from the stream to the file indicated by the file=XXX parameter on the local file system - private void writeToFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - ContentStream stream = getOneInputStream(req, rsp); - if (stream == null) { - return; // Error should already have been logged. - } - - File adminFile = getAdminFileFromFileSystem(req, rsp); - if (adminFile == null || adminFile.isDirectory()) { - String fname = req.getParams().get("file", null); - - if (adminFile == null) { - log.error("File " + fname + " was not found."); - rsp.setException(new SolrException( ErrorCode.BAD_REQUEST, "File " + fname + " was not found.")); - return; - } - log.error("File " + fname + " is a directory."); - rsp.setException(new SolrException( ErrorCode.BAD_REQUEST, "File " + fname + " is a directory.")); - return; - } - if (OP_TEST.equals(req.getParams().get(OP_PARAM))) { - testReloadSuccess(req, rsp, stream); - return; - } - - FileUtils.copyInputStreamToFile(stream.getStream(), adminFile); - log.info("Successfully saved file " + adminFile.getAbsolutePath() + " locally"); - } - - private boolean testReloadSuccess(SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream) { - // Try writing the config to a temporary core and reloading to see that we don't allow people to shoot themselves - // in the foot. - File home = null; - try { - home = new File(FileUtils.getTempDirectory(), "SOLR_5459"); // Unlikely to name a core or collection this! - FileUtils.writeStringToFile(new File(home, "solr.xml"), "", "UTF-8"); // Use auto-discovery - File coll = new File(home, "SOLR_5459"); - - SolrCore core = req.getCore(); - CoreDescriptor desc = core.getCoreDescriptor(); - CoreContainer coreContainer = desc.getCoreContainer(); - - if (coreContainer.isZooKeeperAware()) { - try { - String confPath = ((ZkSolrResourceLoader) core.getResourceLoader()).getCollectionZkPath(); - - ZkController.downloadConfigDir(coreContainer.getZkController().getZkClient(), confPath, - new File(coll, "conf")); - } catch (Exception ex) { - log.error("Error when attempting to download conf from ZooKeeper: " + ex.getMessage()); - rsp.setException(new SolrException(ErrorCode.BAD_REQUEST, - "Error when attempting to download conf from ZooKeeper" + ex.getMessage())); - return false; - } - } else { - FileUtils.copyDirectory(new File(desc.getInstanceDir(), "conf"), - new File(coll, "conf")); - } - - FileUtils.writeStringToFile(new File(coll, "core.properties"), "name=SOLR_5459", "UTF-8"); - - FileUtils.copyInputStreamToFile(stream.getStream(), - new File(new File(coll, "conf"), req.getParams().get("file", null))); - - return tryReloading(rsp, home); - - } catch (IOException ex) { - log.warn("Caught IO exception when trying to verify configs. " + ex.getMessage()); - rsp.setException(new SolrException(ErrorCode.SERVER_ERROR, - "Caught IO exception when trying to verify configs. " + ex.getMessage())); - return false; - } finally { - if (home != null) { - try { - FileUtils.deleteDirectory(home); - } catch (IOException e) { - log.warn("Caught IO exception trying to delete temporary directory " + home + e.getMessage()); - return true; // Don't fail for this reason! - } - } - } - } - - private boolean tryReloading(SolrQueryResponse rsp, File home) { - CoreContainer cc = null; - try { - cc = CoreContainer.createAndLoad(home.getAbsolutePath(), new File(home, "solr.xml")); - if (cc.getCoreInitFailures().size() > 0) { - for (Exception ex : cc.getCoreInitFailures().values()) { - log.error("Error when attempting to reload core: " + ex.getMessage()); - rsp.setException(new SolrException( ErrorCode.BAD_REQUEST, - "Error when attempting to reload core after writing config" + ex.getMessage())); - } - return false; - } - return true; - } finally { - if (cc != null) { - cc.shutdown(); - } - } - } - - // Find the file indicated by the "file=XXX" parameter or the root of the conf directory on the local - // file system. Respects all the "interesting" stuff around what the resource loader does to find files. - private File getAdminFileFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) { - File adminFile = null; - final SolrResourceLoader loader = req.getCore().getResourceLoader(); - File configdir = new File( loader.getConfigDir() ); - if (!configdir.exists()) { - // TODO: maybe we should just open it this way to start with? - try { - configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() ); - } catch (URISyntaxException e) { - log.error("Can not access configuration directory!"); - rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access configuration directory!", e)); - return null; - } - } - String fname = req.getParams().get("file", null); - if( fname == null ) { - adminFile = configdir; - } - else { - fname = fname.replace( '\\', '/' ); // normalize slashes - if( hiddenFiles.contains( fname.toUpperCase(Locale.ROOT) ) ) { - log.error("Can not access: "+ fname); - rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Can not access: "+fname )); - return null; - } - if( fname.indexOf( ".." ) >= 0 ) { - log.error("Invalid path: "+ fname); - rsp.setException(new SolrException( ErrorCode.FORBIDDEN, "Invalid path: "+fname )); - return null; - } - adminFile = new File( configdir, fname ); - } - return adminFile; - } - // Return the file indicated (or the directory listing) from the local file system. private void showFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp) { - File adminFile = getAdminFileFromFileSystem(req, rsp); + File adminFile = getAdminFileFromFileSystem(req, rsp, hiddenFiles); if (adminFile == null) { // exception already recorded return; @@ -561,7 +221,7 @@ public class ShowFileRequestHandler extends RequestHandlerBase String path = f.getAbsolutePath().substring( basePath ); path = path.replace( '\\', '/' ); // normalize slashes - if (isHiddenFile(req, rsp, f.getName().replace('\\', '/'), false)) { + if (isHiddenFile(req, rsp, f.getName().replace('\\', '/'), false, hiddenFiles)) { continue; } @@ -593,6 +253,127 @@ public class ShowFileRequestHandler extends RequestHandlerBase rsp.setHttpCaching(false); } + //////////////////////// Static methods ////////////////////////////// + + public static boolean isHiddenFile(SolrQueryRequest req, SolrQueryResponse rsp, String fnameIn, boolean reportError, + Set hiddenFiles) { + String fname = fnameIn.toUpperCase(Locale.ROOT); + if (hiddenFiles.contains(fname) || hiddenFiles.contains("*")) { + if (reportError) { + log.error("Cannot access " + fname); + rsp.setException(new SolrException(SolrException.ErrorCode.FORBIDDEN, "Can not access: " + fnameIn)); + } + return true; + } + + // This is slightly off, a valid path is something like ./schema.xml. I don't think it's worth the effort though + // to fix it to handle all possibilities though. + if (fname.indexOf("..") >= 0 || fname.startsWith(".")) { + if (reportError) { + log.error("Invalid path: " + fname); + rsp.setException(new SolrException(SolrException.ErrorCode.FORBIDDEN, "Invalid path: " + fnameIn)); + } + return true; + } + + // Make sure that if the schema is managed, we don't allow editing. Don't really want to put + // this in the init since we're not entirely sure when the managed schema will get initialized relative to this + // handler. + SolrCore core = req.getCore(); + IndexSchema schema = core.getLatestSchema(); + if (schema instanceof ManagedIndexSchema) { + String managed = schema.getResourceName(); + + if (fname.equalsIgnoreCase(managed)) { + return true; + } + } + return false; + } + + // Refactored to be usable from multiple methods. Gets the path of the requested file from ZK. + // Returns null if the file is not found. + // + // Assumes that the file is in a parameter called "file". + + public static String getAdminFileFromZooKeeper(SolrQueryRequest req, SolrQueryResponse rsp, SolrZkClient zkClient, + Set hiddenFiles) + throws KeeperException, InterruptedException { + String adminFile = null; + SolrCore core = req.getCore(); + + final ZkSolrResourceLoader loader = (ZkSolrResourceLoader) core + .getResourceLoader(); + String confPath = loader.getCollectionZkPath(); + + String fname = req.getParams().get("file", null); + if (fname == null) { + adminFile = confPath; + } else { + fname = fname.replace('\\', '/'); // normalize slashes + if (isHiddenFile(req, rsp, fname, true, hiddenFiles)) { + return null; + } + if (fname.startsWith("/")) { // Only files relative to conf are valid + fname = fname.substring(1); + } + adminFile = confPath + "/" + fname; + } + + // Make sure the file exists, is readable and is not a hidden file + if (!zkClient.exists(adminFile, true)) { + log.error("Can not find: " + adminFile); + rsp.setException(new SolrException(SolrException.ErrorCode.NOT_FOUND, "Can not find: " + + adminFile)); + return null; + } + + return adminFile; + } + + + // Find the file indicated by the "file=XXX" parameter or the root of the conf directory on the local + // file system. Respects all the "interesting" stuff around what the resource loader does to find files. + public static File getAdminFileFromFileSystem(SolrQueryRequest req, SolrQueryResponse rsp, + Set hiddenFiles) { + File adminFile = null; + final SolrResourceLoader loader = req.getCore().getResourceLoader(); + File configdir = new File( loader.getConfigDir() ); + if (!configdir.exists()) { + // TODO: maybe we should just open it this way to start with? + try { + configdir = new File( loader.getClassLoader().getResource(loader.getConfigDir()).toURI() ); + } catch (URISyntaxException e) { + log.error("Can not access configuration directory!"); + rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Can not access configuration directory!", e)); + return null; + } + } + String fname = req.getParams().get("file", null); + if( fname == null ) { + adminFile = configdir; + } + else { + fname = fname.replace( '\\', '/' ); // normalize slashes + if( hiddenFiles.contains( fname.toUpperCase(Locale.ROOT) ) ) { + log.error("Can not access: "+ fname); + rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Can not access: "+fname )); + return null; + } + if( fname.indexOf( ".." ) >= 0 ) { + log.error("Invalid path: "+ fname); + rsp.setException(new SolrException( SolrException.ErrorCode.FORBIDDEN, "Invalid path: "+fname )); + return null; + } + adminFile = new File( configdir, fname ); + } + return adminFile; + } + + public final Set getHiddenFiles() { + return hiddenFiles; + } + //////////////////////// SolrInfoMBeans methods ////////////////////// @Override diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml index d4a36cd3923..22c5b3ff57b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml @@ -92,6 +92,11 @@ + + + bogus.txt + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index e38e1d6a01f..d2413b09654 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -240,6 +240,12 @@ + + + bogus.txt + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java b/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java index aa36cde2fd7..d92e3e87c04 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestModifyConfFiles.java @@ -16,6 +16,8 @@ package org.apache.solr.cloud; * limitations under the License. */ +import org.apache.commons.io.FileUtils; +import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.cloud.SolrZkClient; @@ -23,6 +25,8 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; +import java.io.File; + public class TestModifyConfFiles extends AbstractFullDistribZkTestBase { public TestModifyConfFiles() { @@ -38,7 +42,7 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase { params.set("op", "write"); params.set("file", "schema.xml"); QueryRequest request = new QueryRequest(params); - request.setPath("/admin/file"); + request.setPath("/admin/fileedit"); try { client.request(request); fail("Should have caught exception"); @@ -50,7 +54,7 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase { params.set("stream.body", "Testing rewrite of schema.xml file."); params.set("op", "test"); request = new QueryRequest(params); - request.setPath("/admin/file"); + request.setPath("/admin/fileedit"); try { client.request(request); fail("Should have caught exception"); @@ -61,7 +65,7 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase { params.set("op", "write"); params.set("file", "bogus.txt"); request = new QueryRequest(params); - request.setPath("/admin/file"); + request.setPath("/admin/fileedit"); try { client.request(request); fail("Should have caught exception"); @@ -69,16 +73,29 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase { assertEquals(e.getMessage(), "Can not access: bogus.txt"); } + try { + params.set("file", "schema.xml"); + request = new QueryRequest(params); + request.setPath("/admin/fileedit"); + client.request(request); + fail("Should have caught exception since it's mal-formed XML"); + } catch (Exception e) { + assertTrue("Should have a sax parser exception here!", + e.getMessage().contains("Invalid XML file: org.xml.sax.SAXParseException")); + } + + String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf"; + params.set("stream.body", FileUtils.readFileToString(new File(top, "schema-tiny.xml"), "UTF-8")); params.set("file", "schema.xml"); request = new QueryRequest(params); - request.setPath("/admin/file"); + request.setPath("/admin/fileedit"); client.request(request); SolrZkClient zkClient = cloudClient.getZkStateReader().getZkClient(); String contents = new String(zkClient.getData("/configs/conf1/schema.xml", null, null, true), "UTF-8"); - assertTrue("Schema contents should have changed!", "Testing rewrite of schema.xml file.".equals(contents)); + assertTrue("Schema contents should have changed!", contents.contains("")); // Create a velocity/whatever node. Put a bit of data in it. See if you can change it. zkClient.makePath("/configs/conf1/velocity/test.vm", false, true); @@ -86,7 +103,7 @@ public class TestModifyConfFiles extends AbstractFullDistribZkTestBase { params.set("stream.body", "Some bogus stuff for a test."); params.set("file", "velocity/test.vm"); request = new QueryRequest(params); - request.setPath("/admin/file"); + request.setPath("/admin/fileedit"); client.request(request); diff --git a/solr/core/src/test/org/apache/solr/schema/ModifyConfFileTest.java b/solr/core/src/test/org/apache/solr/schema/ModifyConfFileTest.java index c763190be19..7d04a4d12a7 100644 --- a/solr/core/src/test/org/apache/solr/schema/ModifyConfFileTest.java +++ b/solr/core/src/test/org/apache/solr/schema/ModifyConfFileTest.java @@ -21,6 +21,7 @@ import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule; import org.apache.commons.codec.Charsets; import org.apache.commons.io.FileUtils; import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.ContentStreamBase; @@ -41,6 +42,7 @@ import java.util.ArrayList; public class ModifyConfFileTest extends SolrTestCaseJ4 { private File solrHomeDirectory = new File(TEMP_DIR, this.getClass().getName()); + @Rule public TestRule solrTestRules = RuleChain.outerRule(new SystemPropertiesRestoreRule()); @@ -69,26 +71,42 @@ public class ModifyConfFileTest extends SolrTestCaseJ4 { SolrCore core = cc.getCore("core1"); SolrQueryResponse rsp = new SolrQueryResponse(); - SolrRequestHandler handler = core.getRequestHandler("/admin/file"); + SolrRequestHandler handler = core.getRequestHandler("/admin/fileedit"); ModifiableSolrParams params = params("file","schema.xml", "op","write"); core.execute(handler, new LocalSolrQueryRequest(core, params), rsp); assertEquals(rsp.getException().getMessage(), "Input stream list was null for admin file write operation."); - params = params("op", "write", "stream.body", "Testing rewrite of schema.xml file."); + params = params("op", "write"); core.execute(handler, new LocalSolrQueryRequest(core, params), rsp); assertEquals(rsp.getException().getMessage(), "No file name specified for write operation."); + ArrayList streams = new ArrayList( 2 ); + streams.add(new ContentStreamBase.StringStream("Testing rewrite of schema.xml file." ) ); params = params("op", "write", "file", "bogus.txt"); - core.execute(handler, new LocalSolrQueryRequest(core, params), rsp); + LocalSolrQueryRequest locReq = new LocalSolrQueryRequest(core, params); + locReq.setContentStreams(streams); + core.execute(handler, locReq, rsp); assertEquals(rsp.getException().getMessage(), "Can not access: bogus.txt"); - ArrayList streams = new ArrayList( 2 ); - streams.add( new ContentStreamBase.StringStream( "Testing rewrite of schema.xml file." ) ); + String top = SolrTestCaseJ4.TEST_HOME() + "/collection1/conf"; + String badConf = FileUtils.readFileToString(new File(top, "solrconfig-minimal.xml"), "UTF-8").replace("", ""); + + params = params("op", "write", "file", "solrconfig.xml"); + locReq = new LocalSolrQueryRequest(core, params); + streams.clear(); + streams.add(new ContentStreamBase.StringStream(badConf)); + locReq.setContentStreams(streams); + core.execute(handler, locReq, rsp); + assertTrue("should have detected an error early!", + rsp.getException().getMessage().contains("\"dataDir\"")); + + assertTrue("should have detected an error early!", + rsp.getException().getMessage().contains("\"\"")); params = params("op", "test", "file", "schema.xml", "stream.body", "Testing rewrite of schema.xml file."); - LocalSolrQueryRequest locReq = new LocalSolrQueryRequest(core, params); + locReq = new LocalSolrQueryRequest(core, params); locReq.setContentStreams(streams); core.execute(handler, locReq, rsp); @@ -116,7 +134,8 @@ public class ModifyConfFileTest extends SolrTestCaseJ4 { streams.clear(); params = params(); locReq = new LocalSolrQueryRequest(core, params); - core.execute(handler, locReq, rsp); + + core.execute(core.getRequestHandler("/admin/file"), locReq, rsp); NamedList res = rsp.getValues(); diff --git a/solr/example/solr/collection1/conf/solrconfig.xml b/solr/example/solr/collection1/conf/solrconfig.xml index fa467336a97..c770599b7e2 100755 --- a/solr/example/solr/collection1/conf/solrconfig.xml +++ b/solr/example/solr/collection1/conf/solrconfig.xml @@ -1139,6 +1139,19 @@ --> + +