From 941c5e92ba6ff76e913746caf68e05b563983f17 Mon Sep 17 00:00:00 2001 From: Steve Rowe Date: Tue, 25 Oct 2016 17:27:37 -0400 Subject: [PATCH 1/6] SOLR-9579: fix intellij compilation: add lucene core dependency to the langid contrib --- dev-tools/idea/solr/contrib/langid/langid.iml | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-tools/idea/solr/contrib/langid/langid.iml b/dev-tools/idea/solr/contrib/langid/langid.iml index 1fa1e5dfc2f..28223bd1352 100644 --- a/dev-tools/idea/solr/contrib/langid/langid.iml +++ b/dev-tools/idea/solr/contrib/langid/langid.iml @@ -28,6 +28,7 @@ + From d25a6181612fa00a8e5a1c1e6d889b6d21053486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 26 Oct 2016 00:37:11 +0200 Subject: [PATCH 2/6] SOLR-9481: Authentication and Authorization plugins now work in standalone mode, including edit API --- solr/CHANGES.txt | 4 + .../org/apache/solr/core/CoreContainer.java | 31 +-- .../handler/admin/SecurityConfHandler.java | 142 +++++++---- .../admin/SecurityConfHandlerLocal.java | 102 ++++++++ .../handler/admin/SecurityConfHandlerZk.java | 92 ++++++++ .../apache/solr/security/BasicAuthPlugin.java | 14 +- .../SecurityConfHandlerLocalForTesting.java | 43 ++++ .../admin/SecurityConfHandlerTest.java | 66 +++--- .../security/BasicAuthIntegrationTest.java | 4 +- .../security/BasicAuthStandaloneTest.java | 220 ++++++++++++++++++ 10 files changed, 624 insertions(+), 94 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerLocal.java create mode 100644 solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerZk.java create mode 100644 solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerLocalForTesting.java create mode 100644 solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 8e6ee7ea19a..62c9d4afff7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -160,6 +160,10 @@ New Features * SOLR-9654: Add "overrequest" parameter to JSON Facet API to control amount of overrequest on a distributed terms facet. (yonik) +* SOLR-9481: Authentication and Authorization plugins now work in standalone mode if security.json is placed in + SOLR_HOME on every node. Editing config through API is supported but affects only that one node. + (janhoy) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index a020b00ff6f..e3d577efdf0 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -48,7 +48,6 @@ import org.apache.solr.cloud.Overseer; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.Utils; @@ -60,6 +59,8 @@ import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.handler.admin.InfoHandler; import org.apache.solr.handler.admin.SecurityConfHandler; +import org.apache.solr.handler.admin.SecurityConfHandlerLocal; +import org.apache.solr.handler.admin.SecurityConfHandlerZk; import org.apache.solr.handler.admin.ZookeeperInfoHandler; import org.apache.solr.handler.component.ShardHandlerFactory; import org.apache.solr.logging.LogWatcher; @@ -78,7 +79,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkNotNull; -import static java.util.Collections.EMPTY_MAP; import static org.apache.solr.common.params.CommonParams.AUTHC_PATH; import static org.apache.solr.common.params.CommonParams.AUTHZ_PATH; import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH; @@ -88,7 +88,6 @@ import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH; import static org.apache.solr.common.params.CommonParams.ZK_PATH; import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP; - /** * * @since solr 1.3 @@ -296,10 +295,10 @@ public class CoreContainer { } if (pluginClassName != null) { - log.info("Authentication plugin class obtained from ZK: "+pluginClassName); + log.debug("Authentication plugin class obtained from security.json: "+pluginClassName); } else if (System.getProperty(AUTHENTICATION_PLUGIN_PROP) != null) { pluginClassName = System.getProperty(AUTHENTICATION_PLUGIN_PROP); - log.info("Authentication plugin class obtained from system property '" + + log.debug("Authentication plugin class obtained from system property '" + AUTHENTICATION_PLUGIN_PROP + "': " + pluginClassName); } else { log.debug("No authentication plugin used."); @@ -463,14 +462,11 @@ public class CoreContainer { MDCLoggingContext.setNode(this); - ZkStateReader.ConfigData securityConfig = isZooKeeperAware() ? getZkController().getZkStateReader().getSecurityProps(false) : new ZkStateReader.ConfigData(EMPTY_MAP, -1); - initializeAuthorizationPlugin((Map) securityConfig.data.get("authorization")); - initializeAuthenticationPlugin((Map) securityConfig.data.get("authentication")); - + securityConfHandler = isZooKeeperAware() ? new SecurityConfHandlerZk(this) : new SecurityConfHandlerLocal(this); + reloadSecurityProperties(); this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins()); containerHandlers.put(ZK_PATH, new ZookeeperInfoHandler(this)); - securityConfHandler = new SecurityConfHandler(this); collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class); containerHandlers.put(COLLECTIONS_HANDLER_PATH, collectionsHandler); infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class); @@ -567,10 +563,17 @@ public class CoreContainer { } public void securityNodeChanged() { - log.info("Security node changed"); - ZkStateReader.ConfigData securityConfig = getZkController().getZkStateReader().getSecurityProps(false); - initializeAuthorizationPlugin((Map) securityConfig.data.get("authorization")); - initializeAuthenticationPlugin((Map) securityConfig.data.get("authentication")); + log.info("Security node changed, reloading security.json"); + reloadSecurityProperties(); + } + + /** + * Make sure securityConfHandler is initialized + */ + private void reloadSecurityProperties() { + SecurityConfHandler.SecurityConfig securityConfig = securityConfHandler.getSecurityConfig(false); + initializeAuthorizationPlugin((Map) securityConfig.getData().get("authorization")); + initializeAuthenticationPlugin((Map) securityConfig.getData().get("authentication")); } private static void checkForDuplicateCoreNames(List cds) { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java index 0f4dd7bc847..1fea4312adb 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandler.java @@ -17,6 +17,8 @@ package org.apache.solr.handler.admin; import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -25,7 +27,6 @@ import java.util.Map; import java.util.Objects; import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.ZkStateReader.ConfigData; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; @@ -37,10 +38,14 @@ import org.apache.solr.security.AuthorizationContext; import org.apache.solr.security.ConfigEditablePlugin; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.util.CommandOperation; -import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class SecurityConfHandler extends RequestHandlerBase implements PermissionNameProvider { - private CoreContainer cores; +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; + +public abstract class SecurityConfHandler extends RequestHandlerBase implements PermissionNameProvider { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected CoreContainer cores; public SecurityConfHandler(CoreContainer coreContainer) { this.cores = coreContainer; @@ -92,11 +97,12 @@ public class SecurityConfHandler extends RequestHandlerBase implements Permissio if (ops == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No commands"); } - for (; ; ) { - ConfigData data = getSecurityProps(true); - Map latestConf = (Map) data.data.get(key); + for (int count = 1; count <= 3 ; count++ ) { + SecurityConfig securityConfig = getSecurityConfig(true); + Map data = securityConfig.getData(); + Map latestConf = (Map) data.get(key); if (latestConf == null) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "No configuration present for " + key); + throw new SolrException(SERVER_ERROR, "No configuration present for " + key); } List commandsCopy = CommandOperation.clone(ops); Map out = configEditablePlugin.edit(Utils.getDeepCopy(latestConf, 4) , commandsCopy); @@ -106,20 +112,31 @@ public class SecurityConfHandler extends RequestHandlerBase implements Permissio rsp.add(CommandOperation.ERR_MSGS, errs); return; } - //no edits + log.debug("No edits made"); return; } else { if(!Objects.equals(latestConf.get("class") , out.get("class"))){ - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "class cannot be modified"); + throw new SolrException(SERVER_ERROR, "class cannot be modified"); } Map meta = getMapValue(out, ""); - meta.put("v", data.version+1);//encode the expected zkversion - data.data.put(key, out); - if(persistConf("/security.json", Utils.toJSON(data.data), data.version)) return; + meta.put("v", securityConfig.getVersion()+1);//encode the expected zkversion + data.put(key, out); + + if(persistConf(securityConfig)) { + securityConfEdited(); + return; + } } + log.debug("Security edit operation failed {} time(s)" + count); } + throw new SolrException(SERVER_ERROR, "Failed to persist security config after 3 attempts. Giving up"); } + /** + * Hook where you can do stuff after a config has been edited. Defaults to NOP + */ + protected void securityConfEdited() {} + Object getPlugin(String key) { Object plugin = null; if ("authentication".equals(key)) plugin = cores.getAuthenticationPlugin(); @@ -127,38 +144,14 @@ public class SecurityConfHandler extends RequestHandlerBase implements Permissio return plugin; } - ConfigData getSecurityProps(boolean getFresh) { - return cores.getZkController().getZkStateReader().getSecurityProps(getFresh); - } - - boolean persistConf(String path, byte[] buf, int version) { - try { - cores.getZkController().getZkClient().setData(path,buf,version, true); - return true; - } catch (KeeperException.BadVersionException bdve){ - return false; - } catch (Exception e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, " Unable to persist conf",e); - } - } - - - private void getConf(SolrQueryResponse rsp, String key) { - ConfigData map = cores.getZkController().getZkStateReader().getSecurityProps(false); - Object o = map == null ? null : map.data.get(key); - if (o == null) { - rsp.add(CommandOperation.ERR_MSGS, Collections.singletonList("No " + key + " configured")); - } else { - rsp.add(key+".enabled", getPlugin(key)!=null); - rsp.add(key, o); - } - } + protected abstract void getConf(SolrQueryResponse rsp, String key); public static Map getMapValue(Map lookupMap, String key) { Map m = (Map) lookupMap.get(key); if (m == null) lookupMap.put(key, m = new LinkedHashMap<>()); return m; } + public static List getListValue(Map lookupMap, String key) { List l = (List) lookupMap.get(key); if (l == null) lookupMap.put(key, l= new ArrayList()); @@ -170,6 +163,77 @@ public class SecurityConfHandler extends RequestHandlerBase implements Permissio return "Edit or read security configuration"; } + /** + * Gets security.json from source + */ + public abstract SecurityConfig getSecurityConfig(boolean getFresh); + /** + * Persist security.json to the source, optionally with a version + */ + protected abstract boolean persistConf(SecurityConfig securityConfig) throws IOException; + + /** + * Object to hold security.json as nested Map<String,Object> and optionally its version. + * The version property is optional and defaults to -1 if not initialized. + * The data object defaults to EMPTY_MAP if not set + */ + public static class SecurityConfig { + private Map data = Collections.EMPTY_MAP; + private int version = -1; + + public SecurityConfig() {} + + /** + * Sets the data as a Map + * @param data a Map + * @return SecurityConf object (builder pattern) + */ + public SecurityConfig setData(Map data) { + this.data = data; + return this; + } + + /** + * Sets the data as an Object, but the object needs to be of type Map + * @param data an Object of type Map<String,Object> + * @return SecurityConf object (builder pattern) + */ + public SecurityConfig setData(Object data) { + if (data instanceof Map) { + this.data = (Map) data; + return this; + } else { + throw new SolrException(SERVER_ERROR, "Illegal format when parsing security.json, not object"); + } + } + + /** + * Sets version + * @param version integer for version. Depends on underlying storage + * @return SecurityConf object (builder pattern) + */ + public SecurityConfig setVersion(int version) { + this.version = version; + return this; + } + + public Map getData() { + return data; + } + + public int getVersion() { + return version; + } + + /** + * Set data from input stream + * @param securityJsonInputStream an input stream for security.json + * @return this (builder pattern) + */ + public SecurityConfig setData(InputStream securityJsonInputStream) { + return setData(Utils.fromJSON(securityJsonInputStream)); + } } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerLocal.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerLocal.java new file mode 100644 index 00000000000..d6745c5a6fe --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerLocal.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.util.CommandOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Security Configuration Handler which works on standalone local files + */ +public class SecurityConfHandlerLocal extends SecurityConfHandler { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + protected Path securityJsonPath; + + public SecurityConfHandlerLocal(CoreContainer coreContainer) { + super(coreContainer); + securityJsonPath = SolrResourceLoader.locateSolrHome().resolve("security.json"); + } + + /** + * Fetches security props from SOLR_HOME + * @param getFresh NOP + * @return SecurityConfig whose data property either contains security.json, or an empty map if not found + */ + @Override + public SecurityConfig getSecurityConfig(boolean getFresh) { + if (Files.exists(securityJsonPath)) { + try (InputStream securityJsonIs = Files.newInputStream(securityJsonPath)) { + return new SecurityConfig().setData(securityJsonIs); + } catch (IOException e) { /* Fall through */ } + } + return new SecurityConfig(); + } + + @Override + protected void getConf(SolrQueryResponse rsp, String key) { + SecurityConfig props = getSecurityConfig(false); + Object o = props.getData().get(key); + if (o == null) { + rsp.add(CommandOperation.ERR_MSGS, Collections.singletonList("No " + key + " configured")); + } else { + rsp.add(key+".enabled", getPlugin(key)!=null); + rsp.add(key, o); + } + } + + @Override + protected boolean persistConf(SecurityConfig securityConfig) throws IOException { + if (securityConfig == null || securityConfig.getData().isEmpty()) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed persisting security.json to SOLR_HOME. Object was empty."); + } + try(OutputStream securityJsonOs = Files.newOutputStream(securityJsonPath)) { + securityJsonOs.write(Utils.toJSON(securityConfig.getData())); + log.debug("Persisted security.json to {}", securityJsonPath); + return true; + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Failed persisting security.json to " + securityJsonPath, e); + } + } + + @Override + public String getDescription() { + return "Edit or read security configuration locally in SOLR_HOME"; + } + + @Override + protected void securityConfEdited() { + // Need to call explicitly since we will not get notified of changes to local security.json + cores.securityNodeChanged(); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerZk.java b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerZk.java new file mode 100644 index 00000000000..8323b8a1315 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/handler/admin/SecurityConfHandlerZk.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.Collections; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.util.CommandOperation; +import org.apache.zookeeper.KeeperException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_SECURITY_CONF_PATH; + +/** + * Security Configuration Handler which works with Zookeeper + */ +public class SecurityConfHandlerZk extends SecurityConfHandler { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public SecurityConfHandlerZk(CoreContainer coreContainer) { + super(coreContainer); + } + + /** + * Fetches security props from Zookeeper and adds version + * @param getFresh refresh from ZK + * @return SecurityConfig whose data property either contains security.json, or an empty map if not found + */ + @Override + public SecurityConfig getSecurityConfig(boolean getFresh) { + ZkStateReader.ConfigData configDataFromZk = cores.getZkController().getZkStateReader().getSecurityProps(getFresh); + return configDataFromZk == null ? + new SecurityConfig() : + new SecurityConfig().setData(configDataFromZk.data).setVersion(configDataFromZk.version); + } + + @Override + protected void getConf(SolrQueryResponse rsp, String key) { + ZkStateReader.ConfigData map = cores.getZkController().getZkStateReader().getSecurityProps(false); + Object o = map == null ? null : map.data.get(key); + if (o == null) { + rsp.add(CommandOperation.ERR_MSGS, Collections.singletonList("No " + key + " configured")); + } else { + rsp.add(key+".enabled", getPlugin(key)!=null); + rsp.add(key, o); + } + } + + @Override + protected boolean persistConf(SecurityConfig securityConfig) throws IOException { + try { + cores.getZkController().getZkClient().setData(SOLR_SECURITY_CONF_PATH, + Utils.toJSON(securityConfig.getData()), + securityConfig.getVersion(), true); + log.debug("Persisted security.json to {}", SOLR_SECURITY_CONF_PATH); + return true; + } catch (KeeperException.BadVersionException bdve){ + log.warn("Failed persisting security.json to {}", SOLR_SECURITY_CONF_PATH, bdve); + return false; + } catch (Exception e) { + throw new SolrException(SERVER_ERROR, "Unable to persist security.json", e); + } + } + + @Override + public String getDescription() { + return "Edit or read security configuration from Zookeeper"; + } + +} diff --git a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java index 49c02d72e0c..29a887b9488 100644 --- a/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java +++ b/solr/core/src/java/org/apache/solr/security/BasicAuthPlugin.java @@ -43,12 +43,12 @@ import org.slf4j.LoggerFactory; public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEditablePlugin { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private AuthenticationProvider zkAuthentication; + private AuthenticationProvider authenticationProvider; private final static ThreadLocal
authHeader = new ThreadLocal<>(); private boolean blockUnknown = false; public boolean authenticate(String username, String pwd) { - return zkAuthentication.authenticate(username, pwd); + return authenticationProvider.authenticate(username, pwd); } @Override @@ -61,7 +61,7 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita log.error(e.getMessage()); } } - zkAuthentication = getAuthenticationProvider(pluginConfig); + authenticationProvider = getAuthenticationProvider(pluginConfig); } @Override @@ -79,8 +79,8 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita } } if (!CommandOperation.captureErrors(commands).isEmpty()) return null; - if (zkAuthentication instanceof ConfigEditablePlugin) { - ConfigEditablePlugin editablePlugin = (ConfigEditablePlugin) zkAuthentication; + if (authenticationProvider instanceof ConfigEditablePlugin) { + ConfigEditablePlugin editablePlugin = (ConfigEditablePlugin) authenticationProvider; return editablePlugin.edit(latestConf, commands); } throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "This cannot be edited"); @@ -93,7 +93,7 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita } private void authenticationFailure(HttpServletResponse response, String message) throws IOException { - for (Map.Entry entry : zkAuthentication.getPromptHeaders().entrySet()) { + for (Map.Entry entry : authenticationProvider.getPromptHeaders().entrySet()) { response.setHeader(entry.getKey(), entry.getValue()); } response.sendError(401, message); @@ -143,7 +143,7 @@ public class BasicAuthPlugin extends AuthenticationPlugin implements ConfigEdita if (blockUnknown) { authenticationFailure(response, "require authentication"); } else { - request.setAttribute(AuthenticationPlugin.class.getName(), zkAuthentication.getPromptHeaders()); + request.setAttribute(AuthenticationPlugin.class.getName(), authenticationProvider.getPromptHeaders()); filterChain.doFilter(request, response); return true; } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerLocalForTesting.java b/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerLocalForTesting.java new file mode 100644 index 00000000000..92a18b1d7dc --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerLocalForTesting.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin; + +import java.io.IOException; + +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrResourceLoader; + +/** + * Wrapper for use in tests + */ +public class SecurityConfHandlerLocalForTesting extends SecurityConfHandlerLocal { + + public SecurityConfHandlerLocalForTesting(CoreContainer coreContainer) { + super(coreContainer); + } + + public boolean persistConf(SecurityConfig securityConfig) throws IOException { + // Set JSON_PATH again since the test may have + securityJsonPath = SolrResourceLoader.locateSolrHome().resolve("security.json"); + return super.persistConf(securityConfig); + } + + public void securityConfEdited() { + super.securityConfEdited(); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java index 5e3d4071d24..54c8587bfe7 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/SecurityConfHandlerTest.java @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.cloud.ZkStateReader.ConfigData; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.ContentStreamBase; import org.apache.solr.common.util.Utils; @@ -34,6 +33,7 @@ import org.apache.solr.security.RuleBasedAuthorizationPlugin; import org.apache.solr.util.CommandOperation; import static org.apache.solr.common.util.Utils.makeMap; +import static org.apache.solr.handler.admin.SecurityConfHandler.SecurityConfig; public class SecurityConfHandlerTest extends SolrTestCaseJ4 { @@ -51,8 +51,8 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { handler.handleRequestBody(req,new SolrQueryResponse()); BasicAuthPlugin basicAuth = new BasicAuthPlugin(); - ConfigData securityCfg = (ConfigData) handler.m.get("/security.json"); - basicAuth.init((Map) securityCfg.data.get("authentication")); + SecurityConfig securityCfg = handler.m.get("/security.json"); + basicAuth.init((Map) securityCfg.getData().get("authentication")); assertTrue(basicAuth.authenticate("tom", "TomIsUberCool")); command = "{\n" + @@ -62,9 +62,9 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { o = new ContentStreamBase.ByteArrayStream(command.getBytes(StandardCharsets.UTF_8),""); req.setContentStreams(Collections.singletonList(o)); handler.handleRequestBody(req,new SolrQueryResponse()); - securityCfg = (ConfigData) handler.m.get("/security.json"); - assertEquals(3, securityCfg.version); - Map result = (Map) securityCfg.data.get("authentication"); + securityCfg = handler.m.get("/security.json"); + assertEquals(3, securityCfg.getVersion()); + Map result = (Map) securityCfg.getData().get("authentication"); result = (Map) result.get("credentials"); assertTrue(result.isEmpty()); @@ -86,7 +86,7 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { SolrQueryResponse rsp = new SolrQueryResponse(); handler.handleRequestBody(req, rsp); assertNull(rsp.getValues().get(CommandOperation.ERR_MSGS)); - Map authzconf = (Map) ((ConfigData) handler.m.get("/security.json")).data.get("authorization"); + Map authzconf = (Map) handler.m.get("/security.json").getData().get("authorization"); Map userRoles = (Map) authzconf.get("user-role"); List tomRoles = (List) userRoles.get("tom"); assertTrue(tomRoles.contains("admin")); @@ -108,7 +108,7 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { req.setContentStreams(Collections.singletonList(o)); rsp = new SolrQueryResponse(); handler.handleRequestBody(req, rsp); - authzconf = (Map) ((ConfigData) handler.m.get("/security.json")).data.get("authorization"); + authzconf = (Map) handler.m.get("/security.json").getData().get("authorization"); permissions = (List) authzconf.get("permissions"); Map p = permissions.get(1); @@ -128,7 +128,7 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { req.setContentStreams(Collections.singletonList(o)); rsp = new SolrQueryResponse(); handler.handleRequestBody(req, rsp); - authzconf = (Map) ((ConfigData) handler.m.get("/security.json")).data.get("authorization"); + authzconf = (Map) handler.m.get("/security.json").getData().get("authorization"); permissions = (List) authzconf.get("permissions"); p = permissions.get(0); @@ -151,7 +151,7 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { rsp = new SolrQueryResponse(); handler.handleRequestBody(req, rsp); assertNull(rsp.getValues().get(CommandOperation.ERR_MSGS)); - authzconf = (Map) ((ConfigData) handler.m.get("/security.json")).data.get("authorization"); + authzconf = (Map) handler.m.get("/security.json").getData().get("authorization"); userRoles = (Map) authzconf.get("user-role"); assertEquals(0, userRoles.size()); permissions = (List) authzconf.get("permissions"); @@ -178,25 +178,26 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { public static class MockSecurityHandler extends SecurityConfHandler { - private Map m; + private Map m; final BasicAuthPlugin basicAuthPlugin = new BasicAuthPlugin(); final RuleBasedAuthorizationPlugin rulesBasedAuthorizationPlugin = new RuleBasedAuthorizationPlugin(); - public MockSecurityHandler() { + public MockSecurityHandler() { super(null); m = new HashMap<>(); - ConfigData data = new ConfigData(makeMap("authentication", makeMap("class", "solr."+ BasicAuthPlugin.class.getSimpleName())), 1); - data.data.put("authorization", makeMap("class", "solr."+RuleBasedAuthorizationPlugin.class.getSimpleName())); - m.put("/security.json", data); - + SecurityConfig sp = new SecurityConfig(); + sp.setData(makeMap("authentication", makeMap("class", "solr."+ BasicAuthPlugin.class.getSimpleName()))); + sp.setVersion(1); + sp.getData().put("authorization", makeMap("class", "solr."+RuleBasedAuthorizationPlugin.class.getSimpleName())); + m.put("/security.json", sp); basicAuthPlugin.init(new HashMap<>()); rulesBasedAuthorizationPlugin.init(new HashMap<>()); } - public Map getM() { + public Map getM() { return m; } @@ -212,24 +213,25 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { } @Override - ConfigData getSecurityProps(boolean getFresh) { - return (ConfigData) m.get("/security.json"); + protected void getConf(SolrQueryResponse rsp, String key) { + // NOP } @Override - boolean persistConf(String key, byte[] buf, int version) { - Object data = m.get(key); - if (data instanceof ConfigData) { - ConfigData configData = (ConfigData) data; - if (configData.version == version) { - ConfigData result = new ConfigData((Map) Utils.fromJSON(buf), version + 1); - m.put(key, result); - return true; - } else { - return false; - } + public SecurityConfig getSecurityConfig(boolean getFresh) { + return m.get("/security.json"); + } + + @Override + protected boolean persistConf(SecurityConfig props) { + SecurityConfig fromMap = m.get("/security.json"); + if (fromMap.getVersion() == props.getVersion()) { + props.setVersion(props.getVersion()+1); + m.put("/security.json", props); + return true; + } else { + return false; } - throw new RuntimeException(); } @@ -254,7 +256,7 @@ public class SecurityConfHandlerTest extends SolrTestCaseJ4 { req.setContentStreams(Collections.singletonList(o)); SolrQueryResponse rsp = new SolrQueryResponse(); handleRequestBody(req, rsp); - Map data = ((ConfigData) m.get("/security.json")).data; + Map data = m.get("/security.json").getData(); ((Map)data.get("authentication")).remove(""); ((Map)data.get("authorization")).remove(""); return Utils.toJSONString (data); diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java index 6967b279d44..4134bf28eeb 100644 --- a/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthIntegrationTest.java @@ -292,11 +292,11 @@ public class BasicAuthIntegrationTest extends SolrCloudTestCase { return l.isEmpty() ? null : l.get(0); } - static final Predicate NOT_NULL_PREDICATE = o -> o != null; + protected static final Predicate NOT_NULL_PREDICATE = o -> o != null; //the password is 'SolrRocks' //this could be generated everytime. But , then we will not know if there is any regression - private static final String STD_CONF = "{\n" + + protected static final String STD_CONF = "{\n" + " 'authentication':{\n" + " 'class':'solr.BasicAuthPlugin',\n" + " 'credentials':{'solr':'orwp2Ghgj39lmnrZOTm7Qtre1VqHFDfwAEzr0ApbN3Y= Ju5osoAqOX8iafhWpPP01E5P+sg8tK8tHON7rCYZRRw='}},\n" + diff --git a/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java b/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java new file mode 100644 index 00000000000..829ce9c3590 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/security/BasicAuthStandaloneTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.security; + +import java.lang.invoke.MethodHandles; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.Properties; + +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.message.AbstractHttpMessage; +import org.apache.http.message.BasicHeader; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.embedded.JettySolrRunner; +import org.apache.solr.client.solrj.impl.HttpClientUtil; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.Base64; +import org.apache.solr.common.util.ContentStreamBase; +import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.admin.SecurityConfHandler; +import org.apache.solr.handler.admin.SecurityConfHandlerLocalForTesting; +import org.apache.solr.util.AbstractSolrTestCase; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.solr.security.BasicAuthIntegrationTest.NOT_NULL_PREDICATE; +import static org.apache.solr.security.BasicAuthIntegrationTest.STD_CONF; +import static org.apache.solr.security.BasicAuthIntegrationTest.verifySecurityStatus; + +public class BasicAuthStandaloneTest extends AbstractSolrTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private Path ROOT_DIR = Paths.get(getSolrHome()); + private Path CONF_DIR = ROOT_DIR.resolve("configsets").resolve("configset-2").resolve("conf"); + + SecurityConfHandlerLocalForTesting securityConfHandler; + SolrInstance instance = null; + JettySolrRunner jetty; + + @Before + @Override + public void setUp() throws Exception + { + super.setUp(); + instance = new SolrInstance("inst", null); + instance.setUp(); + System.setProperty("solr.solr.home", instance.getHomeDir().toString()); + jetty = createJetty(instance); + initCore("solrconfig.xml", "schema.xml", instance.getHomeDir().toString()); + securityConfHandler = new SecurityConfHandlerLocalForTesting(jetty.getCoreContainer()); + } + + @Override + @After + public void tearDown() throws Exception { + jetty.stop(); + super.tearDown(); + } + + @Test + public void testBasicAuth() throws Exception { + + String authcPrefix = "/admin/authentication"; + + HttpClient cl = null; + HttpSolrClient httpSolrClient = null; + try { + cl = HttpClientUtil.createClient(null); + String baseUrl = buildUrl(jetty.getLocalPort(), "/solr"); + httpSolrClient = getHttpSolrClient(baseUrl); + + verifySecurityStatus(cl, baseUrl + authcPrefix, "/errorMessages", null, 20); + + // Write security.json locally. Should cause security to be initialized + securityConfHandler.persistConf(new SecurityConfHandler.SecurityConfig() + .setData(Utils.fromJSONString(STD_CONF.replaceAll("'", "\"")))); + securityConfHandler.securityConfEdited(); + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/class", "solr.BasicAuthPlugin", 20); + + String command = "{\n" + + "'set-user': {'harry':'HarryIsCool'}\n" + + "}"; + + GenericSolrRequest genericReq = new GenericSolrRequest(SolrRequest.METHOD.POST, authcPrefix, new ModifiableSolrParams()); + genericReq.setContentStreams(Collections.singletonList(new ContentStreamBase.ByteArrayStream(command.getBytes(UTF_8), ""))); + + HttpSolrClient finalHttpSolrClient = httpSolrClient; + HttpSolrClient.RemoteSolrException exp = expectThrows(HttpSolrClient.RemoteSolrException.class, () -> { + finalHttpSolrClient.request(genericReq); + }); + assertEquals(401, exp.code()); + + command = "{\n" + + "'set-user': {'harry':'HarryIsUberCool'}\n" + + "}"; + + HttpPost httpPost = new HttpPost(baseUrl + authcPrefix); + setBasicAuthHeader(httpPost, "solr", "SolrRocks"); + httpPost.setEntity(new ByteArrayEntity(command.getBytes(UTF_8))); + httpPost.addHeader("Content-Type", "application/json; charset=UTF-8"); + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication.enabled", "true", 20); + HttpResponse r = cl.execute(httpPost); + int statusCode = r.getStatusLine().getStatusCode(); + Utils.consumeFully(r.getEntity()); + assertEquals("proper_cred sent, but access denied", 200, statusCode); + + verifySecurityStatus(cl, baseUrl + authcPrefix, "authentication/credentials/harry", NOT_NULL_PREDICATE, 20); + + // Read file from SOLR_HOME and verify that it contains our new user + assertTrue(new String(Utils.toJSON(securityConfHandler.getSecurityConfig(false).getData()), + Charset.forName("UTF-8")).contains("harry")); + } finally { + if (cl != null) { + HttpClientUtil.close(cl); + httpSolrClient.close(); + } + } + } + + public static void setBasicAuthHeader(AbstractHttpMessage httpMsg, String user, String pwd) { + String userPass = user + ":" + pwd; + String encoded = Base64.byteArrayToBase64(userPass.getBytes(UTF_8)); + httpMsg.setHeader(new BasicHeader("Authorization", "Basic " + encoded)); + log.info("Added Basic Auth security Header {}",encoded ); + } + + private JettySolrRunner createJetty(SolrInstance instance) throws Exception { + Properties nodeProperties = new Properties(); + nodeProperties.setProperty("solr.data.dir", instance.getDataDir().toString()); + JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir().toString(), nodeProperties, buildJettyConfig("/solr")); + jetty.start(); + return jetty; + } + + + private class SolrInstance { + String name; + Integer port; + Path homeDir; + Path confDir; + Path dataDir; + + /** + * if masterPort is null, this instance is a master -- otherwise this instance is a slave, and assumes the master is + * on localhost at the specified port. + */ + public SolrInstance(String name, Integer port) { + this.name = name; + this.port = port; + } + + public Path getHomeDir() { + return homeDir; + } + + public Path getSchemaFile() { + return CONF_DIR.resolve("schema.xml"); + } + + public Path getConfDir() { + return confDir; + } + + public Path getDataDir() { + return dataDir; + } + + public Path getSolrConfigFile() { + return CONF_DIR.resolve("solrconfig.xml"); + } + + public Path getSolrXmlFile() { + return ROOT_DIR.resolve("solr.xml"); + } + + + public void setUp() throws Exception { + homeDir = createTempDir(name).toAbsolutePath(); + dataDir = homeDir.resolve("collection1").resolve("data"); + confDir = homeDir.resolve("collection1").resolve("conf"); + + Files.createDirectories(homeDir); + Files.createDirectories(dataDir); + Files.createDirectories(confDir); + + Files.copy(getSolrXmlFile(), homeDir.resolve("solr.xml")); + Files.copy(getSolrConfigFile(), confDir.resolve("solrconfig.xml")); + Files.copy(getSchemaFile(), confDir.resolve("schema.xml")); + + Files.createFile(homeDir.resolve("collection1").resolve("core.properties")); + } + + } +} From 62bc90d7d2d586fd587c7a133fff83e535892764 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Wed, 26 Oct 2016 09:49:47 +0530 Subject: [PATCH 3/6] SOLR-5245: Add a test to ensure that election contexts are keyed off both collection name and coreNodeName so that killing a shard in one collection does not result in leader election in a different collection. --- solr/CHANGES.txt | 4 + .../cloud/LeaderElectionContextKeyTest.java | 114 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 62c9d4afff7..506ad9aa63a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -354,6 +354,10 @@ Other Changes * SOLR-4531: Add tests to ensure that recovery does not fail on corrupted tlogs. (Simon Scofield, Cao Manh Dat via shalin) +* SOLR-5245: Add a test to ensure that election contexts are keyed off both collection name and coreNodeName + so that killing a shard in one collection does not result in leader election in a different collection. + See SOLR-5243 for the related bug. (Cao Manh Dat via shalin) + ================== 6.2.1 ================== Bug Fixes diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java new file mode 100644 index 00000000000..728ea1bb867 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.cloud; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.zookeeper.KeeperException; +import org.hamcrest.CoreMatchers; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LeaderElectionContextKeyTest extends SolrCloudTestCase { + + private static final String TEST_COLLECTION_1 = "testCollection1"; + private static final String TEST_COLLECTION_2 = "testCollection2"; + + @BeforeClass + public static void setupCluster() throws Exception { + configureCluster(1) + .addConfig("config", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf")) + .configure(); + + CollectionAdminRequest + .createCollection("testCollection1", "config", 2, 1) + .setMaxShardsPerNode(1000) + .process(cluster.getSolrClient()); + CollectionAdminRequest + .createCollection("testCollection2", "config", 2, 1) + .setMaxShardsPerNode(1000) + .process(cluster.getSolrClient()); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("testCollection1", cluster.getSolrClient().getZkStateReader(), + false, true, 30); + AbstractDistribZkTestBase.waitForRecoveriesToFinish("testCollection2", cluster.getSolrClient().getZkStateReader(), + false, true, 30); + } + + @Test + public void test() throws KeeperException, InterruptedException, IOException, SolrServerException { + ZkStateReader stateReader = cluster.getSolrClient().getZkStateReader(); + stateReader.forceUpdateCollection(TEST_COLLECTION_1); + List replicasOfCollection1 = stateReader.getClusterState().getCollection(TEST_COLLECTION_1).getReplicas(); + List replicasOfCollection2 = stateReader.getClusterState().getCollection(TEST_COLLECTION_2).getReplicas(); + Replica replica = findLeaderReplicaWithDuplicatedName(replicasOfCollection1, replicasOfCollection2); + assertNotNull(replica); + + SolrClient shardLeaderClient = new HttpSolrClient.Builder(replica.get("base_url").toString()).build(); + try { + assertEquals(1L, getElectionNodes(TEST_COLLECTION_1, "shard1", stateReader.getZkClient()).size()); + List collection2Shard1Nodes = getElectionNodes(TEST_COLLECTION_2, "shard1", stateReader.getZkClient()); + List collection2Shard2Nodes = getElectionNodes(TEST_COLLECTION_2, "shard2", stateReader.getZkClient()); + CoreAdminRequest.unloadCore(replica.getCoreName(), shardLeaderClient); + // Waiting for leader election being kicked off + long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(60, TimeUnit.SECONDS); + boolean found = false; + while (System.nanoTime() < timeout) { + try { + found = getElectionNodes(TEST_COLLECTION_1, "shard1", stateReader.getZkClient()).size() == 0; + break; + } catch (KeeperException.NoNodeException nne) { + // ignore + } + } + assertTrue(found); + // There are no leader election was kicked off on testCollection2 + assertThat(collection2Shard1Nodes, CoreMatchers.is(getElectionNodes(TEST_COLLECTION_2, "shard1", stateReader.getZkClient()))); + assertThat(collection2Shard2Nodes, CoreMatchers.is(getElectionNodes(TEST_COLLECTION_2, "shard2", stateReader.getZkClient()))); + } finally { + shardLeaderClient.close(); + } + } + + private Replica findLeaderReplicaWithDuplicatedName(List replicas1, List replicas2) { + for (Replica replica1 : replicas1) { + if (!replica1.containsKey("leader")) continue; + for (Replica replica2 : replicas2) { + if (replica1.getName().equals(replica2.getName()) + && replica1.get("base_url").equals(replica2.get("base_url")) + && replica2.containsKey("leader")) { + return replica1; + } + } + } + return null; + } + + private List getElectionNodes(String collection, String shard, SolrZkClient client) throws KeeperException, InterruptedException { + return client.getChildren("/collections/"+collection+"/leader_elect/"+shard+LeaderElector.ELECTION_NODE, null, true); + } +} From b8d9647307c5559706aeec3aad32c2e416188979 Mon Sep 17 00:00:00 2001 From: Shalin Shekhar Mangar Date: Wed, 26 Oct 2016 11:02:10 +0530 Subject: [PATCH 4/6] SOLR-2039: Multivalued fields with dynamic names does not work properly with DIH --- solr/CHANGES.txt | 3 +++ .../solr/handler/dataimport/DocBuilder.java | 2 ++ .../handler/dataimport/TestDocBuilder2.java | 23 +++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 506ad9aa63a..9dfed73de81 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -220,6 +220,9 @@ Bug Fixes * SOLR-9536: OldBackupDirectory timestamp field needs to be initialized to avoid NPE. (Hrishikesh Gadre, hossman via Mark Miller) + +* SOLR-2039: Multivalued fields with dynamic names does not work properly with DIH. + (K A, ruslan.shv, Cao Manh Dat via shalin) Optimizations ---------------------- diff --git a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java index f9ccfb6fc0c..c80d2751398 100644 --- a/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java +++ b/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java @@ -501,7 +501,9 @@ public class DocBuilder { doc.addChildDocument(childDoc); } else { handleSpecialCommands(arow, doc); + vr.addNamespace(epw.getEntity().getName(), arow); addFields(epw.getEntity(), doc, arow, vr); + vr.removeNamespace(epw.getEntity().getName()); } } if (epw.getEntity().getChildren() != null) { diff --git a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java index 40ee2b9d826..32a0d4ad3de 100644 --- a/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java +++ b/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java @@ -115,6 +115,20 @@ public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase { assertQ(req("desc:ApacheSolr"), "//*[@numFound='1']"); } + @Test + @SuppressWarnings("unchecked") + public void testDynamicFieldNames() throws Exception { + List rows = new ArrayList(); + rows.add(createMap("mypk", "101", "text", "ApacheSolr")); + MockDataSource.setIterator("select * from x", rows.iterator()); + + LocalSolrQueryRequest request = lrf.makeRequest("command", "full-import", + "debug", "on", "clean", "true", "commit", "true", + "dataConfig", dataConfigWithDynamicFieldNames); + h.query("/dataimport", request); + assertQ(req("id:101"), "//*[@numFound='1']", "//*[@name='101_s']"); + } + @Test @SuppressWarnings("unchecked") public void testRequestParamsAsFieldName() throws Exception { @@ -398,6 +412,15 @@ public class TestDocBuilder2 extends AbstractDataImportHandlerTestCase { " \n" + ""; + private final String dataConfigWithDynamicFieldNames = "\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + ""; + private final String dataConfigFileList = "\n" + "\t\n" + "\t\t Date: Wed, 26 Oct 2016 11:28:53 +0530 Subject: [PATCH 5/6] SOLR-2212: Add a factory class corresponding to Lucene's NoMergePolicy --- solr/CHANGES.txt | 2 ++ .../solr/index/NoMergePolicyFactory.java | 34 +++++++++++++++++++ .../conf/solrconfig-nomergepolicyfactory.xml | 32 +++++++++++++++++ .../solr/core/TestMergePolicyConfig.java | 20 +++++++++++ 4 files changed, 88 insertions(+) create mode 100644 solr/core/src/java/org/apache/solr/index/NoMergePolicyFactory.java create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 9dfed73de81..ba680a13e55 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -164,6 +164,8 @@ New Features SOLR_HOME on every node. Editing config through API is supported but affects only that one node. (janhoy) +* SOLR-2212: Add a factory class corresponding to Lucene's NoMergePolicy. (Lance Norskog, Cao Manh Dat via shalin) + Bug Fixes ---------------------- diff --git a/solr/core/src/java/org/apache/solr/index/NoMergePolicyFactory.java b/solr/core/src/java/org/apache/solr/index/NoMergePolicyFactory.java new file mode 100644 index 00000000000..66fa18e9746 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/index/NoMergePolicyFactory.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.index; + +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.solr.core.SolrResourceLoader; +import org.apache.solr.schema.IndexSchema; + +public class NoMergePolicyFactory extends SimpleMergePolicyFactory { + public NoMergePolicyFactory(SolrResourceLoader resourceLoader, MergePolicyFactoryArgs args, IndexSchema schema) { + super(resourceLoader, args, schema); + } + + @Override + protected MergePolicy getMergePolicyInstance() { + return NoMergePolicy.INSTANCE; + } +} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml new file mode 100644 index 00000000000..a9e38016d20 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml @@ -0,0 +1,32 @@ + + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + ${useCompoundFile:false} + + + + + + diff --git a/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java b/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java index fd13a8e8cc4..f8e232a8968 100644 --- a/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestMergePolicyConfig.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.LogByteSizeMergePolicy; import org.apache.lucene.index.LogDocMergePolicy; import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.TieredMergePolicy; import org.apache.solr.SolrTestCaseJ4; @@ -128,6 +129,25 @@ public class TestMergePolicyConfig extends SolrTestCaseJ4 { assertCompoundSegments(h.getCore(), false); } + public void testNoMergePolicyFactoryConfig() throws Exception { + initCore("solrconfig-nomergepolicyfactory.xml","schema-minimal.xml"); + IndexWriterConfig iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore()); + NoMergePolicy mergePolicy = assertAndCast(NoMergePolicy.class, + iwc.getMergePolicy()); + + assertCommitSomeNewDocs(); + + assertCommitSomeNewDocs(); + assertNumSegments(h.getCore(), 2); + + assertU(optimize()); + assertNumSegments(h.getCore(), 2); + deleteCore(); + initCore("solrconfig-nomergepolicyfactory.xml","schema-minimal.xml"); + iwc = solrConfig.indexConfig.toIndexWriterConfig(h.getCore()); + assertEquals(mergePolicy, iwc.getMergePolicy()); + } + public void testLogMergePolicyConfig() throws Exception { final Class mpClass = random().nextBoolean() From b69c5d9f27aea722401674ed72b876da4dbdb7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 26 Oct 2016 09:22:55 +0200 Subject: [PATCH 6/6] SOLR-9481: Add info-level log "Initializing authentication plugin: " Move Sha256AuthProv warning "No users configured yet" to debug level, as this is quite normal --- solr/core/src/java/org/apache/solr/core/CoreContainer.java | 1 + .../org/apache/solr/security/Sha256AuthenticationProvider.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index e3d577efdf0..e641b2c5f9c 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -308,6 +308,7 @@ public class CoreContainer { // Initialize the plugin if (pluginClassName != null) { + log.info("Initializing authentication plugin: " + pluginClassName); authenticationPlugin = new SecurityPluginHolder<>(readVersion(authenticationConfig), getResourceLoader().newInstance(pluginClassName, AuthenticationPlugin.class, diff --git a/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java b/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java index 545792fe7e1..69664fda46b 100644 --- a/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java +++ b/solr/core/src/java/org/apache/solr/security/Sha256AuthenticationProvider.java @@ -64,7 +64,7 @@ public class Sha256AuthenticationProvider implements ConfigEditablePlugin, Basi credentials = new LinkedHashMap<>(); Map users = (Map) pluginConfig.get("credentials"); if (users == null) { - log.warn("No users configured yet"); + log.debug("No users configured yet"); return; } for (Map.Entry e : users.entrySet()) {