mirror of https://github.com/apache/lucene.git
SOLR-9324: Support Secure Impersonation / Proxy User for solr authentication
This commit is contained in:
parent
b3505298a5
commit
e50858c314
|
@ -111,6 +111,9 @@ New Features
|
|||
* SOLR-9279: New boolean comparison function queries comparing numeric arguments: gt, gte, lt, lte, eq
|
||||
(Doug Turnbull, David Smiley)
|
||||
|
||||
* SOLR-9346: Support Secure Impersonation / Proxy User for solr authentication
|
||||
(Gregory Chanan)
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
|
@ -18,6 +18,7 @@ package org.apache.solr.security;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.util.Enumeration;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
|
@ -36,8 +37,11 @@ import org.apache.curator.framework.CuratorFrameworkFactory;
|
|||
import org.apache.curator.framework.api.ACLProvider;
|
||||
import org.apache.curator.retry.ExponentialBackoffRetry;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
|
||||
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationFilter;
|
||||
import org.apache.hadoop.security.token.delegation.web.HttpUserGroupInformation;
|
||||
import org.apache.solr.common.cloud.SecurityAwareZkACLProvider;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.common.cloud.ZkACLProvider;
|
||||
|
@ -62,6 +66,27 @@ public class DelegationTokenKerberosFilter extends DelegationTokenAuthentication
|
|||
super.init(conf);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the ProxyUser Configuration. FilterConfig properties beginning with
|
||||
* "solr.impersonator.user.name" will be added to the configuration.
|
||||
*/
|
||||
@Override
|
||||
protected Configuration getProxyuserConfiguration(FilterConfig filterConf)
|
||||
throws ServletException {
|
||||
Configuration conf = new Configuration(false);
|
||||
|
||||
Enumeration<?> names = filterConf.getInitParameterNames();
|
||||
while (names.hasMoreElements()) {
|
||||
String name = (String) names.nextElement();
|
||||
if (name.startsWith(KerberosPlugin.IMPERSONATOR_PREFIX)) {
|
||||
String value = filterConf.getInitParameter(name);
|
||||
conf.set(PROXYUSER_PREFIX + "." + name.substring(KerberosPlugin.IMPERSONATOR_PREFIX.length()), value);
|
||||
conf.set(name, value);
|
||||
}
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void doFilter(ServletRequest request, ServletResponse response,
|
||||
FilterChain filterChain) throws IOException, ServletException {
|
||||
|
@ -76,7 +101,26 @@ public class DelegationTokenKerberosFilter extends DelegationTokenAuthentication
|
|||
return nonNullQueryString;
|
||||
}
|
||||
};
|
||||
super.doFilter(requestNonNullQueryString, response, filterChain);
|
||||
|
||||
// include Impersonator User Name in case someone (e.g. logger) wants it
|
||||
FilterChain filterChainWrapper = new FilterChain() {
|
||||
@Override
|
||||
public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse)
|
||||
throws IOException, ServletException {
|
||||
HttpServletRequest httpRequest = (HttpServletRequest) servletRequest;
|
||||
|
||||
UserGroupInformation ugi = HttpUserGroupInformation.get();
|
||||
if (ugi != null && ugi.getAuthenticationMethod() == UserGroupInformation.AuthenticationMethod.PROXY) {
|
||||
UserGroupInformation realUserUgi = ugi.getRealUser();
|
||||
if (realUserUgi != null) {
|
||||
httpRequest.setAttribute(KerberosPlugin.IMPERSONATOR_USER_NAME, realUserUgi.getShortUserName());
|
||||
}
|
||||
}
|
||||
filterChain.doFilter(servletRequest, servletResponse);
|
||||
}
|
||||
};
|
||||
|
||||
super.doFilter(requestNonNullQueryString, response, filterChainWrapper);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -69,7 +69,7 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
||||
Krb5HttpClientBuilder kerberosBuilder = new Krb5HttpClientBuilder();
|
||||
Filter kerberosFilter;
|
||||
private Filter kerberosFilter;
|
||||
|
||||
public static final String NAME_RULES_PARAM = "solr.kerberos.name.rules";
|
||||
public static final String COOKIE_DOMAIN_PARAM = "solr.kerberos.cookie.domain";
|
||||
|
@ -78,6 +78,7 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
public static final String KEYTAB_PARAM = "solr.kerberos.keytab";
|
||||
public static final String TOKEN_VALID_PARAM = "solr.kerberos.token.valid";
|
||||
public static final String COOKIE_PORT_AWARE_PARAM = "solr.kerberos.cookie.portaware";
|
||||
public static final String IMPERSONATOR_PREFIX = "solr.kerberos.impersonator.user.";
|
||||
public static final String DELEGATION_TOKEN_ENABLED = "solr.kerberos.delegation.token.enabled";
|
||||
public static final String DELEGATION_TOKEN_KIND = "solr.kerberos.delegation.token.kind";
|
||||
public static final String DELEGATION_TOKEN_VALIDITY = "solr.kerberos.delegation.token.validity";
|
||||
|
@ -86,7 +87,10 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
"solr.kerberos.delegation.token.signer.secret.provider.zookeper.path";
|
||||
public static final String DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH =
|
||||
"solr.kerberos.delegation.token.secret.manager.znode.working.path";
|
||||
|
||||
public static final String DELEGATION_TOKEN_TYPE_DEFAULT = "solr-dt";
|
||||
public static final String IMPERSONATOR_DO_AS_HTTP_PARAM = "doAs";
|
||||
public static final String IMPERSONATOR_USER_NAME = "solr.impersonator.user.name";
|
||||
|
||||
// filled in by Plugin/Filter
|
||||
static final String REQUEST_CONTINUES_ATTR =
|
||||
|
@ -94,10 +98,6 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
static final String DELEGATION_TOKEN_ZK_CLIENT =
|
||||
"solr.kerberos.delegation.token.zk.client";
|
||||
|
||||
// allows test to specify an alternate auth handler
|
||||
@VisibleForTesting
|
||||
public static final String AUTH_HANDLER_PARAM = "solr.kerberos.auth.handler";
|
||||
|
||||
private final CoreContainer coreContainer;
|
||||
|
||||
public KerberosPlugin(CoreContainer coreContainer) {
|
||||
|
@ -107,16 +107,24 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
@Override
|
||||
public void init(Map<String, Object> pluginConfig) {
|
||||
try {
|
||||
FilterConfig conf = getInitFilterConfig(pluginConfig, false);
|
||||
kerberosFilter.init(conf);
|
||||
} catch (ServletException e) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Error initializing kerberos authentication plugin: "+e);
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected FilterConfig getInitFilterConfig(Map<String, Object> pluginConfig, boolean skipKerberosChecking) {
|
||||
Map<String, String> params = new HashMap();
|
||||
putParam(params, "type", AUTH_HANDLER_PARAM, "kerberos");
|
||||
params.put("type", "kerberos");
|
||||
putParam(params, "kerberos.name.rules", NAME_RULES_PARAM, "DEFAULT");
|
||||
putParam(params, "token.valid", TOKEN_VALID_PARAM, "30");
|
||||
putParam(params, "cookie.path", COOKIE_PATH_PARAM, "/");
|
||||
if ("kerberos".equals(params.get("type"))) {
|
||||
if (!skipKerberosChecking) {
|
||||
putParam(params, "kerberos.principal", PRINCIPAL_PARAM, null);
|
||||
putParam(params, "kerberos.keytab", KEYTAB_PARAM, null);
|
||||
} else {
|
||||
// allow tests which specify AUTH_HANDLER_PARAM to avoid specifying kerberos principal/keytab
|
||||
putParamOptional(params, "kerberos.principal", PRINCIPAL_PARAM);
|
||||
putParamOptional(params, "kerberos.keytab", KEYTAB_PARAM);
|
||||
}
|
||||
|
@ -167,14 +175,25 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
params.put("cookie.domain", host + ":" + port);
|
||||
}
|
||||
|
||||
// check impersonator config
|
||||
for (Enumeration e = System.getProperties().propertyNames(); e.hasMoreElements();) {
|
||||
String key = e.nextElement().toString();
|
||||
if (key.startsWith(IMPERSONATOR_PREFIX)) {
|
||||
if (!delegationTokenEnabled) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR,
|
||||
"Impersonator configuration requires delegation tokens to be enabled: " + key);
|
||||
}
|
||||
params.put(key, System.getProperty(key));
|
||||
}
|
||||
}
|
||||
final ServletContext servletContext = new AttributeOnlyServletContext();
|
||||
if (controller != null) {
|
||||
servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient());
|
||||
}
|
||||
if (delegationTokenEnabled) {
|
||||
kerberosFilter = new DelegationTokenKerberosFilter();
|
||||
// pass an attribute-enabled context in order to pass the zkClient
|
||||
// and because the filter may pass a curator instance.
|
||||
if (controller != null) {
|
||||
servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient());
|
||||
}
|
||||
} else {
|
||||
kerberosFilter = new KerberosFilter();
|
||||
}
|
||||
|
@ -202,10 +221,7 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
}
|
||||
};
|
||||
|
||||
kerberosFilter.init(conf);
|
||||
} catch (ServletException e) {
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Error initializing kerberos authentication plugin: "+e);
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
|
||||
private void putParam(Map<String, String> params, String internalParamName, String externalParamName, String defaultValue) {
|
||||
|
@ -260,11 +276,16 @@ public class KerberosPlugin extends AuthenticationPlugin implements HttpClientBu
|
|||
return kerberosBuilder.getBuilder(builder);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
kerberosFilter.destroy();
|
||||
kerberosBuilder.close();
|
||||
}
|
||||
|
||||
protected Filter getKerberosFilter() { return kerberosFilter; }
|
||||
|
||||
protected void setKerberosFilter(Filter kerberosFilter) { this.kerberosFilter = kerberosFilter; }
|
||||
|
||||
protected static class AttributeOnlyServletContext implements ServletContext {
|
||||
private Map<String, Object> attributes = new HashMap<String, Object>();
|
||||
|
||||
|
|
|
@ -30,10 +30,10 @@ import org.apache.solr.common.SolrException.ErrorCode;
|
|||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import static org.apache.solr.security.HttpParamDelegationTokenAuthenticationHandler.USER_PARAM;
|
||||
import static org.apache.solr.security.HttpParamDelegationTokenPlugin.USER_PARAM;
|
||||
|
||||
import org.apache.http.HttpStatus;
|
||||
import org.apache.solr.security.HttpParamDelegationTokenAuthenticationHandler;
|
||||
import org.apache.solr.security.HttpParamDelegationTokenPlugin;
|
||||
import org.apache.solr.security.KerberosPlugin;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
|
@ -59,10 +59,8 @@ public class TestSolrCloudWithDelegationTokens extends SolrTestCaseJ4 {
|
|||
|
||||
@BeforeClass
|
||||
public static void startup() throws Exception {
|
||||
System.setProperty("authenticationPlugin", KerberosPlugin.class.getName());
|
||||
System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName());
|
||||
System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true");
|
||||
System.setProperty(KerberosPlugin.AUTH_HANDLER_PARAM,
|
||||
HttpParamDelegationTokenAuthenticationHandler.class.getName());
|
||||
System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1");
|
||||
|
||||
miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), buildJettyConfig("/solr"));
|
||||
|
@ -88,7 +86,6 @@ public class TestSolrCloudWithDelegationTokens extends SolrTestCaseJ4 {
|
|||
solrClientSecondary = null;
|
||||
System.clearProperty("authenticationPlugin");
|
||||
System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED);
|
||||
System.clearProperty(KerberosPlugin.AUTH_HANDLER_PARAM);
|
||||
System.clearProperty("solr.kerberos.cookie.domain");
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,340 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.cloud;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.io.File;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.solr.SolrTestCaseJ4;
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.handler.admin.CollectionsHandler;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.security.HttpParamDelegationTokenPlugin;
|
||||
import org.apache.solr.security.KerberosPlugin;
|
||||
import org.apache.solr.servlet.SolrRequestParsers;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
|
||||
import static org.apache.solr.security.HttpParamDelegationTokenPlugin.USER_PARAM;
|
||||
import static org.apache.solr.security.HttpParamDelegationTokenPlugin.REMOTE_HOST_PARAM;
|
||||
import static org.apache.solr.security.HttpParamDelegationTokenPlugin.REMOTE_ADDRESS_PARAM;
|
||||
|
||||
public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
|
||||
private static final int NUM_SERVERS = 2;
|
||||
private static MiniSolrCloudCluster miniCluster;
|
||||
private static SolrClient solrClient;
|
||||
|
||||
private static String getUsersFirstGroup() throws Exception {
|
||||
org.apache.hadoop.security.Groups hGroups =
|
||||
new org.apache.hadoop.security.Groups(new Configuration());
|
||||
List<String> g = hGroups.getGroups(System.getProperty("user.name"));
|
||||
return g.get(0);
|
||||
}
|
||||
|
||||
private static Map<String, String> getImpersonatorSettings() throws Exception {
|
||||
Map<String, String> filterProps = new TreeMap<String, String>();
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noGroups.hosts", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.groups", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.hosts", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "wrongHost.hosts", "1.1.1.1.1.1");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "wrongHost.groups", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noHosts.groups", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "localHostAnyGroup.groups", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "localHostAnyGroup.hosts", "127.0.0.1");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostUsersGroup.groups", getUsersFirstGroup());
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostUsersGroup.hosts", "*");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "bogusGroup.groups", "__some_bogus_group");
|
||||
filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "bogusGroup.hosts", "*");
|
||||
return filterProps;
|
||||
}
|
||||
|
||||
@BeforeClass
|
||||
public static void startup() throws Exception {
|
||||
System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName());
|
||||
System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true");
|
||||
|
||||
System.setProperty("solr.kerberos.cookie.domain", "127.0.0.1");
|
||||
Map<String, String> impSettings = getImpersonatorSettings();
|
||||
for (Map.Entry<String, String> entry : impSettings.entrySet()) {
|
||||
System.setProperty(entry.getKey(), entry.getValue());
|
||||
}
|
||||
System.setProperty("solr.test.sys.prop1", "propone");
|
||||
System.setProperty("solr.test.sys.prop2", "proptwo");
|
||||
|
||||
SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(true);
|
||||
String solrXml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML.replace("</solr>",
|
||||
" <str name=\"collectionsHandler\">" + ImpersonatorCollectionsHandler.class.getName() + "</str>\n" +
|
||||
"</solr>");
|
||||
|
||||
miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, createTempDir(), solrXml, buildJettyConfig("/solr"));
|
||||
JettySolrRunner runner = miniCluster.getJettySolrRunners().get(0);
|
||||
solrClient = new HttpSolrClient.Builder(runner.getBaseUrl().toString()).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that impersonator info is preserved in the request
|
||||
*/
|
||||
public static class ImpersonatorCollectionsHandler extends CollectionsHandler {
|
||||
public static AtomicBoolean called = new AtomicBoolean(false);
|
||||
|
||||
public ImpersonatorCollectionsHandler() { super(); }
|
||||
|
||||
public ImpersonatorCollectionsHandler(final CoreContainer coreContainer) {
|
||||
super(coreContainer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
|
||||
called.set(true);
|
||||
super.handleRequestBody(req, rsp);
|
||||
String doAs = req.getParams().get(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM);
|
||||
if (doAs != null) {
|
||||
HttpServletRequest httpRequest = (HttpServletRequest)req.getContext().get("httpRequest");
|
||||
assertNotNull(httpRequest);
|
||||
String user = (String)httpRequest.getAttribute(USER_PARAM);
|
||||
assertNotNull(user);
|
||||
assertEquals(user, httpRequest.getAttribute(KerberosPlugin.IMPERSONATOR_USER_NAME));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void shutdown() throws Exception {
|
||||
if (miniCluster != null) {
|
||||
miniCluster.shutdown();
|
||||
}
|
||||
miniCluster = null;
|
||||
solrClient.close();
|
||||
solrClient = null;
|
||||
System.clearProperty("authenticationPlugin");
|
||||
System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED);
|
||||
System.clearProperty("solr.kerberos.cookie.domain");
|
||||
Map<String, String> impSettings = getImpersonatorSettings();
|
||||
for (Map.Entry<String, String> entry : impSettings.entrySet()) {
|
||||
System.clearProperty(entry.getKey());
|
||||
}
|
||||
System.clearProperty("solr.test.sys.prop1");
|
||||
System.clearProperty("solr.test.sys.prop2");
|
||||
SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false);
|
||||
}
|
||||
|
||||
private void create1ShardCollection(String name, String config, MiniSolrCloudCluster solrCluster) throws Exception {
|
||||
CollectionAdminResponse response;
|
||||
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create() {
|
||||
@Override
|
||||
public SolrParams getParams() {
|
||||
ModifiableSolrParams msp = new ModifiableSolrParams(super.getParams());
|
||||
msp.set(USER_PARAM, "user");
|
||||
return msp;
|
||||
}
|
||||
};
|
||||
create.setConfigName(config);
|
||||
create.setCollectionName(name);
|
||||
create.setNumShards(1);
|
||||
create.setReplicationFactor(1);
|
||||
create.setMaxShardsPerNode(1);
|
||||
response = create.process(solrCluster.getSolrClient());
|
||||
|
||||
if (response.getStatus() != 0 || response.getErrorMessages() != null) {
|
||||
fail("Could not create collection. Response" + response.toString());
|
||||
}
|
||||
ZkStateReader zkStateReader = solrCluster.getSolrClient().getZkStateReader();
|
||||
AbstractDistribZkTestBase.waitForRecoveriesToFinish(name, zkStateReader, false, true, 100);
|
||||
}
|
||||
|
||||
private SolrRequest getProxyRequest(String user, String doAs) {
|
||||
return getProxyRequest(user, doAs, null);
|
||||
}
|
||||
|
||||
private SolrRequest getProxyRequest(String user, String doAs, String remoteHost) {
|
||||
return getProxyRequest(user, doAs, remoteHost, null);
|
||||
}
|
||||
|
||||
private SolrRequest getProxyRequest(String user, String doAs, String remoteHost, String remoteAddress) {
|
||||
return new CollectionAdminRequest.List() {
|
||||
@Override
|
||||
public SolrParams getParams() {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
|
||||
params.set(USER_PARAM, user);
|
||||
params.set(KerberosPlugin.IMPERSONATOR_DO_AS_HTTP_PARAM, doAs);
|
||||
if (remoteHost != null) params.set(REMOTE_HOST_PARAM, remoteHost);
|
||||
if (remoteAddress != null) params.set(REMOTE_ADDRESS_PARAM, remoteAddress);
|
||||
return params;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private String getExpectedGroupExMsg(String user, String doAs) {
|
||||
return "User: " + user + " is not allowed to impersonate " + doAs;
|
||||
}
|
||||
|
||||
private String getExpectedHostExMsg(String user) {
|
||||
return "Unauthorized connection for super-user: " + user;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyNoConfigGroups() throws Exception {
|
||||
try {
|
||||
solrClient.request(getProxyRequest("noGroups","bar"));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("noGroups", "bar")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyWrongHost() throws Exception {
|
||||
try {
|
||||
solrClient.request(getProxyRequest("wrongHost","bar"));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
assertTrue(ex.getMessage().contains(getExpectedHostExMsg("wrongHost")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyNoConfigHosts() throws Exception {
|
||||
try {
|
||||
solrClient.request(getProxyRequest("noHosts","bar"));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
// FixMe: this should return an exception about the host being invalid,
|
||||
// but a bug (HADOOP-11077) causes an NPE instead.
|
||||
//assertTrue(ex.getMessage().contains(getExpectedHostExMsg("noHosts")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyValidateAnyHostAnyUser() throws Exception {
|
||||
solrClient.request(getProxyRequest("anyHostAnyUser", "bar", null));
|
||||
assertTrue(ImpersonatorCollectionsHandler.called.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyInvalidProxyUser() throws Exception {
|
||||
try {
|
||||
// wrong direction, should fail
|
||||
solrClient.request(getProxyRequest("bar","anyHostAnyUser"));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bar", "anyHostAnyUser")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyValidateHost() throws Exception {
|
||||
solrClient.request(getProxyRequest("localHostAnyGroup", "bar"));
|
||||
assertTrue(ImpersonatorCollectionsHandler.called.get());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testProxyValidateGroup() throws Exception {
|
||||
solrClient.request(getProxyRequest("anyHostUsersGroup", System.getProperty("user.name"), null));
|
||||
assertTrue(ImpersonatorCollectionsHandler.called.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyUnknownRemote() throws Exception {
|
||||
try {
|
||||
// Use a reserved ip address
|
||||
String nonProxyUserConfiguredIpAddress = "255.255.255.255";
|
||||
solrClient.request(getProxyRequest("localHostAnyGroup", "bar", "unknownhost.bar.foo", nonProxyUserConfiguredIpAddress));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
assertTrue(ex.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyInvalidRemote() throws Exception {
|
||||
try {
|
||||
String invalidIpAddress = "-127.-128";
|
||||
solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", invalidIpAddress));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
assertTrue(ex.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyInvalidGroup() throws Exception {
|
||||
try {
|
||||
solrClient.request(getProxyRequest("bogusGroup","bar", null));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bogusGroup", "bar")));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testProxyNullProxyUser() throws Exception {
|
||||
try {
|
||||
solrClient.request(getProxyRequest("","bar"));
|
||||
fail("Expected RemoteSolrException");
|
||||
}
|
||||
catch (HttpSolrClient.RemoteSolrException ex) {
|
||||
// this exception is specific to our implementation, don't check a specific message.
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testForwarding() throws Exception {
|
||||
String collectionName = "forwardingCollection";
|
||||
File configDir = getFile("solr").toPath().resolve("collection1/conf").toFile();
|
||||
miniCluster.uploadConfigDir(configDir, "conf1");
|
||||
create1ShardCollection(collectionName, "conf1", miniCluster);
|
||||
|
||||
// try a command to each node, one of them must be forwarded
|
||||
for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) {
|
||||
HttpSolrClient client =
|
||||
new HttpSolrClient.Builder(jetty.getBaseUrl().toString() + "/" + collectionName).build();
|
||||
try {
|
||||
ModifiableSolrParams params = new ModifiableSolrParams();
|
||||
params.set("q", "*:*");
|
||||
params.set(USER_PARAM, "user");
|
||||
client.query(params);
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,109 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.hadoop.security.authentication.client.AuthenticationException;
|
||||
import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
|
||||
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
|
||||
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
|
||||
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.utils.URLEncodedUtils;
|
||||
|
||||
/**
|
||||
* AuthenticationHandler that supports delegation tokens and simple
|
||||
* authentication via the "user" http parameter
|
||||
*/
|
||||
public class HttpParamDelegationTokenAuthenticationHandler extends
|
||||
DelegationTokenAuthenticationHandler {
|
||||
|
||||
public static final String USER_PARAM = "user";
|
||||
|
||||
public HttpParamDelegationTokenAuthenticationHandler() {
|
||||
super(new HttpParamAuthenticationHandler());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Properties config) throws ServletException {
|
||||
Properties conf = new Properties();
|
||||
for (Map.Entry entry : config.entrySet()) {
|
||||
conf.setProperty((String) entry.getKey(), (String) entry.getValue());
|
||||
}
|
||||
conf.setProperty(TOKEN_KIND, KerberosPlugin.DELEGATION_TOKEN_TYPE_DEFAULT);
|
||||
super.init(conf);
|
||||
}
|
||||
|
||||
private static String getHttpParam(HttpServletRequest request, String param) {
|
||||
List<NameValuePair> pairs =
|
||||
URLEncodedUtils.parse(request.getQueryString(), Charset.forName("UTF-8"));
|
||||
for (NameValuePair nvp : pairs) {
|
||||
if(param.equals(nvp.getName())) {
|
||||
return nvp.getValue();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static class HttpParamAuthenticationHandler
|
||||
implements AuthenticationHandler {
|
||||
|
||||
@Override
|
||||
public String getType() {
|
||||
return "dummy";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Properties config) throws ServletException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean managementOperation(AuthenticationToken token,
|
||||
HttpServletRequest request, HttpServletResponse response)
|
||||
throws IOException, AuthenticationException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AuthenticationToken authenticate(HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws IOException, AuthenticationException {
|
||||
AuthenticationToken token = null;
|
||||
String userName = getHttpParam(request, USER_PARAM);
|
||||
if (userName != null) {
|
||||
return new AuthenticationToken(userName, userName, "test");
|
||||
} else {
|
||||
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
|
||||
response.setHeader("WWW-Authenticate", "dummy");
|
||||
}
|
||||
return token;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,272 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.solr.security;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.Principal;
|
||||
import java.util.Enumeration;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.servlet.Filter;
|
||||
import javax.servlet.FilterChain;
|
||||
import javax.servlet.FilterConfig;
|
||||
import javax.servlet.ServletContext;
|
||||
import javax.servlet.ServletException;
|
||||
import javax.servlet.ServletRequest;
|
||||
import javax.servlet.ServletResponse;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletRequestWrapper;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import org.apache.hadoop.security.authentication.client.AuthenticationException;
|
||||
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
|
||||
import org.apache.hadoop.security.authentication.server.AuthenticationHandler;
|
||||
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
|
||||
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
|
||||
|
||||
import org.apache.http.HttpException;
|
||||
import org.apache.http.HttpRequest;
|
||||
import org.apache.http.HttpRequestInterceptor;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.utils.URLEncodedUtils;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
import org.apache.solr.client.solrj.impl.HttpClientUtil;
|
||||
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.util.ExecutorUtil;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.request.SolrRequestInfo;
|
||||
|
||||
/**
|
||||
* AuthenticationHandler that supports delegation tokens and simple
|
||||
* authentication via the "user" http parameter
|
||||
*/
|
||||
public class HttpParamDelegationTokenPlugin extends KerberosPlugin {
|
||||
public static final String USER_PARAM = "user"; // http parameter for user authentication
|
||||
public static final String REMOTE_HOST_PARAM = "remoteHost"; // http parameter for indicating remote host
|
||||
public static final String REMOTE_ADDRESS_PARAM = "remoteAddress"; // http parameter for indicating remote address
|
||||
public static final String INTERNAL_REQUEST_HEADER = "internalRequest"; // http header for indicating internal request
|
||||
|
||||
boolean isSolrThread() {
|
||||
return ExecutorUtil.isSolrServerThread();
|
||||
}
|
||||
|
||||
private final HttpRequestInterceptor interceptor = new HttpRequestInterceptor() {
|
||||
@Override
|
||||
public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException {
|
||||
SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
|
||||
String usr;
|
||||
if (reqInfo != null) {
|
||||
Principal principal = reqInfo.getReq().getUserPrincipal();
|
||||
if (principal == null) {
|
||||
//this had a request but not authenticated
|
||||
//so we don't not need to set a principal
|
||||
return;
|
||||
} else {
|
||||
usr = principal.getName();
|
||||
}
|
||||
} else {
|
||||
if (!isSolrThread()) {
|
||||
//if this is not running inside a Solr threadpool (as in testcases)
|
||||
// then no need to add any header
|
||||
return;
|
||||
}
|
||||
//this request seems to be originated from Solr itself
|
||||
usr = "$"; //special name to denote the user is the node itself
|
||||
}
|
||||
httpRequest.setHeader(INTERNAL_REQUEST_HEADER, usr);
|
||||
}
|
||||
};
|
||||
|
||||
public HttpParamDelegationTokenPlugin(CoreContainer coreContainer) {
|
||||
super(coreContainer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Map<String, Object> pluginConfig) {
|
||||
try {
|
||||
final FilterConfig initConf = getInitFilterConfig(pluginConfig, true);
|
||||
|
||||
FilterConfig conf = new FilterConfig() {
|
||||
@Override
|
||||
public ServletContext getServletContext() {
|
||||
return initConf.getServletContext();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Enumeration<String> getInitParameterNames() {
|
||||
return initConf.getInitParameterNames();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getInitParameter(String param) {
|
||||
if (AuthenticationFilter.AUTH_TYPE.equals(param)) {
|
||||
return HttpParamDelegationTokenAuthenticationHandler.class.getName();
|
||||
}
|
||||
return initConf.getInitParameter(param);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getFilterName() {
|
||||
return "HttpParamFilter";
|
||||
}
|
||||
};
|
||||
Filter kerberosFilter = new HttpParamToRequestFilter();
|
||||
kerberosFilter.init(conf);
|
||||
setKerberosFilter(kerberosFilter);
|
||||
} catch (ServletException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Error initializing kerberos authentication plugin: "+e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SolrHttpClientBuilder getHttpClientBuilder(SolrHttpClientBuilder builder) {
|
||||
HttpClientUtil.addRequestInterceptor(interceptor);
|
||||
return super.getHttpClientBuilder(builder);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
HttpClientUtil.removeRequestInterceptor(interceptor);
|
||||
super.close();
|
||||
}
|
||||
|
||||
private static String getHttpParam(HttpServletRequest request, String param) {
|
||||
List<NameValuePair> pairs = URLEncodedUtils.parse(request.getQueryString(), Charset.forName("UTF-8"));
|
||||
for (NameValuePair nvp : pairs) {
|
||||
if (param.equals(nvp.getName())) {
|
||||
return nvp.getValue();
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static class HttpParamDelegationTokenAuthenticationHandler extends
|
||||
DelegationTokenAuthenticationHandler {
|
||||
|
||||
public HttpParamDelegationTokenAuthenticationHandler() {
|
||||
super(new HttpParamAuthenticationHandler());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Properties config) throws ServletException {
|
||||
Properties conf = new Properties();
|
||||
for (Map.Entry entry : config.entrySet()) {
|
||||
conf.setProperty((String) entry.getKey(), (String) entry.getValue());
|
||||
}
|
||||
conf.setProperty(TOKEN_KIND, KerberosPlugin.DELEGATION_TOKEN_TYPE_DEFAULT);
|
||||
super.init(conf);
|
||||
}
|
||||
|
||||
private static class HttpParamAuthenticationHandler implements AuthenticationHandler {
|
||||
@Override
|
||||
public String getType() {
|
||||
return "dummy";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void init(Properties config) throws ServletException {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean managementOperation(AuthenticationToken token,
|
||||
HttpServletRequest request, HttpServletResponse response)
|
||||
throws IOException, AuthenticationException {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AuthenticationToken authenticate(HttpServletRequest request,
|
||||
HttpServletResponse response)
|
||||
throws IOException, AuthenticationException {
|
||||
AuthenticationToken token = null;
|
||||
String userName = getHttpParam(request, USER_PARAM);
|
||||
if (userName == null) {
|
||||
//check if this is an internal request
|
||||
userName = request.getHeader(INTERNAL_REQUEST_HEADER);
|
||||
}
|
||||
if (userName != null) {
|
||||
return new AuthenticationToken(userName, userName, "test");
|
||||
} else {
|
||||
response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
|
||||
response.setHeader("WWW-Authenticate", "dummy");
|
||||
}
|
||||
return token;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter that converts http params to HttpServletRequest params
|
||||
*/
|
||||
private static class HttpParamToRequestFilter extends DelegationTokenKerberosFilter {
|
||||
@Override
|
||||
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
|
||||
throws IOException, ServletException {
|
||||
final HttpServletRequest httpRequest = (HttpServletRequest) request;
|
||||
final HttpServletRequestWrapper requestWrapper = new HttpServletRequestWrapper(httpRequest) {
|
||||
@Override
|
||||
public String getRemoteHost() {
|
||||
String param = getHttpParam(httpRequest, REMOTE_HOST_PARAM);
|
||||
return param != null ? param : httpRequest.getRemoteHost();
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getRemoteAddr() {
|
||||
String param = getHttpParam(httpRequest, REMOTE_ADDRESS_PARAM);
|
||||
return param != null ? param : httpRequest.getRemoteAddr();
|
||||
}
|
||||
};
|
||||
|
||||
super.doFilter(requestWrapper, response, chain);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doFilter(FilterChain filterChain, HttpServletRequest request,
|
||||
HttpServletResponse response) throws IOException, ServletException {
|
||||
// remove the filter-specific authentication information, so it doesn't get accidentally forwarded.
|
||||
List<NameValuePair> newPairs = new LinkedList<NameValuePair>();
|
||||
List<NameValuePair> pairs = URLEncodedUtils.parse(request.getQueryString(), Charset.forName("UTF-8"));
|
||||
for (NameValuePair nvp : pairs) {
|
||||
if (!USER_PARAM.equals(nvp.getName())) {
|
||||
newPairs.add(nvp);
|
||||
}
|
||||
else {
|
||||
request.setAttribute(USER_PARAM, nvp.getValue());
|
||||
}
|
||||
}
|
||||
final String queryStringNoUser = URLEncodedUtils.format(newPairs, StandardCharsets.UTF_8);
|
||||
HttpServletRequest requestWrapper = new HttpServletRequestWrapper(request) {
|
||||
@Override
|
||||
public String getQueryString() {
|
||||
return queryStringNoUser;
|
||||
}
|
||||
};
|
||||
super.doFilter(filterChain, requestWrapper, response);
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue