honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'elastic/master' into feature/sql 

Original commit: elastic/x-pack-elasticsearch@bfdbc2bb75
This commit is contained in:
Igor Motov 2017-11-06 18:51:23 -05:00
parent a211077554 be773363c9
commit de33803d85
13 changed files with 177 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
plugin/src/main/java/org/elasticsearch/xpack/security/Security.java
View File

@ -404,6 +404,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
reservedRolesStore, rolesProviders, threadPool.getThreadContext(), licenseState);
securityLifecycleService.addSecurityIndexHealthChangeListener(allRolesStore::onSecurityIndexHealthChange);
securityLifecycleService.addSecurityIndexOutOfDateListener(allRolesStore::onSecurityIndexOutOfDateChange);
// to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
// minimal
licenseState.addListener(allRolesStore::invalidateAll);

9
plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java
View File

@ -149,6 +149,15 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
securityIndex.addIndexHealthChangeListener(listener);
}
/**
* Adds a listener which will be notified when the security index out of date value changes. The previous and
* current value will be provided to the listener so that the listener can determine if any action
* needs to be taken.
*/
public void addSecurityIndexOutOfDateListener(BiConsumer<Boolean, Boolean> listener) {
securityIndex.addIndexOutOfDateListener(listener);
}
// this is called in a lifecycle listener beforeStop on the cluster service
private void close() {
if (indexAuditTrail != null) {

77
plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java
View File

@ -152,38 +152,46 @@ public class CompositeRolesStore extends AbstractComponent {
} else {
nativeRolesStore.getRoleDescriptors(remainingRoleNames.toArray(Strings.EMPTY_ARRAY), ActionListener.wrap((descriptors) -> {
builtInRoleDescriptors.addAll(descriptors);
if (builtInRoleDescriptors.size() != filteredRoleNames.size()) {
final Set<String> missing = difference(filteredRoleNames, builtInRoleDescriptors);
assert missing.isEmpty() == false : "the missing set should not be empty if the sizes didn't match";
if (licenseState.isCustomRoleProvidersAllowed() && !customRolesProviders.isEmpty()) {
new IteratingActionListener<>(roleDescriptorActionListener, (rolesProvider, listener) -> {
// resolve descriptors with role provider
rolesProvider.accept(missing, ActionListener.wrap((resolvedDescriptors) -> {
builtInRoleDescriptors.addAll(resolvedDescriptors);
// remove resolved descriptors from the set of roles still needed to be resolved
for (RoleDescriptor descriptor : resolvedDescriptors) {
missing.remove(descriptor.getName());
}
if (missing.isEmpty()) {
// no more roles to resolve, send the response
listener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
} else {
// still have roles to resolve, keep trying with the next roles provider
listener.onResponse(null);
}
}, listener::onFailure));
}, customRolesProviders, threadContext, () -> {
negativeLookupCache.addAll(missing);
return builtInRoleDescriptors;
}).run();
} else {
negativeLookupCache.addAll(missing);
roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
}
} else {
roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
}
}, roleDescriptorActionListener::onFailure));
callCustomRoleProvidersIfEnabled(builtInRoleDescriptors, filteredRoleNames, roleDescriptorActionListener);
}, e -> {
logger.warn("role retrieval failed from the native roles store", e);
callCustomRoleProvidersIfEnabled(builtInRoleDescriptors, filteredRoleNames, roleDescriptorActionListener);
}));
}
}
private void callCustomRoleProvidersIfEnabled(Set<RoleDescriptor> builtInRoleDescriptors, Set<String> filteredRoleNames,
ActionListener<Set<RoleDescriptor>> roleDescriptorActionListener) {
if (builtInRoleDescriptors.size() != filteredRoleNames.size()) {
final Set<String> missing = difference(filteredRoleNames, builtInRoleDescriptors);
assert missing.isEmpty() == false : "the missing set should not be empty if the sizes didn't match";
if (licenseState.isCustomRoleProvidersAllowed() && !customRolesProviders.isEmpty()) {
new IteratingActionListener<>(roleDescriptorActionListener, (rolesProvider, listener) -> {
// resolve descriptors with role provider
rolesProvider.accept(missing, ActionListener.wrap((resolvedDescriptors) -> {
builtInRoleDescriptors.addAll(resolvedDescriptors);
// remove resolved descriptors from the set of roles still needed to be resolved
for (RoleDescriptor descriptor : resolvedDescriptors) {
missing.remove(descriptor.getName());
}
if (missing.isEmpty()) {
// no more roles to resolve, send the response
listener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
} else {
// still have roles to resolve, keep trying with the next roles provider
listener.onResponse(null);
}
}, listener::onFailure));
}, customRolesProviders, threadContext, () -> {
negativeLookupCache.addAll(missing);
return builtInRoleDescriptors;
}).run();
} else {
negativeLookupCache.addAll(missing);
roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
}
} else {
roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
}
}
@ -300,6 +308,11 @@ public class CompositeRolesStore extends AbstractComponent {
}
}
public void onSecurityIndexOutOfDateChange(boolean prevOutOfDate, boolean outOfDate) {
assert prevOutOfDate != outOfDate : "this method should only be called if the two values are different";
invalidateAll();
}
/**
* A mutable class that can be used to represent the combination of one or more {@link IndicesPrivileges}
*/

5
plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java
View File

@ -109,14 +109,13 @@ public class NativeRolesStore extends AbstractComponent {
listener.onFailure(new IllegalStateException(
"Security index is not on the current version - the native realm will not be operational until " +
"the upgrade API is run on the security index"));
return;
} else {
try {
QueryBuilder query;
if (names == null || names.length == 0) {
query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE);
} else {
final String[] roleNames = Arrays.asList(names).stream().map(s -> getIdForUser(s)).toArray(String[]::new);
final String[] roleNames = Arrays.stream(names).map(s -> getIdForUser(s)).toArray(String[]::new);
query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(ROLE_DOC_TYPE).addIds(roleNames));
}
SearchRequest request = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
@ -129,7 +128,7 @@ public class NativeRolesStore extends AbstractComponent {
InternalClient.fetchAllByEntity(client, request, listener,
(hit) -> transformRole(hit.getId(), hit.getSourceRef(), logger, licenseState));
} catch (Exception e) {
logger.error((Supplier<?>) () -> new ParameterizedMessage("unable to retrieve roles {}", Arrays.toString(names)), e);
logger.error(new ParameterizedMessage("unable to retrieve roles {}", Arrays.toString(names)), e);
listener.onFailure(e);
}
}

24
plugin/src/main/java/org/elasticsearch/xpack/security/support/IndexLifecycleManager.java
View File

@ -62,6 +62,7 @@ public class IndexLifecycleManager extends AbstractComponent {
private final InternalSecurityClient client;
private final List<BiConsumer<ClusterIndexHealth, ClusterIndexHealth>> indexHealthChangeListeners = new CopyOnWriteArrayList<>();
private final List<BiConsumer<Boolean, Boolean>> indexOutOfDateListeners = new CopyOnWriteArrayList<>();
private volatile boolean templateIsUpToDate;
private volatile boolean indexExists;
@ -107,9 +108,22 @@ public class IndexLifecycleManager extends AbstractComponent {
indexHealthChangeListeners.add(listener);
}
/**
* Adds a listener which will be notified when the security index out of date value changes. The previous and
* current value will be provided to the listener so that the listener can determine if any action
* needs to be taken.
*/
public void addIndexOutOfDateListener(BiConsumer<Boolean, Boolean> listener) {
indexOutOfDateListeners.add(listener);
}
public void clusterChanged(ClusterChangedEvent event) {
final boolean previousUpToDate = this.isIndexUpToDate;
processClusterState(event.state());
checkIndexHealthChange(event);
if (previousUpToDate != this.isIndexUpToDate) {
notifyIndexOutOfDateListeners(previousUpToDate, this.isIndexUpToDate);
}
}
private void processClusterState(ClusterState state) {
@ -157,6 +171,16 @@ public class IndexLifecycleManager extends AbstractComponent {
}
}
private void notifyIndexOutOfDateListeners(boolean previous, boolean current) {
for (BiConsumer<Boolean, Boolean> consumer : indexOutOfDateListeners) {
try {
consumer.accept(previous, current);
} catch (Exception e) {
logger.warn(new ParameterizedMessage("failed to notify listener [{}] of index out of date change", consumer), e);
}
}
}
private boolean checkIndexAvailable(ClusterState state) {
final IndexRoutingTable routingTable = getIndexRoutingTable(state);
if (routingTable != null && routingTable.allPrimaryShardsActive()) {

10
plugin/src/main/java/org/elasticsearch/xpack/ssl/CertUtils.java
View File

@ -51,13 +51,19 @@ import java.util.stream.Collectors;
import org.bouncycastle.asn1.ASN1Encodable;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1TaggedObject;
import org.bouncycastle.asn1.BERSequence;
import org.bouncycastle.asn1.BERTaggedObject;
import org.bouncycastle.asn1.DERIA5String;
import org.bouncycastle.asn1.DERSequence;
import org.bouncycastle.asn1.DERTaggedObject;
import org.bouncycastle.asn1.DERUTF8String;
import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.asn1.x509.AuthorityKeyIdentifier;
import org.bouncycastle.asn1.x509.BasicConstraints;
import org.bouncycastle.asn1.x509.DisplayText;
import org.bouncycastle.asn1.x509.Extension;
import org.bouncycastle.asn1.x509.ExtensionsGenerator;
import org.bouncycastle.asn1.x509.GeneralName;
@ -509,12 +515,12 @@ public class CertUtils {
/**
* Creates an X.509 {@link GeneralName} for use as a <em>Common Name</em> in the certificate's <em>Subject Alternative Names</em>
* extension. A <em>common name</em> is a name with a tag of {@link GeneralName#otherName OTHER}, with an object-id that references
* the {@link #CN_OID cn} attribute, and a DER encoded IA5 (ASCII) string for the name.
* the {@link #CN_OID cn} attribute, an explicit tag of '0', and a DER encoded UTF8 string for the name.
* This usage of using the {@code cn} OID as a <em>Subject Alternative Name</em> is <strong>non-standard</strong> and will not be
* recognised by other X.509/TLS implementations.
*/
static GeneralName createCommonName(String cn) {
final ASN1Encodable[] sequence = { new ASN1ObjectIdentifier(CN_OID), new DERIA5String(cn) };
final ASN1Encodable[] sequence = { new ASN1ObjectIdentifier(CN_OID), new DERTaggedObject(true, 0, new DERUTF8String(cn)) };
return new GeneralName(GeneralName.otherName, new DERSequence(sequence));
}
}

34
plugin/src/main/java/org/elasticsearch/xpack/ssl/RestrictedTrustManager.java
View File

@ -23,7 +23,9 @@ import java.util.stream.Collectors;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Primitive;
import org.bouncycastle.asn1.ASN1Sequence;
import org.bouncycastle.asn1.ASN1String;
import org.bouncycastle.asn1.ASN1TaggedObject;
import org.bouncycastle.asn1.DERTaggedObject;
import org.elasticsearch.common.logging.Loggers;
@ -126,13 +128,35 @@ public final class RestrictedTrustManager extends X509ExtendedTrustManager {
.map(pair -> pair.get(1))
.map(value -> {
ASN1Sequence seq = ASN1Sequence.getInstance(value);
assert seq.size() == 2 : "Incorrect sequence length for 'other name'";
if (seq.size() != 2) {
String message = "Incorrect sequence length for 'other name' [" + seq + "]";
assert false : message;
logger.warn(message);
return null;
}
final String id = ASN1ObjectIdentifier.getInstance(seq.getObjectAt(0)).getId();
if (CertUtils.CN_OID.equals(id)) {
final ASN1TaggedObject object = DERTaggedObject.getInstance(seq.getObjectAt(1));
final String cn = object.getObject().toString();
logger.trace("Read cn [{}] from ASN1Sequence [{}]", cn, seq);
return cn;
ASN1TaggedObject tagged = DERTaggedObject.getInstance(seq.getObjectAt(1));
// The JRE's handling of OtherNames is buggy.
// The internal sun classes go to a lot of trouble to parse the GeneralNames into real object
// And then java.security.cert.X509Certificate just turns them back into bytes
// But in doing so, it ends up wrapping the "other name" bytes with a second tag
// Specifically: sun.security.x509.OtherName(DerValue) never decodes the tagged "nameValue"
// But: sun.security.x509.OtherName.encode() wraps the nameValue in a DER Tag.
// So, there's a good chance that our tagged nameValue contains... a tagged name value.
if (tagged.getObject() instanceof ASN1TaggedObject) {
tagged = (ASN1TaggedObject) tagged.getObject();
}
final ASN1Primitive nameValue = tagged.getObject();
if (nameValue instanceof ASN1String) {
final String cn = ((ASN1String) nameValue).getString();
logger.trace("Read cn [{}] from ASN1Sequence [{}]", cn, seq);
return cn;
} else {
logger.warn("Certificate [{}] has 'otherName' [{}] with unsupported name-value type [{}]",
certificate.getSubjectDN(), seq, nameValue.getClass().getSimpleName());
return null;
}
} else {
logger.debug("Certificate [{}] has 'otherName' [{}] with unsupported object-id [{}]",
certificate.getSubjectDN(), seq, id);

2
plugin/src/test/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java
View File

@ -50,7 +50,7 @@ public class ShrinkIndexWithSecurityTests extends SecurityIntegTestCase {
// wait for green and then shrink
ensureGreen();
assertAcked(client().admin().indices().prepareShrinkIndex("bigindex", "shrunk_bigindex")
assertAcked(client().admin().indices().prepareResizeIndex("bigindex", "shrunk_bigindex")
.setSettings(Settings.builder()
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", 1)

19
plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java
View File

@ -488,6 +488,25 @@ public class CompositeRolesStoreTests extends ESTestCase {
assertEquals(expectedInvalidation, numInvalidation.get());
}
public void testCacheClearOnIndexOutOfDateChange() {
final AtomicInteger numInvalidation = new AtomicInteger(0);
CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
Settings.EMPTY, mock(FileRolesStore.class), mock(NativeRolesStore.class), mock(ReservedRolesStore.class),
Collections.emptyList(), new ThreadContext(Settings.EMPTY), new XPackLicenseState()) {
@Override
public void invalidateAll() {
numInvalidation.incrementAndGet();
}
};
compositeRolesStore.onSecurityIndexOutOfDateChange(false, true);
assertEquals(1, numInvalidation.get());
compositeRolesStore.onSecurityIndexOutOfDateChange(true, false);
assertEquals(2, numInvalidation.get());
}
private static class InMemoryRolesProvider implements BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>> {
private final Function<Set<String>, Set<RoleDescriptor>> roleDescriptorsFunc;

28
plugin/src/test/java/org/elasticsearch/xpack/security/support/IndexLifecycleManagerTests.java
View File

@ -44,7 +44,6 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.security.InternalClient;
import org.elasticsearch.xpack.security.InternalSecurityClient;
import org.elasticsearch.xpack.security.test.SecurityTestUtils;
import org.elasticsearch.xpack.template.TemplateUtils;
@ -204,6 +203,32 @@ public class IndexLifecycleManagerTests extends ESTestCase {
assertEquals(ClusterHealthStatus.GREEN, currentHealth.get().getStatus());
}
public void testIndexOutOfDateListeners() throws Exception {
final AtomicBoolean listenerCalled = new AtomicBoolean(false);
manager.addIndexOutOfDateListener((prev, current) -> {
listenerCalled.set(true);
assertNotEquals(prev, current);
});
assertFalse(manager.isIndexUpToDate());
manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME)));
assertFalse(listenerCalled.get());
assertFalse(manager.isIndexUpToDate());
// index doesn't exist and now exists
final ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME);
markShardsAvailable(clusterStateBuilder);
manager.clusterChanged(event(clusterStateBuilder));
assertTrue(listenerCalled.get());
assertTrue(manager.isIndexUpToDate());
listenerCalled.set(false);
assertFalse(listenerCalled.get());
manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME)));
assertTrue(listenerCalled.get());
assertFalse(manager.isIndexUpToDate());
}
private void assertInitialState() {
assertThat(manager.indexExists(), Matchers.equalTo(false));
assertThat(manager.isAvailable(), Matchers.equalTo(false));
@ -250,6 +275,7 @@ public class IndexLifecycleManagerTests extends ESTestCase {
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), IndexLifecycleManager.INTERNAL_INDEX_FORMAT)
.build());
final Map<String, String> mappings = getTemplateMappings(templateName);

8
plugin/src/test/java/org/elasticsearch/xpack/ssl/CertificateGenerateToolTests.java
View File

@ -45,6 +45,7 @@ import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Sequence;
import org.bouncycastle.asn1.ASN1String;
import org.bouncycastle.asn1.DEROctetString;
import org.bouncycastle.asn1.DERTaggedObject;
import org.bouncycastle.asn1.pkcs.Attribute;
import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
import org.bouncycastle.asn1.x509.Extension;
@ -482,8 +483,11 @@ public class CertificateGenerateToolTests extends ESTestCase {
assertThat(seq.size(), equalTo(2));
assertThat(seq.getObjectAt(0), instanceOf(ASN1ObjectIdentifier.class));
assertThat(seq.getObjectAt(0).toString(), equalTo(CertUtils.CN_OID));
assertThat(seq.getObjectAt(1), instanceOf(ASN1String.class));
assertThat(seq.getObjectAt(1).toString(), Matchers.isIn(certInfo.commonNames));
assertThat(seq.getObjectAt(1), instanceOf(DERTaggedObject.class));
DERTaggedObject taggedName = (DERTaggedObject) seq.getObjectAt(1);
assertThat(taggedName.getTagNo(), equalTo(0));
assertThat(taggedName.getObject(), instanceOf(ASN1String.class));
assertThat(taggedName.getObject().toString(), Matchers.isIn(certInfo.commonNames));
} else {
fail("unknown general name with tag " + generalName.getTagNo());
}

7
plugin/src/test/java/org/elasticsearch/xpack/ssl/CertificateToolTests.java
View File

@ -50,6 +50,7 @@ import org.apache.lucene.util.IOUtils;
import org.bouncycastle.asn1.ASN1ObjectIdentifier;
import org.bouncycastle.asn1.ASN1Sequence;
import org.bouncycastle.asn1.ASN1String;
import org.bouncycastle.asn1.ASN1TaggedObject;
import org.bouncycastle.asn1.DEROctetString;
import org.bouncycastle.asn1.pkcs.Attribute;
import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
@ -797,8 +798,10 @@ public class CertificateToolTests extends ESTestCase {
assertThat(seq.size(), equalTo(2));
assertThat(seq.getObjectAt(0), instanceOf(ASN1ObjectIdentifier.class));
assertThat(seq.getObjectAt(0).toString(), equalTo(CertUtils.CN_OID));
assertThat(seq.getObjectAt(1), instanceOf(ASN1String.class));
assertThat(seq.getObjectAt(1).toString(), Matchers.isIn(certInfo.commonNames));
assertThat(seq.getObjectAt(1), instanceOf(ASN1TaggedObject.class));
ASN1TaggedObject tagged = (ASN1TaggedObject) seq.getObjectAt(1);
assertThat(tagged.getObject(), instanceOf(ASN1String.class));
assertThat(tagged.getObject().toString(), Matchers.isIn(certInfo.commonNames));
} else {
fail("unknown general name with tag " + generalName.getTagNo());
}

1
plugin/src/test/resources/org/elasticsearch/transport/actions
View File

@ -44,6 +44,7 @@ indices:admin/refresh
indices:admin/settings/update
indices:admin/shards/search_shards
indices:admin/shrink
indices:admin/resize
indices:admin/rollover
indices:admin/template/delete
indices:admin/template/get