Merge remote-tracking branch 'elastic/master' into feature/sql

Original commit: elastic/x-pack-elasticsearch@bfdbc2bb75

plugin/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -404,6 +404,7 @@ public class Security implements ActionPlugin, IngestPlugin, NetworkPlugin, Clus
         final CompositeRolesStore allRolesStore = new CompositeRolesStore(settings, fileRolesStore, nativeRolesStore,
             reservedRolesStore, rolesProviders, threadPool.getThreadContext(), licenseState);
         securityLifecycleService.addSecurityIndexHealthChangeListener(allRolesStore::onSecurityIndexHealthChange);
+        securityLifecycleService.addSecurityIndexOutOfDateListener(allRolesStore::onSecurityIndexOutOfDateChange);
         // to keep things simple, just invalidate all cached entries on license change. this happens so rarely that the impact should be
         // minimal
         licenseState.addListener(allRolesStore::invalidateAll);

plugin/src/main/java/org/elasticsearch/xpack/security/SecurityLifecycleService.java

@@ -149,6 +149,15 @@ public class SecurityLifecycleService extends AbstractComponent implements Clust
         securityIndex.addIndexHealthChangeListener(listener);
     }
 
+    /**
+     * Adds a listener which will be notified when the security index out of date value changes. The previous and
+     * current value will be provided to the listener so that the listener can determine if any action
+     * needs to be taken.
+     */
+    public void addSecurityIndexOutOfDateListener(BiConsumer<Boolean, Boolean> listener) {
+        securityIndex.addIndexOutOfDateListener(listener);
+    }
+
     // this is called in a lifecycle listener beforeStop on the cluster service
     private void close() {
         if (indexAuditTrail != null) {

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStore.java

@@ -152,38 +152,46 @@ public class CompositeRolesStore extends AbstractComponent {
         } else {
             nativeRolesStore.getRoleDescriptors(remainingRoleNames.toArray(Strings.EMPTY_ARRAY), ActionListener.wrap((descriptors) -> {
                 builtInRoleDescriptors.addAll(descriptors);
-                if (builtInRoleDescriptors.size() != filteredRoleNames.size()) {
+                callCustomRoleProvidersIfEnabled(builtInRoleDescriptors, filteredRoleNames, roleDescriptorActionListener);
-                    final Set<String> missing = difference(filteredRoleNames, builtInRoleDescriptors);
+            }, e -> {
-                    assert missing.isEmpty() == false : "the missing set should not be empty if the sizes didn't match";
+                logger.warn("role retrieval failed from the native roles store", e);
-                    if (licenseState.isCustomRoleProvidersAllowed() && !customRolesProviders.isEmpty()) {
+                callCustomRoleProvidersIfEnabled(builtInRoleDescriptors, filteredRoleNames, roleDescriptorActionListener);
-                        new IteratingActionListener<>(roleDescriptorActionListener, (rolesProvider, listener) -> {
+            }));
-                            // resolve descriptors with role provider
+        }
-                            rolesProvider.accept(missing, ActionListener.wrap((resolvedDescriptors) -> {
+    }
-                                builtInRoleDescriptors.addAll(resolvedDescriptors);
+
-                                // remove resolved descriptors from the set of roles still needed to be resolved
+    private void callCustomRoleProvidersIfEnabled(Set<RoleDescriptor> builtInRoleDescriptors, Set<String> filteredRoleNames,
-                                for (RoleDescriptor descriptor : resolvedDescriptors) {
+                                                  ActionListener<Set<RoleDescriptor>> roleDescriptorActionListener) {
-                                    missing.remove(descriptor.getName());
+        if (builtInRoleDescriptors.size() != filteredRoleNames.size()) {
-                                }
+            final Set<String> missing = difference(filteredRoleNames, builtInRoleDescriptors);
-                                if (missing.isEmpty()) {
+            assert missing.isEmpty() == false : "the missing set should not be empty if the sizes didn't match";
-                                    // no more roles to resolve, send the response
+            if (licenseState.isCustomRoleProvidersAllowed() && !customRolesProviders.isEmpty()) {
-                                    listener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
+                new IteratingActionListener<>(roleDescriptorActionListener, (rolesProvider, listener) -> {
-                                } else {
+                    // resolve descriptors with role provider
-                                    // still have roles to resolve, keep trying with the next roles provider
+                    rolesProvider.accept(missing, ActionListener.wrap((resolvedDescriptors) -> {
-                                    listener.onResponse(null);
+                        builtInRoleDescriptors.addAll(resolvedDescriptors);
-                                }
+                        // remove resolved descriptors from the set of roles still needed to be resolved
-                            }, listener::onFailure));
+                        for (RoleDescriptor descriptor : resolvedDescriptors) {
-                        }, customRolesProviders, threadContext, () -> {
+                            missing.remove(descriptor.getName());
-                            negativeLookupCache.addAll(missing);
+                        }
-                            return builtInRoleDescriptors;
+                        if (missing.isEmpty()) {
-                        }).run();
+                            // no more roles to resolve, send the response
-                    } else {
+                            listener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
-                        negativeLookupCache.addAll(missing);
+                        } else {
-                        roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
+                            // still have roles to resolve, keep trying with the next roles provider
-                    }
+                            listener.onResponse(null);
-                } else {
+                        }
-                    roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
+                    }, listener::onFailure));
-                }
+                }, customRolesProviders, threadContext, () -> {
-            }, roleDescriptorActionListener::onFailure));
+                    negativeLookupCache.addAll(missing);
+                    return builtInRoleDescriptors;
+                }).run();
+            } else {
+                negativeLookupCache.addAll(missing);
+                roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
+            }
+        } else {
+            roleDescriptorActionListener.onResponse(Collections.unmodifiableSet(builtInRoleDescriptors));
         }
     }
 
@@ -300,6 +308,11 @@ public class CompositeRolesStore extends AbstractComponent {
         }
     }
 
+    public void onSecurityIndexOutOfDateChange(boolean prevOutOfDate, boolean outOfDate) {
+        assert prevOutOfDate != outOfDate : "this method should only be called if the two values are different";
+        invalidateAll();
+    }
+
     /**
      * A mutable class that can be used to represent the combination of one or more {@link IndicesPrivileges}
      */

plugin/src/main/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStore.java

@@ -109,14 +109,13 @@ public class NativeRolesStore extends AbstractComponent {
             listener.onFailure(new IllegalStateException(
                 "Security index is not on the current version - the native realm will not be operational until " +
                 "the upgrade API is run on the security index"));
-            return;
         } else {
             try {
                 QueryBuilder query;
                 if (names == null || names.length == 0) {
                     query = QueryBuilders.termQuery(RoleDescriptor.Fields.TYPE.getPreferredName(), ROLE_TYPE);
                 } else {
-                    final String[] roleNames = Arrays.asList(names).stream().map(s -> getIdForUser(s)).toArray(String[]::new);
+                    final String[] roleNames = Arrays.stream(names).map(s -> getIdForUser(s)).toArray(String[]::new);
                     query = QueryBuilders.boolQuery().filter(QueryBuilders.idsQuery(ROLE_DOC_TYPE).addIds(roleNames));
                 }
                 SearchRequest request = client.prepareSearch(SecurityLifecycleService.SECURITY_INDEX_NAME)
@@ -129,7 +128,7 @@ public class NativeRolesStore extends AbstractComponent {
                 InternalClient.fetchAllByEntity(client, request, listener,
                         (hit) -> transformRole(hit.getId(), hit.getSourceRef(), logger, licenseState));
             } catch (Exception e) {
-                logger.error((Supplier<?>) () -> new ParameterizedMessage("unable to retrieve roles {}", Arrays.toString(names)), e);
+                logger.error(new ParameterizedMessage("unable to retrieve roles {}", Arrays.toString(names)), e);
                 listener.onFailure(e);
             }
         }

plugin/src/main/java/org/elasticsearch/xpack/security/support/IndexLifecycleManager.java

@@ -62,6 +62,7 @@ public class IndexLifecycleManager extends AbstractComponent {
     private final InternalSecurityClient client;
 
     private final List<BiConsumer<ClusterIndexHealth, ClusterIndexHealth>> indexHealthChangeListeners = new CopyOnWriteArrayList<>();
+    private final List<BiConsumer<Boolean, Boolean>> indexOutOfDateListeners = new CopyOnWriteArrayList<>();
 
     private volatile boolean templateIsUpToDate;
     private volatile boolean indexExists;
@@ -107,9 +108,22 @@ public class IndexLifecycleManager extends AbstractComponent {
         indexHealthChangeListeners.add(listener);
     }
 
+    /**
+     * Adds a listener which will be notified when the security index out of date value changes. The previous and
+     * current value will be provided to the listener so that the listener can determine if any action
+     * needs to be taken.
+     */
+    public void addIndexOutOfDateListener(BiConsumer<Boolean, Boolean> listener) {
+        indexOutOfDateListeners.add(listener);
+    }
+
     public void clusterChanged(ClusterChangedEvent event) {
+        final boolean previousUpToDate = this.isIndexUpToDate;
         processClusterState(event.state());
         checkIndexHealthChange(event);
+        if (previousUpToDate != this.isIndexUpToDate) {
+            notifyIndexOutOfDateListeners(previousUpToDate, this.isIndexUpToDate);
+        }
     }
 
     private void processClusterState(ClusterState state) {
@@ -157,6 +171,16 @@ public class IndexLifecycleManager extends AbstractComponent {
         }
     }
 
+    private void notifyIndexOutOfDateListeners(boolean previous, boolean current) {
+        for (BiConsumer<Boolean, Boolean> consumer : indexOutOfDateListeners) {
+            try {
+                consumer.accept(previous, current);
+            } catch (Exception e) {
+                logger.warn(new ParameterizedMessage("failed to notify listener [{}] of index out of date change", consumer), e);
+            }
+        }
+    }
+
     private boolean checkIndexAvailable(ClusterState state) {
         final IndexRoutingTable routingTable = getIndexRoutingTable(state);
         if (routingTable != null && routingTable.allPrimaryShardsActive()) {

plugin/src/main/java/org/elasticsearch/xpack/ssl/CertUtils.java

@@ -51,13 +51,19 @@ import java.util.stream.Collectors;
 
 import org.bouncycastle.asn1.ASN1Encodable;
 import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.ASN1TaggedObject;
+import org.bouncycastle.asn1.BERSequence;
+import org.bouncycastle.asn1.BERTaggedObject;
 import org.bouncycastle.asn1.DERIA5String;
 import org.bouncycastle.asn1.DERSequence;
+import org.bouncycastle.asn1.DERTaggedObject;
+import org.bouncycastle.asn1.DERUTF8String;
 import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
 import org.bouncycastle.asn1.pkcs.PrivateKeyInfo;
 import org.bouncycastle.asn1.x500.X500Name;
 import org.bouncycastle.asn1.x509.AuthorityKeyIdentifier;
 import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.DisplayText;
 import org.bouncycastle.asn1.x509.Extension;
 import org.bouncycastle.asn1.x509.ExtensionsGenerator;
 import org.bouncycastle.asn1.x509.GeneralName;
@@ -509,12 +515,12 @@ public class CertUtils {
     /**
      * Creates an X.509 {@link GeneralName} for use as a <em>Common Name</em> in the certificate's <em>Subject Alternative Names</em>
      * extension. A <em>common name</em> is a name with a tag of {@link GeneralName#otherName OTHER}, with an object-id that references
-     * the {@link #CN_OID cn} attribute, and a DER encoded IA5 (ASCII) string for the name.
+     * the {@link #CN_OID cn} attribute, an explicit tag of '0', and a DER encoded UTF8 string for the name.
      * This usage of using the {@code cn} OID as a <em>Subject Alternative Name</em> is <strong>non-standard</strong> and will not be
      * recognised by other X.509/TLS implementations.
      */
     static GeneralName createCommonName(String cn) {
-        final ASN1Encodable[] sequence = { new ASN1ObjectIdentifier(CN_OID), new DERIA5String(cn) };
+        final ASN1Encodable[] sequence = { new ASN1ObjectIdentifier(CN_OID), new DERTaggedObject(true, 0, new DERUTF8String(cn)) };
         return new GeneralName(GeneralName.otherName, new DERSequence(sequence));
     }
 }

plugin/src/main/java/org/elasticsearch/xpack/ssl/RestrictedTrustManager.java

@@ -23,7 +23,9 @@ import java.util.stream.Collectors;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.ASN1Primitive;
 import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.ASN1String;
 import org.bouncycastle.asn1.ASN1TaggedObject;
 import org.bouncycastle.asn1.DERTaggedObject;
 import org.elasticsearch.common.logging.Loggers;
@@ -126,13 +128,35 @@ public final class RestrictedTrustManager extends X509ExtendedTrustManager {
                 .map(pair -> pair.get(1))
                 .map(value -> {
                     ASN1Sequence seq = ASN1Sequence.getInstance(value);
-                    assert seq.size() == 2 : "Incorrect sequence length for 'other name'";
+                    if (seq.size() != 2) {
+                        String message = "Incorrect sequence length for 'other name' [" + seq + "]";
+                        assert false : message;
+                        logger.warn(message);
+                        return null;
+                    }
                     final String id = ASN1ObjectIdentifier.getInstance(seq.getObjectAt(0)).getId();
                     if (CertUtils.CN_OID.equals(id)) {
-                        final ASN1TaggedObject object = DERTaggedObject.getInstance(seq.getObjectAt(1));
+                        ASN1TaggedObject tagged = DERTaggedObject.getInstance(seq.getObjectAt(1));
-                        final String cn = object.getObject().toString();
+                        // The JRE's handling of OtherNames is buggy.
-                        logger.trace("Read cn [{}] from ASN1Sequence [{}]", cn, seq);
+                        // The internal sun classes go to a lot of trouble to parse the GeneralNames into real object
-                        return cn;
+                        // And then java.security.cert.X509Certificate just turns them back into bytes
+                        // But in doing so, it ends up wrapping the "other name" bytes with a second tag
+                        // Specifically: sun.security.x509.OtherName(DerValue) never decodes the tagged "nameValue"
+                        // But: sun.security.x509.OtherName.encode() wraps the nameValue in a DER Tag.
+                        // So, there's a good chance that our tagged nameValue contains... a tagged name value.
+                        if (tagged.getObject() instanceof ASN1TaggedObject) {
+                            tagged = (ASN1TaggedObject) tagged.getObject();
+                        }
+                        final ASN1Primitive nameValue = tagged.getObject();
+                        if (nameValue instanceof ASN1String) {
+                            final String cn = ((ASN1String) nameValue).getString();
+                            logger.trace("Read cn [{}] from ASN1Sequence [{}]", cn, seq);
+                            return cn;
+                        } else {
+                            logger.warn("Certificate [{}] has 'otherName' [{}] with unsupported name-value type [{}]",
+                                    certificate.getSubjectDN(), seq, nameValue.getClass().getSimpleName());
+                            return null;
+                        }
                     } else {
                         logger.debug("Certificate [{}] has 'otherName' [{}] with unsupported object-id [{}]",
                                 certificate.getSubjectDN(), seq, id);

plugin/src/test/java/org/elasticsearch/integration/ShrinkIndexWithSecurityTests.java

@@ -50,7 +50,7 @@ public class ShrinkIndexWithSecurityTests extends SecurityIntegTestCase {
 
         // wait for green and then shrink
         ensureGreen();
-        assertAcked(client().admin().indices().prepareShrinkIndex("bigindex", "shrunk_bigindex")
+        assertAcked(client().admin().indices().prepareResizeIndex("bigindex", "shrunk_bigindex")
                 .setSettings(Settings.builder()
                         .put("index.number_of_replicas", 0)
                         .put("index.number_of_shards", 1)

plugin/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java

@@ -488,6 +488,25 @@ public class CompositeRolesStoreTests extends ESTestCase {
         assertEquals(expectedInvalidation, numInvalidation.get());
     }
 
+    public void testCacheClearOnIndexOutOfDateChange() {
+        final AtomicInteger numInvalidation = new AtomicInteger(0);
+
+        CompositeRolesStore compositeRolesStore = new CompositeRolesStore(
+                Settings.EMPTY, mock(FileRolesStore.class), mock(NativeRolesStore.class), mock(ReservedRolesStore.class),
+                Collections.emptyList(), new ThreadContext(Settings.EMPTY), new XPackLicenseState()) {
+            @Override
+            public void invalidateAll() {
+                numInvalidation.incrementAndGet();
+            }
+        };
+
+        compositeRolesStore.onSecurityIndexOutOfDateChange(false, true);
+        assertEquals(1, numInvalidation.get());
+
+        compositeRolesStore.onSecurityIndexOutOfDateChange(true, false);
+        assertEquals(2, numInvalidation.get());
+    }
+
     private static class InMemoryRolesProvider implements BiConsumer<Set<String>, ActionListener<Set<RoleDescriptor>>> {
         private final Function<Set<String>, Set<RoleDescriptor>> roleDescriptorsFunc;
 

plugin/src/test/java/org/elasticsearch/xpack/security/support/IndexLifecycleManagerTests.java

@@ -44,7 +44,6 @@ import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.xpack.security.InternalClient;
 import org.elasticsearch.xpack.security.InternalSecurityClient;
 import org.elasticsearch.xpack.security.test.SecurityTestUtils;
 import org.elasticsearch.xpack.template.TemplateUtils;
@@ -204,6 +203,32 @@ public class IndexLifecycleManagerTests extends ESTestCase {
         assertEquals(ClusterHealthStatus.GREEN, currentHealth.get().getStatus());
     }
 
+    public void testIndexOutOfDateListeners() throws Exception {
+        final AtomicBoolean listenerCalled = new AtomicBoolean(false);
+        manager.addIndexOutOfDateListener((prev, current) -> {
+            listenerCalled.set(true);
+            assertNotEquals(prev, current);
+        });
+        assertFalse(manager.isIndexUpToDate());
+
+        manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME)));
+        assertFalse(listenerCalled.get());
+        assertFalse(manager.isIndexUpToDate());
+
+        // index doesn't exist and now exists
+        final ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME);
+        markShardsAvailable(clusterStateBuilder);
+        manager.clusterChanged(event(clusterStateBuilder));
+        assertTrue(listenerCalled.get());
+        assertTrue(manager.isIndexUpToDate());
+
+        listenerCalled.set(false);
+        assertFalse(listenerCalled.get());
+        manager.clusterChanged(event(new ClusterState.Builder(CLUSTER_NAME)));
+        assertTrue(listenerCalled.get());
+        assertFalse(manager.isIndexUpToDate());
+    }
+
     private void assertInitialState() {
         assertThat(manager.indexExists(), Matchers.equalTo(false));
         assertThat(manager.isAvailable(), Matchers.equalTo(false));
@@ -250,6 +275,7 @@ public class IndexLifecycleManagerTests extends ESTestCase {
                 .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                 .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
                 .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), IndexLifecycleManager.INTERNAL_INDEX_FORMAT)
                 .build());
 
         final Map<String, String> mappings = getTemplateMappings(templateName);

plugin/src/test/java/org/elasticsearch/xpack/ssl/CertificateGenerateToolTests.java

@@ -45,6 +45,7 @@ import org.bouncycastle.asn1.ASN1ObjectIdentifier;
 import org.bouncycastle.asn1.ASN1Sequence;
 import org.bouncycastle.asn1.ASN1String;
 import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.DERTaggedObject;
 import org.bouncycastle.asn1.pkcs.Attribute;
 import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
 import org.bouncycastle.asn1.x509.Extension;
@@ -482,8 +483,11 @@ public class CertificateGenerateToolTests extends ESTestCase {
                 assertThat(seq.size(), equalTo(2));
                 assertThat(seq.getObjectAt(0), instanceOf(ASN1ObjectIdentifier.class));
                 assertThat(seq.getObjectAt(0).toString(), equalTo(CertUtils.CN_OID));
-                assertThat(seq.getObjectAt(1), instanceOf(ASN1String.class));
+                assertThat(seq.getObjectAt(1), instanceOf(DERTaggedObject.class));
-                assertThat(seq.getObjectAt(1).toString(), Matchers.isIn(certInfo.commonNames));
+                DERTaggedObject taggedName = (DERTaggedObject) seq.getObjectAt(1);
+                assertThat(taggedName.getTagNo(), equalTo(0));
+                assertThat(taggedName.getObject(), instanceOf(ASN1String.class));
+                assertThat(taggedName.getObject().toString(), Matchers.isIn(certInfo.commonNames));
             } else {
                 fail("unknown general name with tag " + generalName.getTagNo());
             }

plugin/src/test/java/org/elasticsearch/xpack/ssl/CertificateToolTests.java

@@ -50,6 +50,7 @@ import org.apache.lucene.util.IOUtils;
 import org.bouncycastle.asn1.ASN1ObjectIdentifier;
 import org.bouncycastle.asn1.ASN1Sequence;
 import org.bouncycastle.asn1.ASN1String;
+import org.bouncycastle.asn1.ASN1TaggedObject;
 import org.bouncycastle.asn1.DEROctetString;
 import org.bouncycastle.asn1.pkcs.Attribute;
 import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
@@ -797,8 +798,10 @@ public class CertificateToolTests extends ESTestCase {
                 assertThat(seq.size(), equalTo(2));
                 assertThat(seq.getObjectAt(0), instanceOf(ASN1ObjectIdentifier.class));
                 assertThat(seq.getObjectAt(0).toString(), equalTo(CertUtils.CN_OID));
-                assertThat(seq.getObjectAt(1), instanceOf(ASN1String.class));
+                assertThat(seq.getObjectAt(1), instanceOf(ASN1TaggedObject.class));
-                assertThat(seq.getObjectAt(1).toString(), Matchers.isIn(certInfo.commonNames));
+                ASN1TaggedObject tagged = (ASN1TaggedObject) seq.getObjectAt(1);
+                assertThat(tagged.getObject(), instanceOf(ASN1String.class));
+                assertThat(tagged.getObject().toString(), Matchers.isIn(certInfo.commonNames));
             } else {
                 fail("unknown general name with tag " + generalName.getTagNo());
             }

plugin/src/test/resources/org/elasticsearch/transport/actions

@@ -44,6 +44,7 @@ indices:admin/refresh
 indices:admin/settings/update
 indices:admin/shards/search_shards
 indices:admin/shrink
+indices:admin/resize
 indices:admin/rollover
 indices:admin/template/delete
 indices:admin/template/get