From 82e041442e410c19ad90388073ecef9e345fe8ac Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 24 Mar 2020 15:52:56 -0600 Subject: [PATCH] Add version guards around Transform hidden index settings (#54036) This commit ensures that the hidden index settings are only applied to the Transform index templates when the cluster can support those settings. Also unmutes the tests which were failing due to the previous behavior. --- .../xpack/transform/Transform.java | 6 ++++- .../TransformClusterStateListener.java | 14 +++++++--- .../persistence/TransformInternalIndex.java | 26 ++++++++++++------- .../TransformInternalIndexTests.java | 4 ++- .../upgrades/TransformSurvivesUpgradeIT.java | 1 - .../mixed_cluster/80_transform_jobs_crud.yml | 5 ---- .../80_transform_jobs_crud.yml | 4 --- 7 files changed, 34 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index b8698ebcfee..c8ca2d8e2d3 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -9,6 +9,7 @@ package org.elasticsearch.xpack.transform; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.Client; @@ -320,7 +321,10 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa logger.error("Error creating transform index template", e); } try { - templates.put(TransformInternalIndexConstants.AUDIT_INDEX, TransformInternalIndex.getAuditIndexTemplateMetaData()); + // Template upgraders are only ever called on the master nodes, so we can use the current node version as the compatibility + // version here because we can be sure that this node, if elected master, will be compatible with itself. + templates.put(TransformInternalIndexConstants.AUDIT_INDEX, + TransformInternalIndex.getAuditIndexTemplateMetaData(Version.CURRENT)); } catch (IOException e) { logger.warn("Error creating transform audit index", e); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java index 30ad79b9743..8f87d188d16 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.elasticsearch.xpack.transform.persistence.TransformInternalIndex.HIDDEN_INTRODUCED_VERSION; class TransformClusterStateListener implements ClusterStateListener { @@ -77,11 +78,16 @@ class TransformClusterStateListener implements ClusterStateListener { return; } + final IndicesAliasesRequest.AliasActions aliasAction = IndicesAliasesRequest.AliasActions.add() + .index(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED) + .alias(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS); + + if (state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION)) { + aliasAction.isHidden(true); + } + final IndicesAliasesRequest request = client.admin().indices().prepareAliases() - .addAliasAction(IndicesAliasesRequest.AliasActions.add() - .index(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED) - .alias(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS) - .isHidden(true)) + .addAliasAction(aliasAction) .request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), TRANSFORM_ORIGIN, request, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index e5f6b42f16c..baeff9ab172 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -39,6 +39,7 @@ import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformIn import java.io.IOException; import java.util.Collections; +import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_INDEX_HIDDEN; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN; @@ -79,6 +80,8 @@ public final class TransformInternalIndex { public static final String KEYWORD = "keyword"; public static final String BOOLEAN = "boolean"; + public static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0; + public static IndexTemplateMetaData getIndexTemplateMetaData() throws IOException { IndexTemplateMetaData transformTemplate = IndexTemplateMetaData.builder(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME) .patterns(Collections.singletonList(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)) @@ -97,19 +100,22 @@ public final class TransformInternalIndex { return transformTemplate; } - public static IndexTemplateMetaData getAuditIndexTemplateMetaData() throws IOException { + public static IndexTemplateMetaData getAuditIndexTemplateMetaData(Version compatibilityVersion) throws IOException { + final Settings.Builder auditIndexSettings = Settings.builder() + // the audits are expected to be small + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"); + final AliasMetaData.Builder alias = AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS); + if (compatibilityVersion.onOrAfter(HIDDEN_INTRODUCED_VERSION)) { + auditIndexSettings.put(SETTING_INDEX_HIDDEN, true); + alias.isHidden(true); + } IndexTemplateMetaData transformTemplate = IndexTemplateMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX) .patterns(Collections.singletonList(TransformInternalIndexConstants.AUDIT_INDEX_PREFIX + "*")) .version(Version.CURRENT.id) - .settings( - Settings.builder() - // the audits are expected to be small - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1") - .put(IndexMetaData.SETTING_INDEX_HIDDEN, true) - ) + .settings(auditIndexSettings) .putMapping(MapperService.SINGLE_MAPPING_NAME, Strings.toString(auditMappings())) - .putAlias(AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS).isHidden(true)) + .putAlias(alias) .build(); return transformTemplate; } @@ -406,7 +412,7 @@ public final class TransformInternalIndex { // Installing the template involves communication with the master node, so it's more expensive but much rarer try { - IndexTemplateMetaData indexTemplateMetaData = getAuditIndexTemplateMetaData(); + IndexTemplateMetaData indexTemplateMetaData = getAuditIndexTemplateMetaData(clusterService.state().nodes().getMinNodeVersion()); BytesReference jsonMappings = new BytesArray(indexTemplateMetaData.mappings().get(SINGLE_MAPPING_NAME).uncompressed()); PutIndexTemplateRequest request = new PutIndexTemplateRequest(TransformInternalIndexConstants.AUDIT_INDEX).patterns( indexTemplateMetaData.patterns() diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java index b49546d8f11..17bf9805b52 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.transform.persistence; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.AdminClient; @@ -55,7 +56,8 @@ public class TransformInternalIndexTests extends ESTestCase { mapBuilder = ImmutableOpenMap.builder(); try { - mapBuilder.put(TransformInternalIndexConstants.AUDIT_INDEX, TransformInternalIndex.getAuditIndexTemplateMetaData()); + mapBuilder.put(TransformInternalIndexConstants.AUDIT_INDEX, + TransformInternalIndex.getAuditIndexTemplateMetaData(Version.CURRENT)); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index ea7cc6cf4ac..713dac4aae5 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -128,7 +128,6 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase { * The purpose of this test is to ensure that when a job is open through a rolling upgrade we upgrade the results * index mappings when it is assigned to an upgraded node even if no other ML endpoint is called after the upgrade */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53931") public void testTransformRollingUpgrade() throws Exception { assumeTrue("Continuous transform time sync not fixed until 7.4", UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_4_0)); Request adjustLoggingLevels = new Request("PUT", "/_cluster/settings"); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_transform_jobs_crud.yml index 64c7d0ffcb1..7e7a12c1229 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_transform_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_transform_jobs_crud.yml @@ -1,8 +1,3 @@ -setup: - - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/53931" - --- "Test put batch transform on mixed cluster": - skip: diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml index c8afef5a46a..9103f95dbea 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml @@ -1,8 +1,4 @@ setup: - - skip: - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/53931" - - do: cluster.health: wait_for_status: green