honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add version guards around Transform hidden index settings (#54036) 

This commit ensures that the hidden index settings are only applied to the
Transform index templates when the cluster can support those settings.

Also unmutes the tests which were failing due to the previous behavior.
This commit is contained in:
Gordon Brown 2020-03-24 15:52:56 -06:00 committed by GitHub
parent 55f2e8bff0
commit 82e041442e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java
View File

@ -9,6 +9,7 @@ package org.elasticsearch.xpack.transform;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.client.Client;
@ -320,7 +321,10 @@ public class Transform extends Plugin implements SystemIndexPlugin, PersistentTa
logger.error("Error creating transform index template", e);
}
try {
templates.put(TransformInternalIndexConstants.AUDIT_INDEX, TransformInternalIndex.getAuditIndexTemplateMetaData());
// Template upgraders are only ever called on the master nodes, so we can use the current node version as the compatibility
// version here because we can be sure that this node, if elected master, will be compatible with itself.
templates.put(TransformInternalIndexConstants.AUDIT_INDEX,
TransformInternalIndex.getAuditIndexTemplateMetaData(Version.CURRENT));
} catch (IOException e) {
logger.warn("Error creating transform audit index", e);
}

14
x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformClusterStateListener.java
View File

@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
import static org.elasticsearch.xpack.transform.persistence.TransformInternalIndex.HIDDEN_INTRODUCED_VERSION;
class TransformClusterStateListener implements ClusterStateListener {
@ -77,11 +78,16 @@ class TransformClusterStateListener implements ClusterStateListener {
return;
}
final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
.addAliasAction(IndicesAliasesRequest.AliasActions.add()
final IndicesAliasesRequest.AliasActions aliasAction = IndicesAliasesRequest.AliasActions.add()
.index(TransformInternalIndexConstants.AUDIT_INDEX_DEPRECATED)
.alias(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS)
.isHidden(true))
.alias(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS);
if (state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION)) {
aliasAction.isHidden(true);
}
final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
.addAliasAction(aliasAction)
.request();
executeAsyncWithOrigin(client.threadPool().getThreadContext(), TRANSFORM_ORIGIN, request,

26
x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java
View File

@ -39,6 +39,7 @@ import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformIn
import java.io.IOException;
import java.util.Collections;
import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_INDEX_HIDDEN;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.TRANSFORM_ORIGIN;
@ -79,6 +80,8 @@ public final class TransformInternalIndex {
public static final String KEYWORD = "keyword";
public static final String BOOLEAN = "boolean";
public static final Version HIDDEN_INTRODUCED_VERSION = Version.V_7_7_0;
public static IndexTemplateMetaData getIndexTemplateMetaData() throws IOException {
IndexTemplateMetaData transformTemplate = IndexTemplateMetaData.builder(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)
.patterns(Collections.singletonList(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME))
@ -97,19 +100,22 @@ public final class TransformInternalIndex {
return transformTemplate;
}
public static IndexTemplateMetaData getAuditIndexTemplateMetaData() throws IOException {
public static IndexTemplateMetaData getAuditIndexTemplateMetaData(Version compatibilityVersion) throws IOException {
final Settings.Builder auditIndexSettings = Settings.builder()
// the audits are expected to be small
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1");
final AliasMetaData.Builder alias = AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS);
if (compatibilityVersion.onOrAfter(HIDDEN_INTRODUCED_VERSION)) {
auditIndexSettings.put(SETTING_INDEX_HIDDEN, true);
alias.isHidden(true);
}
IndexTemplateMetaData transformTemplate = IndexTemplateMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX)
.patterns(Collections.singletonList(TransformInternalIndexConstants.AUDIT_INDEX_PREFIX + "*"))
.version(Version.CURRENT.id)
.settings(
Settings.builder()
// the audits are expected to be small
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_INDEX_HIDDEN, true)
)
.settings(auditIndexSettings)
.putMapping(MapperService.SINGLE_MAPPING_NAME, Strings.toString(auditMappings()))
.putAlias(AliasMetaData.builder(TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS).isHidden(true))
.putAlias(alias)
.build();
return transformTemplate;
}
@ -406,7 +412,7 @@ public final class TransformInternalIndex {
// Installing the template involves communication with the master node, so it's more expensive but much rarer
try {
IndexTemplateMetaData indexTemplateMetaData = getAuditIndexTemplateMetaData();
IndexTemplateMetaData indexTemplateMetaData = getAuditIndexTemplateMetaData(clusterService.state().nodes().getMinNodeVersion());
BytesReference jsonMappings = new BytesArray(indexTemplateMetaData.mappings().get(SINGLE_MAPPING_NAME).uncompressed());
PutIndexTemplateRequest request = new PutIndexTemplateRequest(TransformInternalIndexConstants.AUDIT_INDEX).patterns(
indexTemplateMetaData.patterns()

4
x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndexTests.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.transform.persistence;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.AdminClient;
@ -55,7 +56,8 @@ public class TransformInternalIndexTests extends ESTestCase {
mapBuilder = ImmutableOpenMap.builder();
try {
mapBuilder.put(TransformInternalIndexConstants.AUDIT_INDEX, TransformInternalIndex.getAuditIndexTemplateMetaData());
mapBuilder.put(TransformInternalIndexConstants.AUDIT_INDEX,
TransformInternalIndex.getAuditIndexTemplateMetaData(Version.CURRENT));
} catch (IOException e) {
throw new UncheckedIOException(e);
}

1
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
View File

@ -128,7 +128,6 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase {
* The purpose of this test is to ensure that when a job is open through a rolling upgrade we upgrade the results
* index mappings when it is assigned to an upgraded node even if no other ML endpoint is called after the upgrade
*/
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53931")
public void testTransformRollingUpgrade() throws Exception {
assumeTrue("Continuous transform time sync not fixed until 7.4", UPGRADE_FROM_VERSION.onOrAfter(Version.V_7_4_0));
Request adjustLoggingLevels = new Request("PUT", "/_cluster/settings");

5
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/80_transform_jobs_crud.yml
View File

@ -1,8 +1,3 @@
setup:
- skip:
version: "all"
reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/53931"
---
"Test put batch transform on mixed cluster":
- skip:

4
x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml
View File

@ -1,8 +1,4 @@
setup:
- skip:
version: "all"
reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/53931"
- do:
cluster.health:
wait_for_status: green