honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

X-Pack: Add index.internal.format index setting (elastic/x-pack-elasticsearch#1700) 

This introduces a new index setting called xpack.internal.format to
x-pack, which is configured for all of our index templates and set to
"v6". This indicates the version of compatibility of this index.

In addition a setting named index.xpack.version has been removed,
as it was unused.

Watcher does not start, if the watches and the triggered watches
index is not compatible with this setting.

Original commit: elastic/x-pack-elasticsearch@e430691c51
This commit is contained in:
Alexander Reelsen 2017-06-19 08:34:43 +02:00 committed by GitHub
parent b7d4600c3c
commit 65228e4379
13 changed files with 116 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java
View File

@ -46,7 +46,6 @@ import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ExecutorBuilder;
@ -107,11 +106,7 @@ import java.io.IOException;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.GeneralSecurityException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.PrivilegedAction;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
@ -151,6 +146,11 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
// inside of YAML settings we still use xpack do not having handle issues with dashes
private static final String SETTINGS_NAME = "xpack";
// internal index format is used for upgrading
public static final Setting<Integer> INDEX_INTERNAL_FORMAT_SETTING =
Setting.intSetting("index.internal.format", 0, Setting.Property.IndexScope);
public static final int INTERNAL_INDEX_FORMAT = 6;
// TODO: clean up this library to not ask for write access to all system properties!
static {
// invoke this clinit in unbound with permissions to access all system properties
@ -359,8 +359,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
settings.addAll(licensing.getSettings());
settings.addAll(XPackSettings.getAllSettings());
// we add the `xpack.version` setting to all internal indices
settings.add(Setting.simpleString("index.xpack.version", Setting.Property.IndexScope));
// an internal index format description, allowing us to find out if this index is upgraded or needs upgrading
settings.add(INDEX_INTERNAL_FORMAT_SETTING);
// notification services
settings.add(SlackService.SLACK_ACCOUNT_SETTING);

32
plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
View File

@ -9,6 +9,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.AllocationId;
import org.elasticsearch.cluster.routing.RoutingNode;
@ -19,6 +20,8 @@ import org.elasticsearch.common.component.LifecycleListener;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
import org.elasticsearch.xpack.watcher.watch.Watch;
import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils;
@ -92,8 +95,10 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
}
}
/*
* TODO possible optimization
/**
*
* @param event The event of containing the new cluster state
*
* stop certain parts of watcher, when there are no watcher indices on this node by checking the shardrouting
* note that this is not easily possible, because of the execute watch api, that needs to be able to execute anywhere!
* this means, only certain components can be stopped
@ -121,6 +126,7 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
DiscoveryNode localNode = event.state().nodes().getLocalNode();
RoutingNode routingNode = event.state().getRoutingNodes().node(localNode.getId());
IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(Watch.INDEX, event.state().metaData());
// no watcher index, time to pause, if we currently have shards here
if (watcherIndexMetaData == null) {
if (previousAllocationIds.get().isEmpty() == false) {
@ -151,10 +157,32 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
executor.execute(() -> watcherService.reload(event.state(), "different shard allocation ids"));
}
} else if (watcherService.state() != WatcherState.STARTED && watcherService.state() != WatcherState.STARTING) {
if (isIndexInternalFormat(event.state().metaData(), Watch.INDEX) &&
isIndexInternalFormat(event.state().metaData(), TriggeredWatchStore.INDEX_NAME)) {
executor.execute(() -> start(event.state(), false));
} else {
logger.error("Not starting watcher, the indices have not been upgraded yet. Please run the Upgrade API");
}
}
}
}
/**
* If the supplied index exists, ensure that the 'index.internal.format' setting is configured appropriately.
* This ensures that watcher can use this index as needed.
*
* @param metaData The cluster state meta data
* @param index The index name to check for. Aliases will be resolved properly.
* @return true if the index does not exist or the internal format is set properly
*/
boolean isIndexInternalFormat(MetaData metaData, String index) {
IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(index, metaData);
if (watcherIndexMetaData == null) {
return true;
} else {
return XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.get(watcherIndexMetaData.getSettings()) == XPackPlugin.INTERNAL_INDEX_FORMAT;
}
}
public WatcherMetaData watcherMetaData() {
return watcherMetaData;

1
plugin/src/main/resources/monitoring-alerts.json
View File

@ -5,6 +5,7 @@
"index": {
"number_of_shards": 1,
"number_of_replicas": 1,
"internal.format": 6,
"codec": "best_compression"
}
},

1
plugin/src/main/resources/monitoring-beats.json
View File

@ -4,6 +4,7 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.internal.format": 6,
"index.codec": "best_compression"
},
"mappings": {

3
plugin/src/main/resources/monitoring-es.json
View File

@ -4,7 +4,8 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression"
"index.codec": "best_compression",
"index.internal.format": 6
},
"mappings": {
"doc": {

3
plugin/src/main/resources/monitoring-kibana.json
View File

@ -4,7 +4,8 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression"
"index.codec": "best_compression",
"index.internal.format": 6
},
"mappings": {
"doc": {

3
plugin/src/main/resources/monitoring-logstash.json
View File

@ -4,7 +4,8 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression"
"index.codec": "best_compression",
"index.internal.format": 6
},
"mappings": {
"doc": {

1
plugin/src/main/resources/security-index-template.json
View File

@ -7,6 +7,7 @@
"number_of_replicas" : 0,
"auto_expand_replicas" : "0-all",
"index.priority": 1000,
"index.internal.format": 6,
"analysis" : {
"filter" : {
"email" : {

3
plugin/src/main/resources/security_audit_log.json
View File

@ -2,7 +2,8 @@
"index_patterns": ".security_audit_log*",
"order": 2147483647,
"settings": {
"index.mapper.dynamic" : false
"index.mapper.dynamic" : false,
"index.internal.format": 6
},
"mappings": {
"event": {

3
plugin/src/main/resources/triggered-watches.json
View File

@ -5,7 +5,8 @@
"index.number_of_shards": 1,
"index.mapper.dynamic" : false,
"index.refresh_interval" : "-1",
"index.priority": 900
"index.priority": 900,
"index.internal.format": 6
},
"mappings": {
"doc": {

1
plugin/src/main/resources/watch-history.json
View File

@ -3,6 +3,7 @@
"order": 2147483647,
"settings": {
"xpack.watcher.template.version": "${xpack.watcher.template.version}",
"index.internal.format": 6,
"index.number_of_shards": 1,
"index.mapper.dynamic": false
},

3
plugin/src/main/resources/watches.json
View File

@ -4,7 +4,8 @@
"settings": {
"index.number_of_shards": 1,
"index.mapper.dynamic" : false,
"index.priority": 800
"index.priority": 800,
"index.internal.format": 6
},
"mappings": {
"doc": {

62
plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java
View File

@ -28,6 +28,8 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
import org.elasticsearch.xpack.watcher.watch.Watch;
import org.junit.Before;
import org.mockito.stubbing.Answer;
@ -39,6 +41,7 @@ import java.util.concurrent.ExecutorService;
import static java.util.Arrays.asList;
import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
import static org.hamcrest.Matchers.is;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
@ -352,6 +355,65 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
verify(watcherService, times(1)).pauseExecution(anyObject());
}
public void testWatcherDoesNotStartWithOldIndexFormat() throws Exception {
String index = randomFrom(Watch.INDEX, TriggeredWatchStore.INDEX_NAME);
Index watchIndex = new Index(index, "foo");
ShardId shardId = new ShardId(watchIndex, 0);
IndexRoutingTable watchRoutingTable = IndexRoutingTable.builder(watchIndex)
.addShard(TestShardRouting.newShardRouting(shardId, "node_1", true, STARTED)).build();
DiscoveryNodes nodes = new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")).build();
Settings.Builder indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
if (randomBoolean()) {
int randomNumber = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1),
randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT, Integer.MAX_VALUE));
indexSettings.put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), randomNumber);
}
IndexMetaData.Builder newIndexMetaDataBuilder = IndexMetaData.builder(index).settings(indexSettings);
ClusterState clusterStateWithWatcherIndex = ClusterState.builder(new ClusterName("my-cluster"))
.nodes(nodes)
.routingTable(RoutingTable.builder().add(watchRoutingTable).build())
.metaData(MetaData.builder().put(newIndexMetaDataBuilder))
.build();
ClusterState emptyClusterState = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).build();
when(watcherService.state()).thenReturn(WatcherState.STOPPED);
when(watcherService.validate(eq(clusterStateWithWatcherIndex))).thenReturn(true);
lifeCycleService.clusterChanged(new ClusterChangedEvent("any", clusterStateWithWatcherIndex, emptyClusterState));
verify(watcherService, never()).start(any(ClusterState.class));
}
public void testInternalIndexFormatCheck() {
int indexFormat = XPackPlugin.INTERNAL_INDEX_FORMAT;;
boolean expectedIndexInternalFormat = randomBoolean();
if (expectedIndexInternalFormat == false) {
indexFormat = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1),
randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT+1, Integer.MAX_VALUE));
}
Settings.Builder indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), indexFormat);
IndexMetaData.Builder metaDataBuilder = IndexMetaData.builder(Watch.INDEX).settings(indexSettings);
MetaData metaData = MetaData.builder().put(metaDataBuilder).build();
boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX);
assertThat(indexInternalFormat, is(expectedIndexInternalFormat));
}
public void testInternalIndexFormatCheckOnNonExistingIndex() {
MetaData metaData = MetaData.builder().build();
boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX);
assertThat(indexInternalFormat, is(true));
}
private static DiscoveryNode newNode(String nodeName) {
return new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);