honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "X-Pack: Add index.internal.format index setting (elastic/x-pack-elasticsearch#1700)" 

This reverts commit elastic/x-pack-elasticsearch@e430691c51.

The setting is going to be applied in core, so that only parts
of these changes need to be applied here (in a seperate PR).

Original commit: elastic/x-pack-elasticsearch@9e178c87fa
This commit is contained in:
Alexander Reelsen 2017-06-19 16:12:08 +02:00
parent 160fb8ee53
commit c1685c70ea
13 changed files with 16 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
plugin/src/main/java/org/elasticsearch/xpack/XPackPlugin.java
View File

@ -46,6 +46,7 @@ import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.ScriptPlugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ExecutorBuilder;
@ -106,7 +107,11 @@ import java.io.IOException;
import java.nio.file.Path;
import java.security.AccessController;
import java.security.GeneralSecurityException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.PrivilegedAction;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.time.Clock;
import java.util.ArrayList;
import java.util.Arrays;
@ -146,11 +151,6 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
// inside of YAML settings we still use xpack do not having handle issues with dashes
private static final String SETTINGS_NAME = "xpack";
// internal index format is used for upgrading
public static final Setting<Integer> INDEX_INTERNAL_FORMAT_SETTING =
Setting.intSetting("index.internal.format", 0, Setting.Property.IndexScope);
public static final int INTERNAL_INDEX_FORMAT = 6;
// TODO: clean up this library to not ask for write access to all system properties!
static {
// invoke this clinit in unbound with permissions to access all system properties
@ -359,8 +359,8 @@ public class XPackPlugin extends Plugin implements ScriptPlugin, ActionPlugin, I
settings.addAll(licensing.getSettings());
settings.addAll(XPackSettings.getAllSettings());
// an internal index format description, allowing us to find out if this index is upgraded or needs upgrading
settings.add(INDEX_INTERNAL_FORMAT_SETTING);
// we add the `xpack.version` setting to all internal indices
settings.add(Setting.simpleString("index.xpack.version", Setting.Property.IndexScope));
// notification services
settings.add(SlackService.SLACK_ACCOUNT_SETTING);

32
plugin/src/main/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleService.java
View File

@ -9,7 +9,6 @@ import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.routing.AllocationId;
import org.elasticsearch.cluster.routing.RoutingNode;
@ -20,8 +19,6 @@ import org.elasticsearch.common.component.LifecycleListener;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.gateway.GatewayService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
import org.elasticsearch.xpack.watcher.watch.Watch;
import org.elasticsearch.xpack.watcher.watch.WatchStoreUtils;
@ -95,10 +92,8 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
}
}
/**
*
* @param event The event of containing the new cluster state
*
/*
* TODO possible optimization
* stop certain parts of watcher, when there are no watcher indices on this node by checking the shardrouting
* note that this is not easily possible, because of the execute watch api, that needs to be able to execute anywhere!
* this means, only certain components can be stopped
@ -126,7 +121,6 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
DiscoveryNode localNode = event.state().nodes().getLocalNode();
RoutingNode routingNode = event.state().getRoutingNodes().node(localNode.getId());
IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(Watch.INDEX, event.state().metaData());
// no watcher index, time to pause, if we currently have shards here
if (watcherIndexMetaData == null) {
if (previousAllocationIds.get().isEmpty() == false) {
@ -157,32 +151,10 @@ public class WatcherLifeCycleService extends AbstractComponent implements Cluste
executor.execute(() -> watcherService.reload(event.state(), "different shard allocation ids"));
}
} else if (watcherService.state() != WatcherState.STARTED && watcherService.state() != WatcherState.STARTING) {
if (isIndexInternalFormat(event.state().metaData(), Watch.INDEX) &&
isIndexInternalFormat(event.state().metaData(), TriggeredWatchStore.INDEX_NAME)) {
executor.execute(() -> start(event.state(), false));
} else {
logger.error("Not starting watcher, the indices have not been upgraded yet. Please run the Upgrade API");
}
}
}
}
/**
* If the supplied index exists, ensure that the 'index.internal.format' setting is configured appropriately.
* This ensures that watcher can use this index as needed.
*
* @param metaData The cluster state meta data
* @param index The index name to check for. Aliases will be resolved properly.
* @return true if the index does not exist or the internal format is set properly
*/
boolean isIndexInternalFormat(MetaData metaData, String index) {
IndexMetaData watcherIndexMetaData = WatchStoreUtils.getConcreteIndex(index, metaData);
if (watcherIndexMetaData == null) {
return true;
} else {
return XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.get(watcherIndexMetaData.getSettings()) == XPackPlugin.INTERNAL_INDEX_FORMAT;
}
}
public WatcherMetaData watcherMetaData() {
return watcherMetaData;

1
plugin/src/main/resources/monitoring-alerts.json
View File

@ -5,7 +5,6 @@
"index": {
"number_of_shards": 1,
"number_of_replicas": 1,
"internal.format": 6,
"codec": "best_compression"
}
},

1
plugin/src/main/resources/monitoring-beats.json
View File

@ -4,7 +4,6 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.internal.format": 6,
"index.codec": "best_compression"
},
"mappings": {

3
plugin/src/main/resources/monitoring-es.json
View File

@ -4,8 +4,7 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.internal.format": 6
"index.codec": "best_compression"
},
"mappings": {
"doc": {

3
plugin/src/main/resources/monitoring-kibana.json
View File

@ -4,8 +4,7 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.internal.format": 6
"index.codec": "best_compression"
},
"mappings": {
"doc": {

3
plugin/src/main/resources/monitoring-logstash.json
View File

@ -4,8 +4,7 @@
"settings": {
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.internal.format": 6
"index.codec": "best_compression"
},
"mappings": {
"doc": {

1
plugin/src/main/resources/security-index-template.json
View File

@ -7,7 +7,6 @@
"number_of_replicas" : 0,
"auto_expand_replicas" : "0-all",
"index.priority": 1000,
"index.internal.format": 6,
"analysis" : {
"filter" : {
"email" : {

3
plugin/src/main/resources/security_audit_log.json
View File

@ -2,8 +2,7 @@
"index_patterns": ".security_audit_log*",
"order": 2147483647,
"settings": {
"index.mapper.dynamic" : false,
"index.internal.format": 6
"index.mapper.dynamic" : false
},
"mappings": {
"event": {

3
plugin/src/main/resources/triggered-watches.json
View File

@ -5,8 +5,7 @@
"index.number_of_shards": 1,
"index.mapper.dynamic" : false,
"index.refresh_interval" : "-1",
"index.priority": 900,
"index.internal.format": 6
"index.priority": 900
},
"mappings": {
"doc": {

1
plugin/src/main/resources/watch-history.json
View File

@ -3,7 +3,6 @@
"order": 2147483647,
"settings": {
"xpack.watcher.template.version": "${xpack.watcher.template.version}",
"index.internal.format": 6,
"index.number_of_shards": 1,
"index.mapper.dynamic": false
},

3
plugin/src/main/resources/watches.json
View File

@ -4,8 +4,7 @@
"settings": {
"index.number_of_shards": 1,
"index.mapper.dynamic" : false,
"index.priority": 800,
"index.internal.format": 6
"index.priority": 800
},
"mappings": {
"doc": {

62
plugin/src/test/java/org/elasticsearch/xpack/watcher/WatcherLifeCycleServiceTests.java
View File

@ -28,8 +28,6 @@ import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.watcher.execution.TriggeredWatchStore;
import org.elasticsearch.xpack.watcher.watch.Watch;
import org.junit.Before;
import org.mockito.stubbing.Answer;
@ -41,7 +39,6 @@ import java.util.concurrent.ExecutorService;
import static java.util.Arrays.asList;
import static org.elasticsearch.cluster.routing.ShardRoutingState.RELOCATING;
import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED;
import static org.hamcrest.Matchers.is;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
@ -355,65 +352,6 @@ public class WatcherLifeCycleServiceTests extends ESTestCase {
verify(watcherService, times(1)).pauseExecution(anyObject());
}
public void testWatcherDoesNotStartWithOldIndexFormat() throws Exception {
String index = randomFrom(Watch.INDEX, TriggeredWatchStore.INDEX_NAME);
Index watchIndex = new Index(index, "foo");
ShardId shardId = new ShardId(watchIndex, 0);
IndexRoutingTable watchRoutingTable = IndexRoutingTable.builder(watchIndex)
.addShard(TestShardRouting.newShardRouting(shardId, "node_1", true, STARTED)).build();
DiscoveryNodes nodes = new DiscoveryNodes.Builder().masterNodeId("node_1").localNodeId("node_1").add(newNode("node_1")).build();
Settings.Builder indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
if (randomBoolean()) {
int randomNumber = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1),
randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT, Integer.MAX_VALUE));
indexSettings.put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), randomNumber);
}
IndexMetaData.Builder newIndexMetaDataBuilder = IndexMetaData.builder(index).settings(indexSettings);
ClusterState clusterStateWithWatcherIndex = ClusterState.builder(new ClusterName("my-cluster"))
.nodes(nodes)
.routingTable(RoutingTable.builder().add(watchRoutingTable).build())
.metaData(MetaData.builder().put(newIndexMetaDataBuilder))
.build();
ClusterState emptyClusterState = ClusterState.builder(new ClusterName("my-cluster")).nodes(nodes).build();
when(watcherService.state()).thenReturn(WatcherState.STOPPED);
when(watcherService.validate(eq(clusterStateWithWatcherIndex))).thenReturn(true);
lifeCycleService.clusterChanged(new ClusterChangedEvent("any", clusterStateWithWatcherIndex, emptyClusterState));
verify(watcherService, never()).start(any(ClusterState.class));
}
public void testInternalIndexFormatCheck() {
int indexFormat = XPackPlugin.INTERNAL_INDEX_FORMAT;;
boolean expectedIndexInternalFormat = randomBoolean();
if (expectedIndexInternalFormat == false) {
indexFormat = randomFrom(randomIntBetween(0, XPackPlugin.INTERNAL_INDEX_FORMAT-1),
randomIntBetween(XPackPlugin.INTERNAL_INDEX_FORMAT+1, Integer.MAX_VALUE));
}
Settings.Builder indexSettings = Settings.builder()
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(XPackPlugin.INDEX_INTERNAL_FORMAT_SETTING.getKey(), indexFormat);
IndexMetaData.Builder metaDataBuilder = IndexMetaData.builder(Watch.INDEX).settings(indexSettings);
MetaData metaData = MetaData.builder().put(metaDataBuilder).build();
boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX);
assertThat(indexInternalFormat, is(expectedIndexInternalFormat));
}
public void testInternalIndexFormatCheckOnNonExistingIndex() {
MetaData metaData = MetaData.builder().build();
boolean indexInternalFormat = lifeCycleService.isIndexInternalFormat(metaData, Watch.INDEX);
assertThat(indexInternalFormat, is(true));
}
private static DiscoveryNode newNode(String nodeName) {
return new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), Collections.emptyMap(),
new HashSet<>(asList(DiscoveryNode.Role.values())), Version.CURRENT);