From e5c89def42ccdd8eb8707c0bfaa18754e302a602 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 1 Jul 2012 18:16:04 +0200 Subject: [PATCH] Support wildcard and +/- notation for multi index APIs, closes #2074. --- .../cluster/metadata/AliasMetaData.java | 4 + .../cluster/metadata/IndexMetaData.java | 4 + .../cluster/metadata/MetaData.java | 254 ++++++++++++------ .../unit/cluster/metadata/MetaDataTests.java | 68 +++++ 4 files changed, 242 insertions(+), 88 deletions(-) create mode 100644 src/test/java/org/elasticsearch/test/unit/cluster/metadata/MetaDataTests.java diff --git a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java index 1ac5f56e740..0ea4d9f68c8 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java @@ -84,6 +84,10 @@ public class AliasMetaData { return indexRouting; } + public static Builder builder(String alias) { + return new Builder(alias); + } + public static Builder newAliasMetaDataBuilder(String alias) { return new Builder(alias); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 7f3adef50a1..101dbbe6c94 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -368,6 +368,10 @@ public class IndexMetaData { return result; } + public static Builder builder(String index) { + return new Builder(index); + } + public static Builder newIndexMetaDataBuilder(String index) { return new Builder(index); } diff --git a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index 0a0a80d1450..ff63af0ffb7 100644 --- a/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -19,7 +19,10 @@ package org.elasticsearch.cluster.metadata; -import com.google.common.collect.*; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.UnmodifiableIterator; import gnu.trove.set.hash.THashSet; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.cluster.block.ClusterBlock; @@ -303,20 +306,6 @@ public class MetaData implements Iterable { return allOpenIndices; } - /** - * Translates the provided indices (possibly aliased) into actual indices. - */ - public String[] concreteIndices(String[] indices) throws IndexMissingException { - return concreteIndices(indices, false, false); - } - - /** - * Translates the provided indices (possibly aliased) into actual indices. - */ - public String[] concreteIndicesIgnoreMissing(String[] indices) { - return concreteIndices(indices, true, false); - } - /** * Returns indexing routing for the given index. */ @@ -346,83 +335,29 @@ public class MetaData implements Iterable { return routing; } - /** - * Sets the same routing for all indices - */ - private Map> resolveSearchRoutingAllIndices(String routing) { - if (routing != null) { - Set r = Strings.splitStringByCommaToSet(routing); - Map> routings = newHashMap(); - String[] concreteIndices = concreteAllIndices(); - for (String index : concreteIndices) { - routings.put(index, r); - } - return routings; - } - return null; - } - public Map> resolveSearchRouting(@Nullable String routing, String aliasOrIndex) { - Map> routings = null; - Set paramRouting = null; - if (routing != null) { - paramRouting = Strings.splitStringByCommaToSet(routing); - } - - ImmutableMap> indexToRoutingMap = aliasToIndexToSearchRoutingMap.get(aliasOrIndex); - if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) { - // It's an alias - for (Map.Entry> indexRouting : indexToRoutingMap.entrySet()) { - if (!indexRouting.getValue().isEmpty()) { - // Routing alias - Set r = new THashSet(indexRouting.getValue()); - if (paramRouting != null) { - r.retainAll(paramRouting); - } - if (!r.isEmpty()) { - if (routings == null) { - routings = newHashMap(); - } - routings.put(indexRouting.getKey(), r); - } - } else { - // Non-routing alias - if (paramRouting != null) { - Set r = new THashSet(paramRouting); - if (routings == null) { - routings = newHashMap(); - } - routings.put(indexRouting.getKey(), r); - } - } - } - } else { - // It's an index - if (paramRouting != null) { - routings = ImmutableMap.of(aliasOrIndex, paramRouting); - } - } - return routings; + return resolveSearchRouting(routing, convertFromWildcards(new String[]{aliasOrIndex}, true, true)); } - public Map> resolveSearchRouting(@Nullable String routing, String[] aliasesOrIndices) { if (aliasesOrIndices == null || aliasesOrIndices.length == 0) { return resolveSearchRoutingAllIndices(routing); } + aliasesOrIndices = convertFromWildcards(aliasesOrIndices, true, true); + if (aliasesOrIndices.length == 1) { if (aliasesOrIndices[0].equals("_all")) { return resolveSearchRoutingAllIndices(routing); } else { - return resolveSearchRouting(routing, aliasesOrIndices[0]); + return resolveSearchRoutingSingleValue(routing, aliasesOrIndices[0]); } } Map> routings = null; Set paramRouting = null; // List of indices that don't require any routing - Set norouting = newHashSet(); + Set norouting = new THashSet(); if (routing != null) { paramRouting = Strings.splitStringByCommaToSet(routing); } @@ -493,30 +428,104 @@ public class MetaData implements Iterable { return routings; } + private Map> resolveSearchRoutingSingleValue(@Nullable String routing, String aliasOrIndex) { + Map> routings = null; + Set paramRouting = null; + if (routing != null) { + paramRouting = Strings.splitStringByCommaToSet(routing); + } + + ImmutableMap> indexToRoutingMap = aliasToIndexToSearchRoutingMap.get(aliasOrIndex); + if (indexToRoutingMap != null && !indexToRoutingMap.isEmpty()) { + // It's an alias + for (Map.Entry> indexRouting : indexToRoutingMap.entrySet()) { + if (!indexRouting.getValue().isEmpty()) { + // Routing alias + Set r = new THashSet(indexRouting.getValue()); + if (paramRouting != null) { + r.retainAll(paramRouting); + } + if (!r.isEmpty()) { + if (routings == null) { + routings = newHashMap(); + } + routings.put(indexRouting.getKey(), r); + } + } else { + // Non-routing alias + if (paramRouting != null) { + Set r = new THashSet(paramRouting); + if (routings == null) { + routings = newHashMap(); + } + routings.put(indexRouting.getKey(), r); + } + } + } + } else { + // It's an index + if (paramRouting != null) { + routings = ImmutableMap.of(aliasOrIndex, paramRouting); + } + } + return routings; + } + + /** + * Sets the same routing for all indices + */ + private Map> resolveSearchRoutingAllIndices(String routing) { + if (routing != null) { + Set r = Strings.splitStringByCommaToSet(routing); + Map> routings = newHashMap(); + String[] concreteIndices = concreteAllIndices(); + for (String index : concreteIndices) { + routings.put(index, r); + } + return routings; + } + return null; + } + /** * Translates the provided indices (possibly aliased) into actual indices. */ - public String[] concreteIndices(String[] indices, boolean ignoreMissing, boolean allOnlyOpen) throws IndexMissingException { - if (indices == null || indices.length == 0) { + public String[] concreteIndices(String[] indices) throws IndexMissingException { + return concreteIndices(indices, false, false); + } + + /** + * Translates the provided indices (possibly aliased) into actual indices. + */ + public String[] concreteIndicesIgnoreMissing(String[] indices) { + return concreteIndices(indices, true, false); + } + + /** + * Translates the provided indices (possibly aliased) into actual indices. + */ + public String[] concreteIndices(String[] aliasesOrIndices, boolean ignoreMissing, boolean allOnlyOpen) throws IndexMissingException { + if (aliasesOrIndices == null || aliasesOrIndices.length == 0) { return allOnlyOpen ? concreteAllOpenIndices() : concreteAllIndices(); } + aliasesOrIndices = convertFromWildcards(aliasesOrIndices, allOnlyOpen, false); // optimize for single element index (common case) - if (indices.length == 1) { - String index = indices[0]; - if (index.length() == 0) { + if (aliasesOrIndices.length == 1) { + String aliasOrIndex = aliasesOrIndices[0]; + if (aliasOrIndex.length() == 0) { return allOnlyOpen ? concreteAllOpenIndices() : concreteAllIndices(); } - if (index.equals("_all")) { + if (aliasOrIndex.equals("_all")) { return allOnlyOpen ? concreteAllOpenIndices() : concreteAllIndices(); } // if a direct index name, just return the array provided - if (this.indices.containsKey(index)) { - return indices; + if (this.indices.containsKey(aliasOrIndex)) { + return aliasesOrIndices; } - String[] actualLst = aliasAndIndexToIndexMap.get(index); + String[] actualLst = aliasAndIndexToIndexMap.get(aliasOrIndex); if (actualLst == null) { if (!ignoreMissing) { - throw new IndexMissingException(new Index(index)); + throw new IndexMissingException(new Index(aliasOrIndex)); } else { return Strings.EMPTY_ARRAY; } @@ -528,18 +537,18 @@ public class MetaData implements Iterable { // check if its a possible aliased index, if not, just return the // passed array boolean possiblyAliased = false; - for (String index : indices) { + for (String index : aliasesOrIndices) { if (!this.indices.containsKey(index)) { possiblyAliased = true; break; } } if (!possiblyAliased) { - return indices; + return aliasesOrIndices; } - Set actualIndices = Sets.newHashSetWithExpectedSize(indices.length); - for (String index : indices) { + Set actualIndices = new THashSet(); + for (String index : aliasesOrIndices) { String[] actualLst = aliasAndIndexToIndexMap.get(index); if (actualLst == null) { if (!ignoreMissing) { @@ -570,6 +579,75 @@ public class MetaData implements Iterable { return lst[0]; } + public String[] convertFromWildcards(String[] aliasesOrIndices, boolean wildcardOnlyOpen, boolean ignoreMissing) { + Set result = null; + for (int i = 0; i < aliasesOrIndices.length; i++) { + String aliasOrIndex = aliasesOrIndices[i]; + if (aliasAndIndexToIndexMap.containsKey(aliasOrIndex)) { + if (result != null) { + result.add(aliasOrIndex); + } + continue; + } + boolean add = true; + if (aliasOrIndex.charAt(0) == '+') { + add = true; + aliasOrIndex = aliasOrIndex.substring(1); + } else if (aliasOrIndex.charAt(0) == '-') { + // if its the first, fill it with all the indices... + if (i == 0) { + result = new THashSet(Arrays.asList(wildcardOnlyOpen ? concreteAllOpenIndices() : concreteAllIndices())); + } + add = false; + aliasOrIndex = aliasOrIndex.substring(1); + } + if (!Regex.isSimpleMatchPattern(aliasOrIndex)) { + if (result != null) { + if (add) { + result.add(aliasOrIndex); + } else { + result.remove(aliasOrIndex); + } + } + continue; + } + if (result == null) { + // add all the previous ones... + result = new THashSet(); + result.addAll(Arrays.asList(aliasesOrIndices).subList(0, i)); + } + String[] indices = wildcardOnlyOpen ? concreteAllOpenIndices() : concreteAllIndices(); + boolean found = false; + for (String index : indices) { + if (Regex.simpleMatch(aliasOrIndex, index)) { + found = true; + if (add) { + result.add(index); + } else { + result.remove(index); + } + } + } + for (String alias : aliases.keySet()) { + if (Regex.simpleMatch(aliasOrIndex, alias)) { + found = true; + if (add) { + result.add(alias); + } else { + result.remove(alias); + } + } + } + if (!found && !ignoreMissing) { + throw new IndexMissingException(new Index(aliasOrIndex)); + } + } + if (result == null) { + return aliasesOrIndices; + } + return result.toArray(new String[result.size()]); + } + public boolean hasIndex(String index) { return indices.containsKey(index); } diff --git a/src/test/java/org/elasticsearch/test/unit/cluster/metadata/MetaDataTests.java b/src/test/java/org/elasticsearch/test/unit/cluster/metadata/MetaDataTests.java new file mode 100644 index 00000000000..519c7616f72 --- /dev/null +++ b/src/test/java/org/elasticsearch/test/unit/cluster/metadata/MetaDataTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.unit.cluster.metadata; + +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.testng.annotations.Test; + +import static com.google.common.collect.Sets.newHashSet; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + */ +public class MetaDataTests { + + @Test + public void convertWildcardsJustIndicesTests() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("testXXX")) + .put(indexBuilder("testXYY")) + .put(indexBuilder("testYYY")) + .put(indexBuilder("kuku")); + MetaData md = mdBuilder.build(); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"testXXX"}, true, true)), equalTo(newHashSet("testXXX"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"testXXX", "testYYY"}, true, true)), equalTo(newHashSet("testXXX", "testYYY"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"testXXX", "ku*"}, true, true)), equalTo(newHashSet("testXXX", "kuku"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"test*"}, true, true)), equalTo(newHashSet("testXXX", "testXYY", "testYYY"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"testX*"}, true, true)), equalTo(newHashSet("testXXX", "testXYY"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"testX*", "kuku"}, true, true)), equalTo(newHashSet("testXXX", "testXYY", "kuku"))); + } + + @Test + public void convertWildcardsTests() { + MetaData.Builder mdBuilder = MetaData.builder() + .put(indexBuilder("testXXX").putAlias(AliasMetaData.builder("alias1")).putAlias(AliasMetaData.builder("alias2"))) + .put(indexBuilder("testXYY").putAlias(AliasMetaData.builder("alias2"))) + .put(indexBuilder("testYYY").putAlias(AliasMetaData.builder("alias3"))) + .put(indexBuilder("kuku")); + MetaData md = mdBuilder.build(); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"testYY*", "alias*"}, true, true)), equalTo(newHashSet("alias1", "alias2", "alias3", "testYYY"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"-kuku"}, true, true)), equalTo(newHashSet("testXXX", "testXYY", "testYYY"))); + assertThat(newHashSet(md.convertFromWildcards(new String[]{"+test*", "-testYYY"}, true, true)), equalTo(newHashSet("testXXX", "testXYY"))); + } + + private IndexMetaData.Builder indexBuilder(String index) { + return IndexMetaData.builder(index).settings(ImmutableSettings.settingsBuilder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)); + } +}