honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add IndexAliasService

This commit is contained in:
Igor Motov 2011-05-21 19:53:17 -04:00 committed by kimchy
parent 10de33c94e
commit 3d5d25b03b
10 changed files with 446 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/AliasMetaData.java
View File

@ -77,6 +77,7 @@ public class AliasMetaData {
public Builder(AliasMetaData aliasMetaData) {
this(aliasMetaData.alias());
filter = aliasMetaData.filter();
}
public String alias() {

58
modules/elasticsearch/src/main/java/org/elasticsearch/index/aliases/IndexAlias.java Normal file
View File

@ -0,0 +1,58 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.aliases;
import org.apache.lucene.search.Filter;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.compress.CompressedString;
/**
* @author imotov
*/
public class IndexAlias {
private String alias;
private CompressedString filter;
private Filter parsedFilter;
public IndexAlias(String alias, @Nullable CompressedString filter, @Nullable Filter parsedFilter) {
this.alias = alias;
this.filter = filter;
this.parsedFilter = parsedFilter;
}
public String alias() {
return alias;
}
@Nullable
public CompressedString filter() {
return filter;
}
@Nullable
public Filter parsedFilter() {
return parsedFilter;
}
}

140
modules/elasticsearch/src/main/java/org/elasticsearch/index/aliases/IndexAliasesService.java Normal file
View File

@ -0,0 +1,140 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.aliases;
import org.apache.lucene.search.*;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.UnmodifiableIterator;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.search.XBooleanFilter;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.query.xcontent.XContentIndexQueryParser;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.indices.AliasFilterParsingException;
import java.io.IOException;
import java.util.List;
import static org.elasticsearch.common.collect.Lists.newArrayList;
import static org.elasticsearch.common.collect.MapBuilder.newMapBuilder;
/**
* @author imotov
*/
public class IndexAliasesService extends AbstractIndexComponent implements Iterable<IndexAlias> {
private final IndexQueryParserService indexQueryParserService;
private volatile ImmutableMap<String, IndexAlias> aliases = ImmutableMap.of();
private final Object mutex = new Object();
@Inject public IndexAliasesService(Index index, @IndexSettings Settings indexSettings, IndexQueryParserService indexQueryParserService) {
super(index, indexSettings);
this.indexQueryParserService = indexQueryParserService;
}
public boolean hasAlias(String alias) {
return aliases.containsKey(alias);
}
public IndexAlias alias(String alias) {
return aliases.get(alias);
}
public void add(String alias, @Nullable CompressedString filter) {
add(new IndexAlias(alias, filter, parse(alias, filter)));
}
public Filter aliasFilter(String... indices) {
List<Filter> filters = null;
for (String alias : indices) {
// The list contains the index itself - no filtering needed
if (alias.equals(index.name())) {
return null;
}
IndexAlias indexAlias = aliases.get(alias);
if (indexAlias != null) {
// The list contains a non-filtering alias - no filtering needed
if (indexAlias.parsedFilter() == null) {
return null;
} else {
if (filters == null) {
filters = newArrayList();
}
filters.add(indexAlias.parsedFilter());
}
}
}
if (filters == null) {
return null;
}
if (filters.size() == 1) {
return filters.get(0);
} else {
XBooleanFilter combined = new XBooleanFilter();
for (Filter filter : filters) {
combined.add(new FilterClause(filter, BooleanClause.Occur.SHOULD));
}
return combined;
}
}
private void add(IndexAlias indexAlias) {
synchronized (mutex) {
aliases = newMapBuilder(aliases).put(indexAlias.alias(), indexAlias).immutableMap();
}
}
public void remove(String alias) {
synchronized (mutex) {
aliases = newMapBuilder(aliases).remove(alias).immutableMap();
}
}
private Filter parse(String alias, CompressedString filter) {
if (filter == null) {
return null;
}
XContentIndexQueryParser indexQueryParser = (XContentIndexQueryParser) indexQueryParserService.defaultIndexQueryParser();
try {
byte[] filterSource = filter.uncompressed();
XContentParser parser = XContentFactory.xContent(filterSource).createParser(filterSource);
try {
return indexQueryParser.parseInnerFilter(parser);
} finally {
parser.close();
}
} catch (IOException ex) {
throw new AliasFilterParsingException(index, alias, "Invalid alias filter", ex);
}
}
@Override public UnmodifiableIterator<IndexAlias> iterator() {
return aliases.values().iterator();
}
}

32
modules/elasticsearch/src/main/java/org/elasticsearch/index/aliases/IndexAliasesServiceModule.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.aliases;
import org.elasticsearch.common.inject.AbstractModule;
/**
* @author imotov
*/
public class IndexAliasesServiceModule extends AbstractModule {
@Override protected void configure() {
bind(IndexAliasesService.class).asEagerSingleton();
}
}

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/service/IndexService.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.ImmutableSet;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.index.IndexComponent;
import org.elasticsearch.index.IndexShardMissingException;
import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.IndexEngine;
@ -56,6 +57,8 @@ public interface IndexService extends IndexComponent, Iterable<IndexShard> {
SimilarityService similarityService();
IndexAliasesService aliasesService();
IndexEngine engine();
IndexStore store();

11
modules/elasticsearch/src/main/java/org/elasticsearch/index/service/InternalIndexService.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.gateway.none.NoneGateway;
import org.elasticsearch.index.*;
import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.deletionpolicy.DeletionPolicyModule;
@ -103,6 +104,8 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
private final SimilarityService similarityService;
private final IndexAliasesService aliasesService;
private final IndexCache indexCache;
private final IndexEngine indexEngine;
@ -118,7 +121,8 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
private volatile boolean closed = false;
@Inject public InternalIndexService(Injector injector, Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv, ThreadPool threadPool,
PercolatorService percolatorService, AnalysisService analysisService, MapperService mapperService, IndexQueryParserService queryParserService, SimilarityService similarityService,
PercolatorService percolatorService, AnalysisService analysisService, MapperService mapperService,
IndexQueryParserService queryParserService, SimilarityService similarityService, IndexAliasesService aliasesService,
IndexCache indexCache, IndexEngine indexEngine, IndexGateway indexGateway, IndexStore indexStore) {
super(index, indexSettings);
this.injector = injector;
@ -130,6 +134,7 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
this.mapperService = mapperService;
this.queryParserService = queryParserService;
this.similarityService = similarityService;
this.aliasesService = aliasesService;
this.indexCache = indexCache;
this.indexEngine = indexEngine;
this.indexGateway = indexGateway;
@ -203,6 +208,10 @@ public class InternalIndexService extends AbstractIndexComponent implements Inde
return similarityService;
}
@Override public IndexAliasesService aliasesService() {
return aliasesService;
}
@Override public IndexEngine engine() {
return indexEngine;
}

39
modules/elasticsearch/src/main/java/org/elasticsearch/indices/AliasFilterParsingException.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.indices;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexException;
/**
* @author imotov
*/
public class AliasFilterParsingException extends IndexException {
public AliasFilterParsingException(Index index, String name, String desc) {
super(index, "[" + name + "], " + desc);
}
public AliasFilterParsingException(Index index, String name, String desc, Throwable ex) {
super(index, "[" + name + "], " + desc, ex);
}
}

2
modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.common.util.concurrent.ThreadSafe;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.gateway.Gateway;
import org.elasticsearch.index.*;
import org.elasticsearch.index.aliases.IndexAliasesServiceModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.analysis.AnalysisService;
import org.elasticsearch.index.cache.CacheStats;
@ -242,6 +243,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
modules.add(new IndexCacheModule(indexSettings));
modules.add(new IndexQueryParserModule(indexSettings));
modules.add(new MapperServiceModule());
modules.add(new IndexAliasesServiceModule());
modules.add(new IndexGatewayModule(indexSettings, injector.getInstance(Gateway.class)));
modules.add(new IndexModule());
modules.add(new PercolatorModule());

49
modules/elasticsearch/src/main/java/org/elasticsearch/indices/cluster/IndicesClusterStateService.java
View File

@ -29,6 +29,7 @@ import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
import org.elasticsearch.cluster.action.index.NodeMappingCreatedAction;
import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
import org.elasticsearch.cluster.action.shard.ShardStateAction;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
@ -44,6 +45,8 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.index.IndexShardAlreadyExistsException;
import org.elasticsearch.index.IndexShardMissingException;
import org.elasticsearch.index.aliases.IndexAlias;
import org.elasticsearch.index.aliases.IndexAliasesService;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.gateway.IndexShardGatewayRecoveryException;
import org.elasticsearch.index.gateway.IndexShardGatewayService;
@ -143,6 +146,7 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
synchronized (mutex) {
applyNewIndices(event);
applyMappings(event);
applyAliases(event);
applyNewOrUpdatedShards(event);
applyDeletedIndices(event);
applyDeletedShards(event);
@ -368,6 +372,51 @@ public class IndicesClusterStateService extends AbstractLifecycleComponent<Indic
return requiresRefresh;
}
private void applyAliases(ClusterChangedEvent event) {
// go over and update aliases
for (IndexMetaData indexMetaData : event.state().metaData()) {
if (!indicesService.hasIndex(indexMetaData.index())) {
// we only create / update here
continue;
}
String index = indexMetaData.index();
IndexService indexService = indicesService.indexService(index);
IndexAliasesService indexAliasesService = indexService.aliasesService();
for (AliasMetaData aliasesMd : indexMetaData.aliases().values()) {
processAlias(index, aliasesMd.alias(), aliasesMd.filter(), indexAliasesService);
}
// go over and remove aliases
for (IndexAlias indexAlias : indexAliasesService) {
if (!indexMetaData.aliases().containsKey(indexAlias.alias())) {
// we have it in our aliases, but not in the metadata, remove it
indexAliasesService.remove(indexAlias.alias());
}
}
}
}
private void processAlias(String index, String alias, CompressedString filter, IndexAliasesService indexAliasesService) {
try {
if (!indexAliasesService.hasAlias(alias)) {
if (logger.isDebugEnabled()) {
logger.debug("[{}] adding alias [{}], filter [{}]", index, alias, filter);
}
indexAliasesService.add(alias, filter);
} else {
if ((filter == null && indexAliasesService.alias(alias).filter() != null) ||
(filter != null && !filter.equals(indexAliasesService.alias(alias).filter()))) {
if (logger.isDebugEnabled()) {
logger.debug("[{}] updating alias [{}], filter [{}]", index, alias, filter);
}
indexAliasesService.add(alias, filter);
}
}
} catch (Exception e) {
logger.warn("[{}] failed to add alias [{}], filter [{}]", e, index, alias, filter);
}
}
private void applyNewOrUpdatedShards(final ClusterChangedEvent event) throws ElasticSearchException {
if (!indicesService.changesAllowed())
return;

112
modules/elasticsearch/src/test/java/org/elasticsearch/index/aliases/IndexAliasesServiceTests.java Normal file
View File

@ -0,0 +1,112 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.aliases;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.inject.Injector;
import org.elasticsearch.common.inject.ModulesBuilder;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.SettingsModule;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNameModule;
import org.elasticsearch.index.analysis.AnalysisModule;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.engine.IndexEngineModule;
import org.elasticsearch.index.query.IndexQueryParserModule;
import org.elasticsearch.index.query.IndexQueryParserService;
import org.elasticsearch.index.query.xcontent.XContentFilterBuilder;
import org.elasticsearch.index.settings.IndexSettingsModule;
import org.elasticsearch.index.similarity.SimilarityModule;
import org.elasticsearch.script.ScriptModule;
import org.testng.annotations.Test;
import java.io.IOException;
import static org.elasticsearch.index.query.xcontent.FilterBuilders.termFilter;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;
/**
* @author imotov
*/
public class IndexAliasesServiceTests {
public static IndexAliasesService newIndexAliasesService() {
return new IndexAliasesService(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS, newIndexQueryParserService());
}
public static IndexQueryParserService newIndexQueryParserService() {
Injector injector = new ModulesBuilder().add(
new IndexSettingsModule(new Index("test"), ImmutableSettings.Builder.EMPTY_SETTINGS),
new IndexNameModule(new Index("test")),
new IndexQueryParserModule(ImmutableSettings.Builder.EMPTY_SETTINGS),
new AnalysisModule(ImmutableSettings.Builder.EMPTY_SETTINGS),
new SimilarityModule(ImmutableSettings.Builder.EMPTY_SETTINGS),
new ScriptModule(ImmutableSettings.Builder.EMPTY_SETTINGS),
new SettingsModule(ImmutableSettings.Builder.EMPTY_SETTINGS),
new IndexEngineModule(ImmutableSettings.Builder.EMPTY_SETTINGS),
new IndexCacheModule(ImmutableSettings.Builder.EMPTY_SETTINGS)
).createInjector();
return injector.getInstance(IndexQueryParserService.class);
}
public static CompressedString filter(XContentFilterBuilder filterBuilder) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder();
filterBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.close();
return new CompressedString(builder.string());
}
@Test public void testAliasFilter() throws Exception {
IndexAliasesService indexAliasesService = newIndexAliasesService();
indexAliasesService.add("cats", filter(termFilter("animal", "cat")));
indexAliasesService.add("dogs", filter(termFilter("animal", "dog")));
indexAliasesService.add("all", null);
assertThat(indexAliasesService.hasAlias("cats"), equalTo(true));
assertThat(indexAliasesService.hasAlias("dogs"), equalTo(true));
assertThat(indexAliasesService.hasAlias("turtles"), equalTo(false));
assertThat(indexAliasesService.aliasFilter("cats").toString(), equalTo("FilterCacheFilterWrapper(animal:cat)"));
assertThat(indexAliasesService.aliasFilter("cats", "dogs").toString(), equalTo("BooleanFilter( FilterCacheFilterWrapper(animal:cat) FilterCacheFilterWrapper(animal:dog))"));
// Non-filtering alias should turn off all filters because filters are ORed
assertThat(indexAliasesService.aliasFilter("all"), nullValue());
assertThat(indexAliasesService.aliasFilter("cats", "all"), nullValue());
assertThat(indexAliasesService.aliasFilter("all", "cats"), nullValue());
// Index itself should turn off al filters as well
assertThat(indexAliasesService.aliasFilter("test", "cats"), nullValue());
// Unknown aliases should be ignored
assertThat(indexAliasesService.aliasFilter("unknown", "cats").toString(), equalTo("FilterCacheFilterWrapper(animal:cat)"));
assertThat(indexAliasesService.aliasFilter("unknown", "all"), nullValue());
indexAliasesService.remove("cats");
assertThat(indexAliasesService.aliasFilter("cats"), nullValue());
indexAliasesService.add("cats", filter(termFilter("animal", "feline")));
indexAliasesService.add("dogs", filter(termFilter("animal", "canine")));
assertThat(indexAliasesService.aliasFilter("dogs", "cats").toString(), equalTo("BooleanFilter( FilterCacheFilterWrapper(animal:canine) FilterCacheFilterWrapper(animal:feline))"));
}
}