honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Remove] Type Specific Index Stats (#2198) 

Removes type specific index stats since only one type is allowed.

Signed-off-by: Nicholas Walter Knize <nknize@apache.org>
This commit is contained in:
Nick Knize 2022-02-21 16:06:43 -06:00 committed by GitHub
parent d7b8b323a5
commit c9f7c37367
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 26 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
client/rest-high-level/src/test/java/org/opensearch/client/CrudIT.java
View File

@ -438,7 +438,7 @@ public class CrudIT extends OpenSearchRestHighLevelClientTestCase {
}
}
public void testMultiGetWithTypes() throws IOException {
public void testMultiGetWithIds() throws IOException {
BulkRequest bulk = new BulkRequest();
bulk.setRefreshPolicy(RefreshPolicy.IMMEDIATE);
bulk.add(new IndexRequest("index", "type", "id1").source("{\"field\":\"value1\"}", XContentType.JSON));

34
server/src/internalClusterTest/java/org/opensearch/indices/stats/IndexStatsIT.java
View File

@ -1079,40 +1079,6 @@ public class IndexStatsIT extends OpenSearchIntegTestCase {
}
public void testTypesParam() throws Exception {
createIndex("test1");
createIndex("test2");
ensureGreen();
client().prepareIndex("test1", "bar", Integer.toString(1)).setSource("foo", "bar").execute().actionGet();
client().prepareIndex("test2", "baz", Integer.toString(1)).setSource("foo", "bar").execute().actionGet();
refresh();
IndicesStatsRequestBuilder builder = client().admin().indices().prepareStats();
IndicesStatsResponse stats = builder.execute().actionGet();
assertThat(stats.getTotal().indexing.getTotal().getIndexCount(), greaterThan(0L));
assertThat(stats.getTotal().indexing.getTypeStats(), is(nullValue()));
stats = builder.setTypes("bar").execute().actionGet();
assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
assertThat(stats.getTotal().indexing.getTypeStats().containsKey("baz"), is(false));
stats = builder.setTypes("bar", "baz").execute().actionGet();
assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
assertThat(stats.getTotal().indexing.getTypeStats().get("baz").getIndexCount(), greaterThan(0L));
stats = builder.setTypes("*").execute().actionGet();
assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
assertThat(stats.getTotal().indexing.getTypeStats().get("baz").getIndexCount(), greaterThan(0L));
stats = builder.setTypes("*r").execute().actionGet();
assertThat(stats.getTotal().indexing.getTypeStats().get("bar").getIndexCount(), greaterThan(0L));
assertThat(stats.getTotal().indexing.getTypeStats().containsKey("baz"), is(false));
}
private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean set) {
switch (flag) {
case Docs:

2
server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStats.java
View File

@ -190,7 +190,7 @@ public class CommonStats implements Writeable, ToXContentFragment {
store = indexShard.storeStats();
break;
case Indexing:
indexing = indexShard.indexingStats(flags.types());
indexing = indexShard.indexingStats();
break;
case Get:
get = indexShard.getStats();

29
server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
View File

@ -34,6 +34,7 @@ package org.opensearch.action.admin.indices.stats;
import org.opensearch.LegacyESVersion;
import org.opensearch.Version;
import org.opensearch.common.Strings;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
@ -48,7 +49,6 @@ public class CommonStatsFlags implements Writeable, Cloneable {
public static final CommonStatsFlags NONE = new CommonStatsFlags().clear();
private EnumSet<Flag> flags = EnumSet.allOf(Flag.class);
private String[] types = null;
private String[] groups = null;
private String[] fieldDataFields = null;
private String[] completionDataFields = null;
@ -75,7 +75,9 @@ public class CommonStatsFlags implements Writeable, Cloneable {
flags.add(flag);
}
}
types = in.readStringArray();
if (in.getVersion().before(Version.V_2_0_0)) {
in.readStringArray();
}
groups = in.readStringArray();
fieldDataFields = in.readStringArray();
completionDataFields = in.readStringArray();
@ -97,7 +99,9 @@ public class CommonStatsFlags implements Writeable, Cloneable {
}
out.writeLong(longFlags);
out.writeStringArrayNullable(types);
if (out.getVersion().before(Version.V_2_0_0)) {
out.writeStringArrayNullable(Strings.EMPTY_ARRAY);
}
out.writeStringArrayNullable(groups);
out.writeStringArrayNullable(fieldDataFields);
out.writeStringArrayNullable(completionDataFields);
@ -116,7 +120,6 @@ public class CommonStatsFlags implements Writeable, Cloneable {
*/
public CommonStatsFlags all() {
flags = EnumSet.allOf(Flag.class);
types = null;
groups = null;
fieldDataFields = null;
completionDataFields = null;
@ -132,7 +135,6 @@ public class CommonStatsFlags implements Writeable, Cloneable {
*/
public CommonStatsFlags clear() {
flags = EnumSet.noneOf(Flag.class);
types = null;
groups = null;
fieldDataFields = null;
completionDataFields = null;
@ -151,23 +153,6 @@ public class CommonStatsFlags implements Writeable, Cloneable {
return flags.toArray(new Flag[flags.size()]);
}
/**
* Document types to return stats for. Mainly affects {@link Flag#Indexing} when
* enabled, returning specific indexing stats for those types.
*/
public CommonStatsFlags types(String... types) {
this.types = types;
return this;
}
/**
* Document types to return stats for. Mainly affects {@link Flag#Indexing} when
* enabled, returning specific indexing stats for those types.
*/
public String[] types() {
return this.types;
}
/**
* Sets specific search group stats to retrieve the stats for. Mainly affects search
* when enabled.

17
server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequest.java
View File

@ -90,23 +90,6 @@ public class IndicesStatsRequest extends BroadcastRequest<IndicesStatsRequest> {
return this;
}
/**
* Document types to return stats for. Mainly affects {@link #indexing(boolean)} when
* enabled, returning specific indexing stats for those types.
*/
public IndicesStatsRequest types(String... types) {
flags.types(types);
return this;
}
/**
* Document types to return stats for. Mainly affects {@link #indexing(boolean)} when
* enabled, returning specific indexing stats for those types.
*/
public String[] types() {
return this.flags.types();
}
/**
* Sets specific search group stats to retrieve the stats for. Mainly affects search
* when enabled.

9
server/src/main/java/org/opensearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java
View File

@ -78,15 +78,6 @@ public class IndicesStatsRequestBuilder extends BroadcastOperationRequestBuilder
return this;
}
/**
* Document types to return stats for. Mainly affects {@link #setIndexing(boolean)} when
* enabled, returning specific indexing stats for those types.
*/
public IndicesStatsRequestBuilder setTypes(String... types) {
request.types(types);
return this;
}
public IndicesStatsRequestBuilder setGroups(String... groups) {
request.groups(groups);
return this;

2
server/src/main/java/org/opensearch/action/update/TransportUpdateAction.java
View File

@ -325,7 +325,7 @@ public class TransportUpdateAction extends TransportInstanceSingleOperationActio
if (indexServiceOrNull != null) {
IndexShard shard = indexService.getShardOrNull(shardId.getId());
if (shard != null) {
shard.noopUpdate(request.type());
shard.noopUpdate();
}
}
listener.onResponse(update);

8
server/src/main/java/org/opensearch/index/shard/IndexShard.java
View File

@ -1259,7 +1259,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
throttled = engine.isThrottled();
throttleTimeInMillis = engine.getIndexThrottleTimeInMillis();
}
return internalIndexingStats.stats(throttled, throttleTimeInMillis, types);
return internalIndexingStats.stats(throttled, throttleTimeInMillis);
}
public SearchStats searchStats(String... groups) {
@ -2847,11 +2847,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
/**
* Should be called for each no-op update operation to increment relevant statistics.
*
* @param type the doc type of the update
*/
public void noopUpdate(String type) {
internalIndexingStats.noopUpdate(type);
public void noopUpdate() {
internalIndexingStats.noopUpdate();
}
public void maybeCheckIndex() {

55
server/src/main/java/org/opensearch/index/shard/IndexingStats.java
View File

@ -32,7 +32,7 @@
package org.opensearch.index.shard;
import org.opensearch.common.Nullable;
import org.opensearch.Version;
import org.opensearch.common.io.stream.StreamInput;
import org.opensearch.common.io.stream.StreamOutput;
import org.opensearch.common.io.stream.Writeable;
@ -40,9 +40,9 @@ import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.ToXContent;
import org.opensearch.common.xcontent.ToXContentFragment;
import org.opensearch.common.xcontent.XContentBuilder;
import org.opensearch.index.mapper.MapperService;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
public class IndexingStats implements Writeable, ToXContentFragment {
@ -219,47 +219,30 @@ public class IndexingStats implements Writeable, ToXContentFragment {
private final Stats totalStats;
@Nullable
private Map<String, Stats> typeStats;
public IndexingStats() {
totalStats = new Stats();
}
public IndexingStats(StreamInput in) throws IOException {
totalStats = new Stats(in);
if (in.readBoolean()) {
typeStats = in.readMap(StreamInput::readString, Stats::new);
if (in.getVersion().before(Version.V_2_0_0)) {
if (in.readBoolean()) {
Map<String, Stats> typeStats = in.readMap(StreamInput::readString, Stats::new);
assert typeStats.size() == 1;
assert typeStats.containsKey(MapperService.SINGLE_MAPPING_NAME);
}
}
}
public IndexingStats(Stats totalStats, @Nullable Map<String, Stats> typeStats) {
public IndexingStats(Stats totalStats) {
this.totalStats = totalStats;
this.typeStats = typeStats;
}
public void add(IndexingStats indexingStats) {
add(indexingStats, true);
}
public void add(IndexingStats indexingStats, boolean includeTypes) {
if (indexingStats == null) {
return;
}
addTotals(indexingStats);
if (includeTypes && indexingStats.typeStats != null && !indexingStats.typeStats.isEmpty()) {
if (typeStats == null) {
typeStats = new HashMap<>(indexingStats.typeStats.size());
}
for (Map.Entry<String, Stats> entry : indexingStats.typeStats.entrySet()) {
Stats stats = typeStats.get(entry.getKey());
if (stats == null) {
typeStats.put(entry.getKey(), entry.getValue());
} else {
stats.add(entry.getValue());
}
}
}
}
public void addTotals(IndexingStats indexingStats) {
@ -273,31 +256,16 @@ public class IndexingStats implements Writeable, ToXContentFragment {
return this.totalStats;
}
@Nullable
public Map<String, Stats> getTypeStats() {
return this.typeStats;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(Fields.INDEXING);
totalStats.toXContent(builder, params);
if (typeStats != null && !typeStats.isEmpty()) {
builder.startObject(Fields.TYPES);
for (Map.Entry<String, Stats> entry : typeStats.entrySet()) {
builder.startObject(entry.getKey());
entry.getValue().toXContent(builder, params);
builder.endObject();
}
builder.endObject();
}
builder.endObject();
return builder;
}
static final class Fields {
static final String INDEXING = "indexing";
static final String TYPES = "types";
static final String INDEX_TOTAL = "index_total";
static final String INDEX_TIME = "index_time";
static final String INDEX_TIME_IN_MILLIS = "index_time_in_millis";
@ -316,11 +284,8 @@ public class IndexingStats implements Writeable, ToXContentFragment {
@Override
public void writeTo(StreamOutput out) throws IOException {
totalStats.writeTo(out);
if (typeStats == null || typeStats.isEmpty()) {
if (out.getVersion().before(Version.V_2_0_0)) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeMap(typeStats, StreamOutput::writeString, (stream, stats) -> stats.writeTo(stream));
}
}
}

54
server/src/main/java/org/opensearch/index/shard/InternalIndexingStats.java
View File

@ -32,56 +32,33 @@
package org.opensearch.index.shard;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.common.metrics.CounterMetric;
import org.opensearch.common.metrics.MeanMetric;
import org.opensearch.common.regex.Regex;
import org.opensearch.index.engine.Engine;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static java.util.Collections.emptyMap;
/**
* Internal class that maintains relevant indexing statistics / metrics.
* @see IndexShard
*/
final class InternalIndexingStats implements IndexingOperationListener {
private final StatsHolder totalStats = new StatsHolder();
private volatile Map<String, StatsHolder> typesStats = emptyMap();
/**
* Returns the stats, including type specific stats. If the types are null/0 length, then nothing
* is returned for them. If they are set, then only types provided will be returned, or
* {@code _all} for all types.
*/
IndexingStats stats(boolean isThrottled, long currentThrottleInMillis, String... types) {
IndexingStats stats(boolean isThrottled, long currentThrottleInMillis) {
IndexingStats.Stats total = totalStats.stats(isThrottled, currentThrottleInMillis);
Map<String, IndexingStats.Stats> typesSt = null;
if (types != null && types.length > 0) {
typesSt = new HashMap<>(typesStats.size());
if (types.length == 1 && types[0].equals("_all")) {
for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
typesSt.put(entry.getKey(), entry.getValue().stats(isThrottled, currentThrottleInMillis));
}
} else {
for (Map.Entry<String, StatsHolder> entry : typesStats.entrySet()) {
if (Regex.simpleMatch(types, entry.getKey())) {
typesSt.put(entry.getKey(), entry.getValue().stats(isThrottled, currentThrottleInMillis));
}
}
}
}
return new IndexingStats(total, typesSt);
return new IndexingStats(total);
}
@Override
public Engine.Index preIndex(ShardId shardId, Engine.Index operation) {
if (operation.origin().isRecovery() == false) {
totalStats.indexCurrent.inc();
typeStats(operation.type()).indexCurrent.inc();
}
return operation;
}
@ -94,9 +71,6 @@ final class InternalIndexingStats implements IndexingOperationListener {
long took = result.getTook();
totalStats.indexMetric.inc(took);
totalStats.indexCurrent.dec();
StatsHolder typeStats = typeStats(index.type());
typeStats.indexMetric.inc(took);
typeStats.indexCurrent.dec();
}
break;
case FAILURE:
@ -111,9 +85,7 @@ final class InternalIndexingStats implements IndexingOperationListener {
public void postIndex(ShardId shardId, Engine.Index index, Exception ex) {
if (!index.origin().isRecovery()) {
totalStats.indexCurrent.dec();
typeStats(index.type()).indexCurrent.dec();
totalStats.indexFailed.inc();
typeStats(index.type()).indexFailed.inc();
}
}
@ -121,7 +93,6 @@ final class InternalIndexingStats implements IndexingOperationListener {
public Engine.Delete preDelete(ShardId shardId, Engine.Delete delete) {
if (!delete.origin().isRecovery()) {
totalStats.deleteCurrent.inc();
typeStats(delete.type()).deleteCurrent.inc();
}
return delete;
@ -135,9 +106,6 @@ final class InternalIndexingStats implements IndexingOperationListener {
long took = result.getTook();
totalStats.deleteMetric.inc(took);
totalStats.deleteCurrent.dec();
StatsHolder typeStats = typeStats(delete.type());
typeStats.deleteMetric.inc(took);
typeStats.deleteCurrent.dec();
}
break;
case FAILURE:
@ -152,27 +120,11 @@ final class InternalIndexingStats implements IndexingOperationListener {
public void postDelete(ShardId shardId, Engine.Delete delete, Exception ex) {
if (!delete.origin().isRecovery()) {
totalStats.deleteCurrent.dec();
typeStats(delete.type()).deleteCurrent.dec();
}
}
public void noopUpdate(String type) {
void noopUpdate() {
totalStats.noopUpdates.inc();
typeStats(type).noopUpdates.inc();
}
private StatsHolder typeStats(String type) {
StatsHolder stats = typesStats.get(type);
if (stats == null) {
synchronized (this) {
stats = typesStats.get(type);
if (stats == null) {
stats = new StatsHolder();
typesStats = MapBuilder.newMapBuilder(typesStats).put(type, stats).immutableMap();
}
}
}
return stats;
}
static class StatsHolder {

3
server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java
View File

@ -193,9 +193,6 @@ public class RestNodesStatsAction extends BaseRestHandler {
if (nodesStatsRequest.indices().isSet(Flag.Search) && (request.hasParam("groups"))) {
nodesStatsRequest.indices().groups(request.paramAsStringArray("groups", null));
}
if (nodesStatsRequest.indices().isSet(Flag.Indexing) && (request.hasParam("types"))) {
nodesStatsRequest.indices().types(request.paramAsStringArray("types", null));
}
if (nodesStatsRequest.indices().isSet(Flag.Segments)) {
nodesStatsRequest.indices().includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false));
}

5
server/src/main/java/org/opensearch/rest/action/admin/indices/RestIndicesStatsAction.java
View File

@ -102,7 +102,6 @@ public class RestIndicesStatsAction extends BaseRestHandler {
+ "options changed";
indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, defaultIndicesOption));
indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types")));
Set<String> metrics = Strings.tokenizeByCommaToSet(request.param("metric", "_all"));
// short cut, if no metrics have been specified in URI
@ -139,10 +138,6 @@ public class RestIndicesStatsAction extends BaseRestHandler {
indicesStatsRequest.groups(Strings.splitStringByCommaToArray(request.param("groups")));
}
if (request.hasParam("types")) {
indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types")));
}
if (indicesStatsRequest.completion() && (request.hasParam("fields") || request.hasParam("completion_fields"))) {
indicesStatsRequest.completionFields(
request.paramAsStringArray("completion_fields", request.paramAsStringArray("fields", Strings.EMPTY_ARRAY))