honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Transform] fixing naming in HLRC and _cat to match API content (#54300) (#54408) 

Fixing the naming of the HLRC values to match the ToXContent field names (i.e. the field names returned from an API call).

Also fixes the names in the _cat API as well.

closes #53946
This commit is contained in:
Benjamin Trent 2020-03-30 08:57:02 -04:00 committed by GitHub
parent 12cfdc24b0
commit 374e76d7cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 260 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java
View File

@ -24,7 +24,6 @@ import org.elasticsearch.common.ParseField;
import java.util.Objects;
public abstract class IndexerJobStats {
public static final String NAME = "data_frame_indexer_transform_stats";
public static ParseField NUM_PAGES = new ParseField("pages_processed");
public static ParseField NUM_INPUT_DOCUMENTS = new ParseField("documents_processed");
public static ParseField NUM_OUTPUT_DOCUMENTS = new ParseField("documents_indexed");

207
client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.client.transform.transforms;
import org.elasticsearch.client.core.IndexerJobStats;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
@ -29,11 +28,24 @@ import java.util.Objects;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
public class TransformIndexerStats extends IndexerJobStats {
public class TransformIndexerStats {
public static final String NAME = "transform_indexer_stats";
static ParseField EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS = new ParseField("exponential_avg_checkpoint_duration_ms");
static ParseField EXPONENTIAL_AVG_DOCUMENTS_INDEXED = new ParseField("exponential_avg_documents_indexed");
static ParseField EXPONENTIAL_AVG_DOCUMENTS_PROCESSED = new ParseField("exponential_avg_documents_processed");
static ParseField PAGES_PROCESSED = new ParseField("pages_processed");
static ParseField DOCUMENTS_PROCESSED = new ParseField("documents_processed");
static ParseField DOCUMENTS_INDEXED = new ParseField("documents_indexed");
static ParseField TRIGGER_COUNT = new ParseField("trigger_count");
static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms");
static ParseField INDEX_TOTAL = new ParseField("index_total");
static ParseField SEARCH_TOTAL = new ParseField("search_total");
static ParseField PROCESSING_TOTAL = new ParseField("processing_total");
static ParseField SEARCH_FAILURES = new ParseField("search_failures");
static ParseField INDEX_FAILURES = new ParseField("index_failures");
public static final ConstructingObjectParser<TransformIndexerStats, Void> LENIENT_PARSER = new ConstructingObjectParser<>(
NAME,
@ -58,10 +70,10 @@ public class TransformIndexerStats extends IndexerJobStats {
);
static {
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES);
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS);
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS);
LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS);
LENIENT_PARSER.declareLong(optionalConstructorArg(), PAGES_PROCESSED);
LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_PROCESSED);
LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_INDEXED);
LENIENT_PARSER.declareLong(optionalConstructorArg(), TRIGGER_COUNT);
LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS);
LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS);
LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS);
@ -82,12 +94,24 @@ public class TransformIndexerStats extends IndexerJobStats {
private final double expAvgCheckpointDurationMs;
private final double expAvgDocumentsIndexed;
private final double expAvgDocumentsProcessed;
private final long pagesProcessed;
private final long documentsProcessed;
private final long documentsIndexed;
private final long triggerCount;
private final long indexTime;
private final long indexTotal;
private final long searchTime;
private final long searchTotal;
private final long processingTime;
private final long processingTotal;
private final long indexFailures;
private final long searchFailures;
public TransformIndexerStats(
long numPages,
long numInputDocuments,
long numOuputDocuments,
long numInvocations,
long pagesProcessed,
long documentsProcessed,
long documentsIndexed,
long triggerCount,
long indexTime,
long searchTime,
long processingTime,
@ -100,20 +124,18 @@ public class TransformIndexerStats extends IndexerJobStats {
double expAvgDocumentsIndexed,
double expAvgDocumentsProcessed
) {
super(
numPages,
numInputDocuments,
numOuputDocuments,
numInvocations,
indexTime,
searchTime,
processingTime,
indexTotal,
searchTotal,
processingTotal,
indexFailures,
searchFailures
);
this.pagesProcessed = pagesProcessed;
this.documentsProcessed = documentsProcessed;
this.documentsIndexed = documentsIndexed;
this.triggerCount = triggerCount;
this.indexTime = indexTime;
this.indexTotal = indexTotal;
this.searchTime = searchTime;
this.searchTotal = searchTotal;
this.processingTime = processingTime;
this.processingTotal = processingTotal;
this.indexFailures = indexFailures;
this.searchFailures = searchFailures;
this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs;
this.expAvgDocumentsIndexed = expAvgDocumentsIndexed;
this.expAvgDocumentsProcessed = expAvgDocumentsProcessed;
@ -131,6 +153,127 @@ public class TransformIndexerStats extends IndexerJobStats {
return expAvgDocumentsProcessed;
}
/**
* The number of pages read from the input indices
*/
public long getPagesProcessed() {
return pagesProcessed;
}
/**
* The number of documents read from the input indices
*/
public long getDocumentsProcessed() {
return documentsProcessed;
}
/**
* Number of times that the job woke up to write documents
*/
public long getTriggerCount() {
return triggerCount;
}
/**
* Number of documents written
*/
public long getDocumentsIndexed() {
return documentsIndexed;
}
/**
* The number of pages read from the input indices
* Deprecated, use {@link TransformIndexerStats#getPagesProcessed()} instead
*/
@Deprecated
public long getNumPages() {
return getPagesProcessed();
}
/**
* The number of documents read from the input indices
* Deprecated, use {@link TransformIndexerStats#getDocumentsProcessed()} instead
*/
@Deprecated
public long getNumDocuments() {
return getDocumentsProcessed();
}
/**
* Number of times that the job woke up to write documents
* Deprecated, use {@link TransformIndexerStats#getTriggerCount()} instead
*/
@Deprecated
public long getNumInvocations() {
return getTriggerCount();
}
/**
* Number of documents written
* Deprecated, use {@link TransformIndexerStats#getDocumentsIndexed()} instead
*/
@Deprecated
public long getOutputDocuments() {
return getDocumentsIndexed();
}
/**
* Number of index failures that have occurred
*/
public long getIndexFailures() {
return indexFailures;
}
/**
* Number of failures that have occurred
*/
public long getSearchFailures() {
return searchFailures;
}
/**
* Returns the time spent indexing (cumulative) in milliseconds
*/
public long getIndexTime() {
return indexTime;
}
/**
* Returns the time spent searching (cumulative) in milliseconds
*/
public long getSearchTime() {
return searchTime;
}
/**
* Returns the time spent processing (cumulative) in milliseconds
*/
public long getProcessingTime() {
return processingTime;
}
/**
* Returns the total number of indexing requests that have been processed
* (Note: this is not the number of _documents_ that have been indexed)
*/
public long getIndexTotal() {
return indexTotal;
}
/**
* Returns the total number of search requests that have been made
*/
public long getSearchTotal() {
return searchTotal;
}
/**
* Returns the total number of processing runs that have been made
*/
public long getProcessingTotal() {
return processingTotal;
}
@Override
public boolean equals(Object other) {
if (this == other) {
@ -143,10 +286,10 @@ public class TransformIndexerStats extends IndexerJobStats {
TransformIndexerStats that = (TransformIndexerStats) other;
return Objects.equals(this.numPages, that.numPages)
&& Objects.equals(this.numInputDocuments, that.numInputDocuments)
&& Objects.equals(this.numOuputDocuments, that.numOuputDocuments)
&& Objects.equals(this.numInvocations, that.numInvocations)
return Objects.equals(this.pagesProcessed, that.pagesProcessed)
&& Objects.equals(this.documentsProcessed, that.documentsProcessed)
&& Objects.equals(this.documentsIndexed, that.documentsIndexed)
&& Objects.equals(this.triggerCount, that.triggerCount)
&& Objects.equals(this.indexTime, that.indexTime)
&& Objects.equals(this.searchTime, that.searchTime)
&& Objects.equals(this.processingTime, that.processingTime)
@ -163,10 +306,10 @@ public class TransformIndexerStats extends IndexerJobStats {
@Override
public int hashCode() {
return Objects.hash(
numPages,
numInputDocuments,
numOuputDocuments,
numInvocations,
pagesProcessed,
documentsProcessed,
documentsIndexed,
triggerCount,
indexTime,
searchTime,
processingTime,

49
client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java
View File

@ -19,7 +19,6 @@
package org.elasticsearch.client.transform.transforms;
import org.elasticsearch.client.core.IndexerJobStats;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.test.ESTestCase;
@ -61,18 +60,18 @@ public class TransformIndexerStatsTests extends ESTestCase {
public static void toXContent(TransformIndexerStats stats, XContentBuilder builder) throws IOException {
builder.startObject();
if (randomBoolean()) {
builder.field(IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
builder.field(IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
builder.field(IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
builder.field(IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
builder.field(IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
builder.field(IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
builder.field(IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
builder.field(IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
builder.field(IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
builder.field(IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
builder.field(IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
builder.field(IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
builder.field(TransformIndexerStats.PAGES_PROCESSED.getPreferredName(), stats.getPagesProcessed());
builder.field(TransformIndexerStats.DOCUMENTS_PROCESSED.getPreferredName(), stats.getDocumentsProcessed());
builder.field(TransformIndexerStats.DOCUMENTS_INDEXED.getPreferredName(), stats.getDocumentsIndexed());
builder.field(TransformIndexerStats.TRIGGER_COUNT.getPreferredName(), stats.getTriggerCount());
builder.field(TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
builder.field(TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
builder.field(TransformIndexerStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
builder.field(TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
builder.field(TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
builder.field(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
builder.field(TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
builder.field(TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
builder.field(
TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),
stats.getExpAvgCheckpointDurationMs()
@ -84,18 +83,18 @@ public class TransformIndexerStatsTests extends ESTestCase {
);
} else {
// a toXContent version which leaves out field with value 0 (simulating the case that an older version misses a field)
xContentFieldIfNotZero(builder, IndexerJobStats.NUM_PAGES.getPreferredName(), stats.getNumPages());
xContentFieldIfNotZero(builder, IndexerJobStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments());
xContentFieldIfNotZero(builder, IndexerJobStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments());
xContentFieldIfNotZero(builder, IndexerJobStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations());
xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
xContentFieldIfNotZero(builder, IndexerJobStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
xContentFieldIfNotZero(builder, IndexerJobStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
xContentFieldIfNotZero(builder, IndexerJobStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
xContentFieldIfNotZero(builder, IndexerJobStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
xContentFieldIfNotZero(builder, TransformIndexerStats.PAGES_PROCESSED.getPreferredName(), stats.getPagesProcessed());
xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_PROCESSED.getPreferredName(), stats.getDocumentsProcessed());
xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_INDEXED.getPreferredName(), stats.getDocumentsIndexed());
xContentFieldIfNotZero(builder, TransformIndexerStats.TRIGGER_COUNT.getPreferredName(), stats.getTriggerCount());
xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime());
xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal());
xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_FAILURES.getPreferredName(), stats.getIndexFailures());
xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(), stats.getSearchTime());
xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal());
xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime());
xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal());
xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures());
xContentFieldIfNotZero(
builder,
TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(),

8
client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java
View File

@ -153,13 +153,13 @@ public class TransformStatsTests extends AbstractResponseTestCase<
assertThat(serverTestInstance.getExpAvgCheckpointDurationMs(), equalTo(clientInstance.getExpAvgCheckpointDurationMs()));
assertThat(serverTestInstance.getExpAvgDocumentsProcessed(), equalTo(clientInstance.getExpAvgDocumentsProcessed()));
assertThat(serverTestInstance.getExpAvgDocumentsIndexed(), equalTo(clientInstance.getExpAvgDocumentsIndexed()));
assertThat(serverTestInstance.getNumPages(), equalTo(clientInstance.getNumPages()));
assertThat(serverTestInstance.getNumPages(), equalTo(clientInstance.getPagesProcessed()));
assertThat(serverTestInstance.getIndexFailures(), equalTo(clientInstance.getIndexFailures()));
assertThat(serverTestInstance.getIndexTime(), equalTo(clientInstance.getIndexTime()));
assertThat(serverTestInstance.getIndexTotal(), equalTo(clientInstance.getIndexTotal()));
assertThat(serverTestInstance.getNumDocuments(), equalTo(clientInstance.getNumDocuments()));
assertThat(serverTestInstance.getNumInvocations(), equalTo(clientInstance.getNumInvocations()));
assertThat(serverTestInstance.getOutputDocuments(), equalTo(clientInstance.getOutputDocuments()));
assertThat(serverTestInstance.getNumDocuments(), equalTo(clientInstance.getDocumentsProcessed()));
assertThat(serverTestInstance.getNumInvocations(), equalTo(clientInstance.getTriggerCount()));
assertThat(serverTestInstance.getOutputDocuments(), equalTo(clientInstance.getDocumentsIndexed()));
assertThat(serverTestInstance.getSearchFailures(), equalTo(clientInstance.getSearchFailures()));
assertThat(serverTestInstance.getSearchTime(), equalTo(clientInstance.getSearchTime()));
assertThat(serverTestInstance.getSearchTotal(), equalTo(clientInstance.getSearchTotal()));

16
docs/reference/cat/transforms.asciidoc
View File

@ -74,7 +74,10 @@ The description of the {transform}.
(Default)
include::{docdir}/rest-api/common-parms.asciidoc[tag=dest-index]
`document_total`, `dt`:::
`documents_indexed`, `doci`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=docs-indexed]
`documents_processed`, `docp`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=docs-processed]
`frequency`, `f`:::
@ -97,14 +100,11 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=index-total]
`indexed_documents_exp_avg`, `idea`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-indexed]
`invocation_total`, `itotal`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=trigger-count]
`max_page_search_size`, `mpsz`:::
(Default)
include::{docdir}/rest-api/common-parms.asciidoc[tag=pivot-max-page-search-size]
`page_total`, `pt`:::
`pages_processed`, `pp`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=pages-processed]
`pipeline`, `p`:::
@ -114,6 +114,9 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=dest-pipeline]
`processed_documents_exp_avg`, `pdea`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=exponential-avg-documents-processed]
`processing_time`, `pt`:::
The total time spent processing documents.
`reason`, `r`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform-reason]
@ -138,6 +141,9 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform]
(Default)
Indicates the type of {transform}: `batch` or `continuous`.
`trigger_count`, `tc`:::
include::{docdir}/rest-api/common-parms.asciidoc[tag=trigger-count]
`version`, `v`:::
(Default)
The version of {es} that existed on the node when the {transform} was

5
docs/reference/rest-api/common-parms.asciidoc
View File

@ -192,6 +192,11 @@ is based on Lucene documents. {es} reclaims the disk space of deleted Lucene
documents when a segment is merged.
end::docs-deleted[]
tag::docs-indexed[]
The number of documents that have been indexed into the destination index
for the {transform}.
end::docs-indexed[]
tag::docs-processed[]
The number of documents that have been processed from the source index of
the {transform}.

4
docs/reference/transform/apis/get-transform-stats.asciidoc
View File

@ -137,8 +137,8 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=state-transform]
`stats`::
(object) An object that provides statistical information about the {transform}.
`stats`.`documents_indexed`:::
(long) The number of documents that have been indexed into the destination index
for the {transform}.
(long)
include::{docdir}/rest-api/common-parms.asciidoc[tag=docs-indexed]
`stats`.`documents_processed`:::
(long)
include::{docdir}/rest-api/common-parms.asciidoc[tag=docs-processed]

4
x-pack/plugin/src/test/resources/rest-api-spec/test/transform/transforms_cat_apis.yml
View File

@ -64,10 +64,10 @@ teardown:
cat.transform:
transform_id: "airline-transform-stats"
v: true
h: id,version,source_index,dest_index,search_total,index_total,dt,cdtea,indexed_documents_exp_avg
h: id,version,source_index,dest_index,search_total,index_total,docp,cdtea,indexed_documents_exp_avg
- match:
$body: |
/^ id \s+ version \s+ source_index \s+ dest_index \s+ search_total \s+ index_total \s+ dt \s+ cdtea \s+ indexed_documents_exp_avg \n
/^ id \s+ version \s+ source_index \s+ dest_index \s+ search_total \s+ index_total \s+ docp \s+ cdtea \s+ indexed_documents_exp_avg \n
(airline\-transform-stats \s+ [^\s]+ \s+ airline-data \s+ airline-data-by-airline \s+ 0 \s+ 0 \s+ 0 \s+ 0.0 \s+ 0.0 \n)+ $/

10
x-pack/plugin/transform/qa/multi-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
View File

@ -109,7 +109,7 @@ public class TransformIT extends TransformIntegTestCase {
waitUntilCheckpoint(config.getId(), 1L);
assertThat(getTransformStats(config.getId()).getTransformsStats().get(0).getState(), equalTo(TransformStats.State.STARTED));
long docsIndexed = getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getNumDocuments();
long docsIndexed = getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getDocumentsIndexed();
TransformConfig storedConfig = getTransform(config.getId()).getTransformConfigurations().get(0);
assertThat(storedConfig.getVersion(), equalTo(Version.CURRENT));
@ -124,7 +124,7 @@ public class TransformIT extends TransformIntegTestCase {
// Assert that we wrote the new docs
assertThat(
getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getNumDocuments(),
getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getDocumentsIndexed(),
greaterThan(docsIndexed)
);
@ -158,7 +158,7 @@ public class TransformIT extends TransformIntegTestCase {
oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING)
);
long docsIndexed = getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getNumDocuments();
long docsIndexed = getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getDocumentsIndexed();
TransformConfig storedConfig = getTransform(config.getId()).getTransformConfigurations().get(0);
assertThat(storedConfig.getVersion(), equalTo(Version.CURRENT));
@ -197,7 +197,7 @@ public class TransformIT extends TransformIntegTestCase {
// Since updates are loaded on checkpoint start, we should see the updated config on this next run
waitUntilCheckpoint(config.getId(), 2L);
long numDocsAfterCp2 = getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getNumDocuments();
long numDocsAfterCp2 = getTransformStats(config.getId()).getTransformsStats().get(0).getIndexerStats().getDocumentsIndexed();
assertThat(numDocsAfterCp2, greaterThan(docsIndexed));
final SearchRequest searchRequest = new SearchRequest(dest).source(
@ -251,7 +251,7 @@ public class TransformIT extends TransformIntegTestCase {
assertBusy(() -> {
TransformStats stateAndStats = getTransformStats(config.getId()).getTransformsStats().get(0);
assertThat(stateAndStats.getState(), equalTo(TransformStats.State.STOPPED));
assertThat(stateAndStats.getIndexerStats().getNumDocuments(), equalTo(1000L));
assertThat(stateAndStats.getIndexerStats().getDocumentsIndexed(), equalTo(1000L));
});
stopTransform(config.getId());

40
x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java
View File

@ -138,7 +138,7 @@ public class RestCatTransformAction extends AbstractCatAction {
.setAliases("f")
.build())
.addCell("max_page_search_size",
TableColumnAttributeBuilder.builder("max page search size ")
TableColumnAttributeBuilder.builder("max page search size")
.setAliases("mpsz")
.build())
@ -149,7 +149,7 @@ public class RestCatTransformAction extends AbstractCatAction {
.setTextAlignment(TableColumnAttributeBuilder.TextAlign.RIGHT)
.build())
.addCell("reason",
TableColumnAttributeBuilder.builder("reason", false)
TableColumnAttributeBuilder.builder("reason for the current state", false)
.setAliases("r", "reason")
.build())
.addCell("changes_last_detection_time",
@ -157,7 +157,7 @@ public class RestCatTransformAction extends AbstractCatAction {
.setAliases("cldt")
.build())
.addCell("search_total",
TableColumnAttributeBuilder.builder("total number of searches", false)
TableColumnAttributeBuilder.builder("total number of search phases", false)
.setAliases("st")
.build())
.addCell("search_failure",
@ -165,11 +165,11 @@ public class RestCatTransformAction extends AbstractCatAction {
.setAliases("sf")
.build())
.addCell("search_time",
TableColumnAttributeBuilder.builder("search time", false)
TableColumnAttributeBuilder.builder("total search time", false)
.setAliases("stime")
.build())
.addCell("index_total",
TableColumnAttributeBuilder.builder("total number of indices", false)
TableColumnAttributeBuilder.builder("total number of index phases done by the transform", false)
.setAliases("it")
.build())
.addCell("index_failure",
@ -177,19 +177,29 @@ public class RestCatTransformAction extends AbstractCatAction {
.setAliases("if")
.build())
.addCell("index_time",
TableColumnAttributeBuilder.builder("index time", false)
TableColumnAttributeBuilder.builder("total time spent indexing documents", false)
.setAliases("itime")
.build())
.addCell("document_total",
TableColumnAttributeBuilder.builder("total number of documents", false)
.setAliases("dt")
.addCell("documents_processed",
TableColumnAttributeBuilder.builder("the number of documents read from source indices and processed",
false)
.setAliases("docp")
.build())
.addCell("invocation_total",
TableColumnAttributeBuilder.builder("total number of invocations", false)
.setAliases("itotal")
.addCell("documents_indexed",
TableColumnAttributeBuilder.builder("the number of documents index to the destination index",
false)
.setAliases("doci")
.build())
.addCell("page_total",
TableColumnAttributeBuilder.builder("total number of pages", false)
.addCell("trigger_count",
TableColumnAttributeBuilder.builder("the number of times the transform has been triggered", false)
.setAliases("tc")
.build())
.addCell("pages_processed",
TableColumnAttributeBuilder.builder("the number of pages processed", false)
.setAliases("pp")
.build())
.addCell("processing_time",
TableColumnAttributeBuilder.builder("the total time spent processing documents", false)
.setAliases("pt")
.build())
.addCell("checkpoint_duration_time_exp_avg",
@ -246,8 +256,10 @@ public class RestCatTransformAction extends AbstractCatAction {
.addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getIndexTime()))
.addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumDocuments())
.addCell(transformIndexerStats == null ? null : transformIndexerStats.getOutputDocuments())
.addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumInvocations())
.addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumPages())
.addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getProcessingTime()))
.addCell(transformIndexerStats == null ? null : transformIndexerStats.getExpAvgCheckpointDurationMs())
.addCell(transformIndexerStats == null ? null : transformIndexerStats.getExpAvgDocumentsIndexed())

12
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
View File

@ -197,8 +197,8 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase {
assertBusy(() -> {
TransformStats stateAndStats = getTransformStats(CONTINUOUS_TRANSFORM_ID);
assertThat(stateAndStats.getIndexerStats().getOutputDocuments(), equalTo((long)ENTITIES.size()));
assertThat(stateAndStats.getIndexerStats().getNumDocuments(), equalTo(totalDocsWritten));
assertThat(stateAndStats.getIndexerStats().getDocumentsIndexed(), equalTo((long)ENTITIES.size()));
assertThat(stateAndStats.getIndexerStats().getDocumentsProcessed(), equalTo(totalDocsWritten));
// Even if we get back to started, we may periodically get set back to `indexing` when triggered.
// Though short lived due to no changes on the source indices, it could result in flaky test behavior
assertThat(stateAndStats.getState(), oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING));
@ -236,8 +236,8 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase {
waitUntilAfterCheckpoint(CONTINUOUS_TRANSFORM_ID, expectedLastCheckpoint);
assertBusy(() -> assertThat(
getTransformStats(CONTINUOUS_TRANSFORM_ID).getIndexerStats().getNumDocuments(),
greaterThanOrEqualTo(docs + previousStateAndStats.getIndexerStats().getNumDocuments())),
getTransformStats(CONTINUOUS_TRANSFORM_ID).getIndexerStats().getDocumentsProcessed(),
greaterThanOrEqualTo(docs + previousStateAndStats.getIndexerStats().getDocumentsProcessed())),
120,
TimeUnit.SECONDS);
TransformStats stateAndStats = getTransformStats(CONTINUOUS_TRANSFORM_ID);
@ -249,9 +249,9 @@ public class TransformSurvivesUpgradeIT extends AbstractUpgradeTestCase {
responseBody))
.get(0);
assertThat((Integer)indexerStats.get("documents_indexed"),
greaterThan(Long.valueOf(previousStateAndStats.getIndexerStats().getOutputDocuments()).intValue()));
greaterThan(Long.valueOf(previousStateAndStats.getIndexerStats().getDocumentsIndexed()).intValue()));
assertThat((Integer)indexerStats.get("documents_processed"),
greaterThan(Long.valueOf(previousStateAndStats.getIndexerStats().getNumDocuments()).intValue()));
greaterThan(Long.valueOf(previousStateAndStats.getIndexerStats().getDocumentsProcessed()).intValue()));
});
}