From 3ce984d74690d47618ee53f19d43e2ef69601dbb Mon Sep 17 00:00:00 2001 From: Paul Sanwald Date: Tue, 7 Aug 2018 08:56:00 -0400 Subject: [PATCH 01/20] mute test while I work on #32215 --- .../bucket/histogram/InternalAutoDateHistogramTests.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java index a14fca63154..981d263d7d6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java @@ -56,7 +56,7 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati List pipelineAggregators, Map metaData, InternalAggregations aggregations) { - + roundingInfos = AutoDateHistogramAggregationBuilder.buildRoundings(null); int nbBuckets = randomNumberOfBuckets(); int targetBuckets = randomIntBetween(1, nbBuckets * 2 + 1); @@ -137,6 +137,12 @@ public class InternalAutoDateHistogramTests extends InternalMultiBucketAggregati assertEquals(expectedCounts, actualCounts); } + @Override + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/32215") + public void testReduceRandom() { + super.testReduceRandom(); + } + @Override protected Writeable.Reader instanceReader() { return InternalAutoDateHistogram::new; From 45066b5e8939c78be7b4c36a691fa90dd5766c8c Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Tue, 7 Aug 2018 15:02:37 +0200 Subject: [PATCH 02/20] Verify primary mode usage with assertions (#32667) Primary terms were introduced as part of the sequence-number effort (#10708) and added in ES 5.0. Subsequent work introduced the replication tracker which lets the primary own its replication group (#25692) to coordinate recovery and replication. The replication tracker explicitly exposes whether it is operating in primary mode or replica mode, independent of the ShardRouting object that's associated with a shard. During a primary relocation, for example, the primary mode is transferred between the primary relocation source and the primary relocation target. After transferring this so-called primary context, the old primary becomes a replication target and the new primary the replication source, reflected in the replication tracker on both nodes. With the most recent PR in this area (#32442), we finally have a clean transition between a shard that's operating as a primary and issuing sequence numbers and a shard that's serving as a replication target. The transition from one state to the other is enforced through the operation-permit system, where we block permit acquisition during such changes and perform the transition under this operation block, ensuring that there are no operations in progress while the transition is being performed. This finally allows us to turn the best-effort checks that were put in place to prevent shards from being used in the wrong way (i.e. primary as replica, or replica as primary) into hard assertions, making it easier to catch any bugs in this area. --- .../elasticsearch/index/shard/IndexShard.java | 47 +++++++++---------- .../index/shard/IndexShardTests.java | 39 +++++++++------ 2 files changed, 45 insertions(+), 41 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 9131055bcd9..3aa281da101 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1446,10 +1446,10 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } } else { if (origin == Engine.Operation.Origin.PRIMARY) { - verifyPrimary(); + assert assertPrimaryMode(); } else { assert origin == Engine.Operation.Origin.REPLICA; - verifyReplicationTarget(); + assert assertReplicationTarget(); } if (writeAllowedStates.contains(state) == false) { throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when shard state is one of " + writeAllowedStates + ", origin [" + origin + "]"); @@ -1457,19 +1457,14 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl } } - private void verifyPrimary() { - if (shardRouting.primary() == false) { - throw new IllegalStateException("shard " + shardRouting + " is not a primary"); - } + private boolean assertPrimaryMode() { + assert shardRouting.primary() && replicationTracker.isPrimaryMode() : "shard " + shardRouting + " is not a primary shard in primary mode"; + return true; } - private void verifyReplicationTarget() { - final IndexShardState state = state(); - if (shardRouting.primary() && shardRouting.active() && replicationTracker.isPrimaryMode()) { - // must use exception that is not ignored by replication logic. See TransportActions.isShardNotAvailableException - throw new IllegalStateException("active primary shard " + shardRouting + " cannot be a replication target before " + - "relocation hand off, state is [" + state + "]"); - } + private boolean assertReplicationTarget() { + assert replicationTracker.isPrimaryMode() == false : "shard " + shardRouting + " in primary mode cannot be a replication target"; + return true; } private void verifyNotClosed() throws IllegalIndexShardStateException { @@ -1716,7 +1711,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @param checkpoint the local checkpoint for the shard */ public void updateLocalCheckpointForShard(final String allocationId, final long checkpoint) { - verifyPrimary(); + assert assertPrimaryMode(); verifyNotClosed(); replicationTracker.updateLocalCheckpoint(allocationId, checkpoint); } @@ -1728,7 +1723,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @param globalCheckpoint the global checkpoint */ public void updateGlobalCheckpointForShard(final String allocationId, final long globalCheckpoint) { - verifyPrimary(); + assert assertPrimaryMode(); verifyNotClosed(); replicationTracker.updateGlobalCheckpointForShard(allocationId, globalCheckpoint); } @@ -1750,7 +1745,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @param allocationId the allocation ID of the shard for which recovery was initiated */ public void initiateTracking(final String allocationId) { - verifyPrimary(); + assert assertPrimaryMode(); replicationTracker.initiateTracking(allocationId); } @@ -1763,7 +1758,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @param localCheckpoint the current local checkpoint on the shard */ public void markAllocationIdAsInSync(final String allocationId, final long localCheckpoint) throws InterruptedException { - verifyPrimary(); + assert assertPrimaryMode(); replicationTracker.markAllocationIdAsInSync(allocationId, localCheckpoint); } @@ -1798,7 +1793,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @return a map from allocation ID to the local knowledge of the global checkpoint for that allocation ID */ public ObjectLongMap getInSyncGlobalCheckpoints() { - verifyPrimary(); + assert assertPrimaryMode(); verifyNotClosed(); return replicationTracker.getInSyncGlobalCheckpoints(); } @@ -1808,11 +1803,12 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * primary. */ public void maybeSyncGlobalCheckpoint(final String reason) { - verifyPrimary(); verifyNotClosed(); + assert shardRouting.primary() : "only call maybeSyncGlobalCheckpoint on primary shard"; if (replicationTracker.isPrimaryMode() == false) { return; } + assert assertPrimaryMode(); // only sync if there are not operations in flight final SeqNoStats stats = getEngine().getSeqNoStats(replicationTracker.getGlobalCheckpoint()); if (stats.getMaxSeqNo() == stats.getGlobalCheckpoint()) { @@ -1838,7 +1834,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @return the replication group */ public ReplicationGroup getReplicationGroup() { - verifyPrimary(); + assert assertPrimaryMode(); verifyNotClosed(); return replicationTracker.getReplicationGroup(); } @@ -1850,7 +1846,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @param reason the reason the global checkpoint was updated */ public void updateGlobalCheckpointOnReplica(final long globalCheckpoint, final String reason) { - verifyReplicationTarget(); + assert assertReplicationTarget(); final long localCheckpoint = getLocalCheckpoint(); if (globalCheckpoint > localCheckpoint) { /* @@ -1877,8 +1873,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @param primaryContext the sequence number context */ public void activateWithPrimaryContext(final ReplicationTracker.PrimaryContext primaryContext) { - verifyPrimary(); - assert shardRouting.isRelocationTarget() : "only relocation target can update allocation IDs from primary context: " + shardRouting; + assert shardRouting.primary() && shardRouting.isRelocationTarget() : "only primary relocation target can update allocation IDs from primary context: " + shardRouting; assert primaryContext.getCheckpointStates().containsKey(routingEntry().allocationId().getId()) && getLocalCheckpoint() == primaryContext.getCheckpointStates().get(routingEntry().allocationId().getId()).getLocalCheckpoint(); synchronized (mutex) { @@ -1892,7 +1887,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl * @return {@code true} if there is at least one shard pending in-sync, otherwise false */ public boolean pendingInSync() { - verifyPrimary(); + assert assertPrimaryMode(); return replicationTracker.pendingInSync(); } @@ -2209,7 +2204,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl */ public void acquirePrimaryOperationPermit(ActionListener onPermitAcquired, String executorOnDelay, Object debugInfo) { verifyNotClosed(); - verifyPrimary(); + assert shardRouting.primary() : "acquirePrimaryOperationPermit should only be called on primary shard: " + shardRouting; indexShardOperationPermits.acquire(onPermitAcquired, executorOnDelay, false, debugInfo); } @@ -2259,7 +2254,6 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl final ActionListener onPermitAcquired, final String executorOnDelay, final Object debugInfo) { verifyNotClosed(); - verifyReplicationTarget(); if (opPrimaryTerm > pendingPrimaryTerm) { synchronized (mutex) { if (opPrimaryTerm > pendingPrimaryTerm) { @@ -2312,6 +2306,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl operationPrimaryTerm); onPermitAcquired.onFailure(new IllegalStateException(message)); } else { + assert assertReplicationTarget(); try { updateGlobalCheckpointOnReplica(globalCheckpoint, "operation"); } catch (Exception e) { diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 03442be7f06..67a28c9b336 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -31,6 +31,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.FilterDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.Constants; +import org.elasticsearch.Assertions; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.flush.FlushRequest; @@ -560,28 +561,20 @@ public class IndexShardTests extends IndexShardTestCase { ShardRouting primaryRouting = newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), null, true, ShardRoutingState.STARTED, replicaRouting.allocationId()); final long newPrimaryTerm = indexShard.getPendingPrimaryTerm() + between(1, 1000); + CountDownLatch latch = new CountDownLatch(1); indexShard.updateShardState(primaryRouting, newPrimaryTerm, (shard, listener) -> { assertThat(TestTranslog.getCurrentTerm(getTranslog(indexShard)), equalTo(newPrimaryTerm)); + latch.countDown(); }, 0L, Collections.singleton(indexShard.routingEntry().allocationId().getId()), new IndexShardRoutingTable.Builder(indexShard.shardId()).addShard(primaryRouting).build(), Collections.emptySet()); + latch.await(); } else { indexShard = newStartedShard(true); } final long primaryTerm = indexShard.getPendingPrimaryTerm(); assertEquals(0, indexShard.getActiveOperationsCount()); - if (indexShard.routingEntry().isRelocationTarget() == false) { - try { - final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); - indexShard.acquireReplicaOperationPermit(primaryTerm, indexShard.getGlobalCheckpoint(), permitAcquiredFuture, - ThreadPool.Names.WRITE, ""); - permitAcquiredFuture.actionGet(); - fail("shard shouldn't accept operations as replica"); - } catch (IllegalStateException ignored) { - - } - } Releasable operation1 = acquirePrimaryOperationPermitBlockingly(indexShard); assertEquals(1, indexShard.getActiveOperationsCount()); Releasable operation2 = acquirePrimaryOperationPermitBlockingly(indexShard); @@ -590,6 +583,22 @@ public class IndexShardTests extends IndexShardTestCase { Releasables.close(operation1, operation2); assertEquals(0, indexShard.getActiveOperationsCount()); + if (Assertions.ENABLED && indexShard.routingEntry().isRelocationTarget() == false) { + assertThat(expectThrows(AssertionError.class, () -> indexShard.acquireReplicaOperationPermit(primaryTerm, + indexShard.getGlobalCheckpoint(), new ActionListener() { + @Override + public void onResponse(Releasable releasable) { + fail(); + } + + @Override + public void onFailure(Exception e) { + fail(); + } + }, + ThreadPool.Names.WRITE, "")).getMessage(), containsString("in primary mode cannot be a replication target")); + } + closeShards(indexShard); } @@ -647,11 +656,11 @@ public class IndexShardTests extends IndexShardTestCase { logger.info("shard routing to {}", shardRouting); assertEquals(0, indexShard.getActiveOperationsCount()); - if (shardRouting.primary() == false) { - final IllegalStateException e = - expectThrows(IllegalStateException.class, + if (shardRouting.primary() == false && Assertions.ENABLED) { + final AssertionError e = + expectThrows(AssertionError.class, () -> indexShard.acquirePrimaryOperationPermit(null, ThreadPool.Names.WRITE, "")); - assertThat(e, hasToString(containsString("shard " + shardRouting + " is not a primary"))); + assertThat(e, hasToString(containsString("acquirePrimaryOperationPermit should only be called on primary shard"))); } final long primaryTerm = indexShard.getPendingPrimaryTerm(); From b3e15851a2830e93b0e34924592e6978057374bd Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Tue, 7 Aug 2018 09:36:37 -0600 Subject: [PATCH 03/20] [TEST] Comment out account breaker assertion while diagnosing Relates to #30290 --- .../java/org/elasticsearch/test/InternalTestCluster.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index a0468091c84..5b4f528913f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2008,8 +2008,10 @@ public final class InternalTestCluster extends TestCluster { final CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class, nodeAndClient.node); CircuitBreaker fdBreaker = breakerService.getBreaker(CircuitBreaker.FIELDDATA); assertThat("Fielddata breaker not reset to 0 on node: " + name, fdBreaker.getUsed(), equalTo(0L)); - CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING); - assertThat("Accounting breaker not reset to 0 on node: " + name, acctBreaker.getUsed(), equalTo(0L)); + // TODO: This is commented out while Lee looks into the failures + // See: https://github.com/elastic/elasticsearch/issues/30290 + // CircuitBreaker acctBreaker = breakerService.getBreaker(CircuitBreaker.ACCOUNTING); + // assertThat("Accounting breaker not reset to 0 on node: " + name, acctBreaker.getUsed(), equalTo(0L)); // Anything that uses transport or HTTP can increase the // request breaker (because they use bigarrays), because of // that the breaker can sometimes be incremented from ping From 6d50d8b5a91b971af5f20235ac580fa16a0935a0 Mon Sep 17 00:00:00 2001 From: Benjamin Trent Date: Tue, 7 Aug 2018 10:51:52 -0500 Subject: [PATCH 04/20] Adding job process pojos to protocol pkg (#32657) * Adding job process pojos to protocol pkg * Removing unused `RESULTS_FIELD` * Addressing PR comments, removing unnecessary methods --- .../xpack/ml/datafeed/DatafeedConfig.java | 8 +- .../xpack/ml/datafeed/DatafeedUpdate.java | 5 +- .../protocol/xpack/ml/job/config/Job.java | 25 ++ .../xpack/ml/job/process/DataCounts.java | 414 ++++++++++++++++++ .../xpack/ml/job/process/ModelSizeStats.java | 293 +++++++++++++ .../xpack/ml/job/process/ModelSnapshot.java | 330 ++++++++++++++ .../xpack/ml/job/process/Quantiles.java | 112 +++++ .../xpack/ml/job/process/TimeUtil.java | 48 ++ .../xpack/ml/job/results/AnomalyRecord.java | 5 +- .../protocol/xpack/ml/job/results/Bucket.java | 5 +- .../ml/job/results/BucketInfluencer.java | 5 +- .../ml/job/results/CategoryDefinition.java | 5 +- .../xpack/ml/job/results/Influencer.java | 5 +- .../xpack/ml/job/results/OverallBucket.java | 5 +- .../protocol/xpack/ml/job/results/Result.java | 1 - .../xpack/ml/job/process/DataCountsTests.java | 130 ++++++ .../ml/job/process/ModelSizeStatsTests.java | 99 +++++ .../ml/job/process/ModelSnapshotTests.java | 186 ++++++++ .../xpack/ml/job/process/QuantilesTests.java | 91 ++++ 19 files changed, 1753 insertions(+), 19 deletions(-) create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/TimeUtil.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java index 85b7a0acea6..929d4dacb90 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedConfig.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -54,7 +55,6 @@ public class DatafeedConfig implements ToXContentObject { public static final ParseField FREQUENCY = new ParseField("frequency"); public static final ParseField INDEXES = new ParseField("indexes"); public static final ParseField INDICES = new ParseField("indices"); - public static final ParseField JOB_ID = new ParseField("job_id"); public static final ParseField TYPES = new ParseField("types"); public static final ParseField QUERY = new ParseField("query"); public static final ParseField SCROLL_SIZE = new ParseField("scroll_size"); @@ -68,7 +68,7 @@ public class DatafeedConfig implements ToXContentObject { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), ID); - PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareStringArray(Builder::setIndices, INDEXES); PARSER.declareStringArray(Builder::setIndices, INDICES); @@ -176,7 +176,7 @@ public class DatafeedConfig implements ToXContentObject { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ID.getPreferredName(), id); - builder.field(JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); if (queryDelay != null) { builder.field(QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); } @@ -257,7 +257,7 @@ public class DatafeedConfig implements ToXContentObject { public Builder(String id, String jobId) { this.id = Objects.requireNonNull(id, ID.getPreferredName()); - this.jobId = Objects.requireNonNull(jobId, JOB_ID.getPreferredName()); + this.jobId = Objects.requireNonNull(jobId, Job.ID.getPreferredName()); } public Builder(DatafeedConfig config) { diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java index 6afcdf1d2d8..787bdf06e5e 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/datafeed/DatafeedUpdate.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -49,7 +50,7 @@ public class DatafeedUpdate implements ToXContentObject { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), DatafeedConfig.ID); - PARSER.declareString(Builder::setJobId, DatafeedConfig.JOB_ID); + PARSER.declareString(Builder::setJobId, Job.ID); PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDEXES); PARSER.declareStringArray(Builder::setIndices, DatafeedConfig.INDICES); PARSER.declareStringArray(Builder::setTypes, DatafeedConfig.TYPES); @@ -112,7 +113,7 @@ public class DatafeedUpdate implements ToXContentObject { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(DatafeedConfig.ID.getPreferredName(), id); - addOptionalField(builder, DatafeedConfig.JOB_ID, jobId); + addOptionalField(builder, Job.ID, jobId); if (queryDelay != null) { builder.field(DatafeedConfig.QUERY_DELAY.getPreferredName(), queryDelay.getStringRep()); } diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java new file mode 100644 index 00000000000..4f34f85aa1a --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/config/Job.java @@ -0,0 +1,25 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; + +public class Job { + public static final ParseField ID = new ParseField("job_id"); +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java new file mode 100644 index 00000000000..79ad031f5dc --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCounts.java @@ -0,0 +1,414 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Job processed record counts. + *

+ * The getInput... methods return the actual number of + * fields/records sent the the API including invalid records. + * The getProcessed... methods are the number sent to the + * Engine. + *

+ * The inputRecordCount field is calculated so it + * should not be set in deserialization but it should be serialised + * so the field is visible. + */ +public class DataCounts implements ToXContentObject { + + public static final ParseField PROCESSED_RECORD_COUNT = new ParseField("processed_record_count"); + public static final ParseField PROCESSED_FIELD_COUNT = new ParseField("processed_field_count"); + public static final ParseField INPUT_BYTES = new ParseField("input_bytes"); + public static final ParseField INPUT_RECORD_COUNT = new ParseField("input_record_count"); + public static final ParseField INPUT_FIELD_COUNT = new ParseField("input_field_count"); + public static final ParseField INVALID_DATE_COUNT = new ParseField("invalid_date_count"); + public static final ParseField MISSING_FIELD_COUNT = new ParseField("missing_field_count"); + public static final ParseField OUT_OF_ORDER_TIME_COUNT = new ParseField("out_of_order_timestamp_count"); + public static final ParseField EMPTY_BUCKET_COUNT = new ParseField("empty_bucket_count"); + public static final ParseField SPARSE_BUCKET_COUNT = new ParseField("sparse_bucket_count"); + public static final ParseField BUCKET_COUNT = new ParseField("bucket_count"); + public static final ParseField EARLIEST_RECORD_TIME = new ParseField("earliest_record_timestamp"); + public static final ParseField LATEST_RECORD_TIME = new ParseField("latest_record_timestamp"); + public static final ParseField LAST_DATA_TIME = new ParseField("last_data_time"); + public static final ParseField LATEST_EMPTY_BUCKET_TIME = new ParseField("latest_empty_bucket_timestamp"); + public static final ParseField LATEST_SPARSE_BUCKET_TIME = new ParseField("latest_sparse_bucket_timestamp"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_counts", true, + a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3], (long) a[4], (long) a[5], (long) a[6], + (long) a[7], (long) a[8], (long) a[9], (long) a[10], (Date) a[11], (Date) a[12], (Date) a[13], (Date) a[14], + (Date) a[15])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_RECORD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_BYTES); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INVALID_DATE_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MISSING_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_OF_ORDER_TIME_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), EMPTY_BUCKET_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), SPARSE_BUCKET_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_COUNT); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), + (p) -> TimeUtil.parseTimeField(p, EARLIEST_RECORD_TIME.getPreferredName()), + EARLIEST_RECORD_TIME, + ValueType.VALUE); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), + (p) -> TimeUtil.parseTimeField(p, LATEST_RECORD_TIME.getPreferredName()), + LATEST_RECORD_TIME, + ValueType.VALUE); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), + (p) -> TimeUtil.parseTimeField(p, LAST_DATA_TIME.getPreferredName()), + LAST_DATA_TIME, + ValueType.VALUE); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), + (p) -> TimeUtil.parseTimeField(p, LATEST_EMPTY_BUCKET_TIME.getPreferredName()), + LATEST_EMPTY_BUCKET_TIME, + ValueType.VALUE); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), + (p) -> TimeUtil.parseTimeField(p, LATEST_SPARSE_BUCKET_TIME.getPreferredName()), + LATEST_SPARSE_BUCKET_TIME, + ValueType.VALUE); + } + + private final String jobId; + private long processedRecordCount; + private long processedFieldCount; + private long inputBytes; + private long inputFieldCount; + private long invalidDateCount; + private long missingFieldCount; + private long outOfOrderTimeStampCount; + private long emptyBucketCount; + private long sparseBucketCount; + private long bucketCount; + private Date earliestRecordTimeStamp; + private Date latestRecordTimeStamp; + private Date lastDataTimeStamp; + private Date latestEmptyBucketTimeStamp; + private Date latestSparseBucketTimeStamp; + + public DataCounts(String jobId, long processedRecordCount, long processedFieldCount, long inputBytes, + long inputFieldCount, long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount, + long emptyBucketCount, long sparseBucketCount, long bucketCount, + Date earliestRecordTimeStamp, Date latestRecordTimeStamp, Date lastDataTimeStamp, + Date latestEmptyBucketTimeStamp, Date latestSparseBucketTimeStamp) { + this.jobId = jobId; + this.processedRecordCount = processedRecordCount; + this.processedFieldCount = processedFieldCount; + this.inputBytes = inputBytes; + this.inputFieldCount = inputFieldCount; + this.invalidDateCount = invalidDateCount; + this.missingFieldCount = missingFieldCount; + this.outOfOrderTimeStampCount = outOfOrderTimeStampCount; + this.emptyBucketCount = emptyBucketCount; + this.sparseBucketCount = sparseBucketCount; + this.bucketCount = bucketCount; + this.latestRecordTimeStamp = latestRecordTimeStamp; + this.earliestRecordTimeStamp = earliestRecordTimeStamp; + this.lastDataTimeStamp = lastDataTimeStamp; + this.latestEmptyBucketTimeStamp = latestEmptyBucketTimeStamp; + this.latestSparseBucketTimeStamp = latestSparseBucketTimeStamp; + } + + DataCounts(String jobId) { + this.jobId = jobId; + } + + public DataCounts(DataCounts lhs) { + jobId = lhs.jobId; + processedRecordCount = lhs.processedRecordCount; + processedFieldCount = lhs.processedFieldCount; + inputBytes = lhs.inputBytes; + inputFieldCount = lhs.inputFieldCount; + invalidDateCount = lhs.invalidDateCount; + missingFieldCount = lhs.missingFieldCount; + outOfOrderTimeStampCount = lhs.outOfOrderTimeStampCount; + emptyBucketCount = lhs.emptyBucketCount; + sparseBucketCount = lhs.sparseBucketCount; + bucketCount = lhs.bucketCount; + latestRecordTimeStamp = lhs.latestRecordTimeStamp; + earliestRecordTimeStamp = lhs.earliestRecordTimeStamp; + lastDataTimeStamp = lhs.lastDataTimeStamp; + latestEmptyBucketTimeStamp = lhs.latestEmptyBucketTimeStamp; + latestSparseBucketTimeStamp = lhs.latestSparseBucketTimeStamp; + } + + public String getJobId() { + return jobId; + } + + /** + * Number of records processed by this job. + * This value is the number of records sent passed on to + * the engine i.e. {@linkplain #getInputRecordCount()} minus + * records with bad dates or out of order + * + * @return Number of records processed by this job {@code long} + */ + public long getProcessedRecordCount() { + return processedRecordCount; + } + + /** + * Number of data points (processed record count * the number + * of analysed fields) processed by this job. This count does + * not include the time field. + * + * @return Number of data points processed by this job {@code long} + */ + public long getProcessedFieldCount() { + return processedFieldCount; + } + + /** + * Total number of input records read. + * This = processed record count + date parse error records count + * + out of order record count. + *

+ * Records with missing fields are counted as they are still written. + * + * @return Total number of input records read {@code long} + */ + public long getInputRecordCount() { + return processedRecordCount + outOfOrderTimeStampCount + + invalidDateCount; + } + + /** + * The total number of bytes sent to this job. + * This value includes the bytes from any records + * that have been discarded for any reason + * e.g. because the date cannot be read + * + * @return Volume in bytes + */ + public long getInputBytes() { + return inputBytes; + } + + /** + * The total number of fields sent to the job + * including fields that aren't analysed. + * + * @return The total number of fields sent to the job + */ + public long getInputFieldCount() { + return inputFieldCount; + } + + /** + * The number of records with an invalid date field that could + * not be parsed or converted to epoch time. + * + * @return The number of records with an invalid date field + */ + public long getInvalidDateCount() { + return invalidDateCount; + } + + /** + * The number of missing fields that had been + * configured for analysis. + * + * @return The number of missing fields + */ + public long getMissingFieldCount() { + return missingFieldCount; + } + + /** + * The number of records with a timestamp that is + * before the time of the latest record. Records should + * be in ascending chronological order + * + * @return The number of records with a timestamp that is before the time of the latest record + */ + public long getOutOfOrderTimeStampCount() { + return outOfOrderTimeStampCount; + } + + /** + * The number of buckets with no records in it. Used to measure general data fitness and/or + * configuration problems (bucket span). + * + * @return Number of empty buckets processed by this job {@code long} + */ + public long getEmptyBucketCount() { + return emptyBucketCount; + } + + /** + * The number of buckets with few records compared to the overall counts. + * Used to measure general data fitness and/or configuration problems (bucket span). + * + * @return Number of sparse buckets processed by this job {@code long} + */ + public long getSparseBucketCount() { + return sparseBucketCount; + } + + /** + * The number of buckets overall. + * + * @return Number of buckets processed by this job {@code long} + */ + public long getBucketCount() { + return bucketCount; + } + + /** + * The time of the first record seen. + * + * @return The first record time + */ + public Date getEarliestRecordTimeStamp() { + return earliestRecordTimeStamp; + } + + /** + * The time of the latest record seen. + * + * @return Latest record time + */ + public Date getLatestRecordTimeStamp() { + return latestRecordTimeStamp; + } + + /** + * The wall clock time the latest record was seen. + * + * @return Wall clock time of the lastest record + */ + public Date getLastDataTimeStamp() { + return lastDataTimeStamp; + } + + /** + * The time of the latest empty bucket seen. + * + * @return Latest empty bucket time + */ + public Date getLatestEmptyBucketTimeStamp() { + return latestEmptyBucketTimeStamp; + } + + /** + * The time of the latest sparse bucket seen. + * + * @return Latest sparse bucket time + */ + public Date getLatestSparseBucketTimeStamp() { + return latestSparseBucketTimeStamp; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(PROCESSED_RECORD_COUNT.getPreferredName(), processedRecordCount); + builder.field(PROCESSED_FIELD_COUNT.getPreferredName(), processedFieldCount); + builder.field(INPUT_BYTES.getPreferredName(), inputBytes); + builder.field(INPUT_FIELD_COUNT.getPreferredName(), inputFieldCount); + builder.field(INVALID_DATE_COUNT.getPreferredName(), invalidDateCount); + builder.field(MISSING_FIELD_COUNT.getPreferredName(), missingFieldCount); + builder.field(OUT_OF_ORDER_TIME_COUNT.getPreferredName(), outOfOrderTimeStampCount); + builder.field(EMPTY_BUCKET_COUNT.getPreferredName(), emptyBucketCount); + builder.field(SPARSE_BUCKET_COUNT.getPreferredName(), sparseBucketCount); + builder.field(BUCKET_COUNT.getPreferredName(), bucketCount); + if (earliestRecordTimeStamp != null) { + builder.timeField(EARLIEST_RECORD_TIME.getPreferredName(), EARLIEST_RECORD_TIME.getPreferredName() + "_string", + earliestRecordTimeStamp.getTime()); + } + if (latestRecordTimeStamp != null) { + builder.timeField(LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string", + latestRecordTimeStamp.getTime()); + } + if (lastDataTimeStamp != null) { + builder.timeField(LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + "_string", + lastDataTimeStamp.getTime()); + } + if (latestEmptyBucketTimeStamp != null) { + builder.timeField(LATEST_EMPTY_BUCKET_TIME.getPreferredName(), LATEST_EMPTY_BUCKET_TIME.getPreferredName() + "_string", + latestEmptyBucketTimeStamp.getTime()); + } + if (latestSparseBucketTimeStamp != null) { + builder.timeField(LATEST_SPARSE_BUCKET_TIME.getPreferredName(), LATEST_SPARSE_BUCKET_TIME.getPreferredName() + "_string", + latestSparseBucketTimeStamp.getTime()); + } + builder.field(INPUT_RECORD_COUNT.getPreferredName(), getInputRecordCount()); + + builder.endObject(); + return builder; + } + + /** + * Equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + DataCounts that = (DataCounts) other; + + return Objects.equals(this.jobId, that.jobId) && + this.processedRecordCount == that.processedRecordCount && + this.processedFieldCount == that.processedFieldCount && + this.inputBytes == that.inputBytes && + this.inputFieldCount == that.inputFieldCount && + this.invalidDateCount == that.invalidDateCount && + this.missingFieldCount == that.missingFieldCount && + this.outOfOrderTimeStampCount == that.outOfOrderTimeStampCount && + this.emptyBucketCount == that.emptyBucketCount && + this.sparseBucketCount == that.sparseBucketCount && + this.bucketCount == that.bucketCount && + Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) && + Objects.equals(this.earliestRecordTimeStamp, that.earliestRecordTimeStamp) && + Objects.equals(this.lastDataTimeStamp, that.lastDataTimeStamp) && + Objects.equals(this.latestEmptyBucketTimeStamp, that.latestEmptyBucketTimeStamp) && + Objects.equals(this.latestSparseBucketTimeStamp, that.latestSparseBucketTimeStamp); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, processedRecordCount, processedFieldCount, + inputBytes, inputFieldCount, invalidDateCount, missingFieldCount, + outOfOrderTimeStampCount, lastDataTimeStamp, emptyBucketCount, sparseBucketCount, bucketCount, + latestRecordTimeStamp, earliestRecordTimeStamp, latestEmptyBucketTimeStamp, latestSparseBucketTimeStamp); + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java new file mode 100644 index 00000000000..e45e25f1aef --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStats.java @@ -0,0 +1,293 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; +import org.elasticsearch.protocol.xpack.ml.job.results.Result; + +import java.io.IOException; +import java.util.Date; +import java.util.Locale; +import java.util.Objects; + +/** + * Provide access to the C++ model memory usage numbers for the Java process. + */ +public class ModelSizeStats implements ToXContentObject { + + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "model_size_stats"; + public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE); + + /** + * Field Names + */ + public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes"); + public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count"); + public static final ParseField TOTAL_OVER_FIELD_COUNT_FIELD = new ParseField("total_over_field_count"); + public static final ParseField TOTAL_PARTITION_FIELD_COUNT_FIELD = new ParseField("total_partition_field_count"); + public static final ParseField BUCKET_ALLOCATION_FAILURES_COUNT_FIELD = new ParseField("bucket_allocation_failures_count"); + public static final ParseField MEMORY_STATUS_FIELD = new ParseField("memory_status"); + public static final ParseField LOG_TIME_FIELD = new ParseField("log_time"); + public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, a -> new Builder((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); + PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalByFieldCount, TOTAL_BY_FIELD_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalOverFieldCount, TOTAL_OVER_FIELD_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalPartitionFieldCount, TOTAL_PARTITION_FIELD_COUNT_FIELD); + PARSER.declareField(Builder::setLogTime, + (p) -> TimeUtil.parseTimeField(p, LOG_TIME_FIELD.getPreferredName()), + LOG_TIME_FIELD, + ValueType.VALUE); + PARSER.declareField(Builder::setTimestamp, + (p) -> TimeUtil.parseTimeField(p, TIMESTAMP_FIELD.getPreferredName()), + TIMESTAMP_FIELD, + ValueType.VALUE); + PARSER.declareField(Builder::setMemoryStatus, p -> MemoryStatus.fromString(p.text()), MEMORY_STATUS_FIELD, ValueType.STRING); + } + + /** + * The status of the memory monitored by the ResourceMonitor. OK is default, + * SOFT_LIMIT means that the models have done some aggressive pruning to + * keep the memory below the limit, and HARD_LIMIT means that samples have + * been dropped + */ + public enum MemoryStatus { + OK, SOFT_LIMIT, HARD_LIMIT; + + public static MemoryStatus fromString(String statusName) { + return valueOf(statusName.trim().toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + private final String jobId; + private final long modelBytes; + private final long totalByFieldCount; + private final long totalOverFieldCount; + private final long totalPartitionFieldCount; + private final long bucketAllocationFailuresCount; + private final MemoryStatus memoryStatus; + private final Date timestamp; + private final Date logTime; + + private ModelSizeStats(String jobId, long modelBytes, long totalByFieldCount, long totalOverFieldCount, + long totalPartitionFieldCount, long bucketAllocationFailuresCount, MemoryStatus memoryStatus, + Date timestamp, Date logTime) { + this.jobId = jobId; + this.modelBytes = modelBytes; + this.totalByFieldCount = totalByFieldCount; + this.totalOverFieldCount = totalOverFieldCount; + this.totalPartitionFieldCount = totalPartitionFieldCount; + this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; + this.memoryStatus = memoryStatus; + this.timestamp = timestamp; + this.logTime = logTime; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes); + builder.field(TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), totalByFieldCount); + builder.field(TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), totalOverFieldCount); + builder.field(TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), totalPartitionFieldCount); + builder.field(BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName(), bucketAllocationFailuresCount); + builder.field(MEMORY_STATUS_FIELD.getPreferredName(), memoryStatus); + builder.timeField(LOG_TIME_FIELD.getPreferredName(), LOG_TIME_FIELD.getPreferredName() + "_string", logTime.getTime()); + if (timestamp != null) { + builder.timeField(TIMESTAMP_FIELD.getPreferredName(), TIMESTAMP_FIELD.getPreferredName() + "_string", timestamp.getTime()); + } + + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public long getModelBytes() { + return modelBytes; + } + + public long getTotalByFieldCount() { + return totalByFieldCount; + } + + public long getTotalPartitionFieldCount() { + return totalPartitionFieldCount; + } + + public long getTotalOverFieldCount() { + return totalOverFieldCount; + } + + public long getBucketAllocationFailuresCount() { + return bucketAllocationFailuresCount; + } + + public MemoryStatus getMemoryStatus() { + return memoryStatus; + } + + /** + * The timestamp of the last processed record when this instance was created. + * + * @return The record time + */ + public Date getTimestamp() { + return timestamp; + } + + /** + * The wall clock time at the point when this instance was created. + * + * @return The wall clock time + */ + public Date getLogTime() { + return logTime; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, + this.bucketAllocationFailuresCount, memoryStatus, timestamp, logTime); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + ModelSizeStats that = (ModelSizeStats) other; + + return this.modelBytes == that.modelBytes && this.totalByFieldCount == that.totalByFieldCount + && this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount + && this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount + && Objects.equals(this.memoryStatus, that.memoryStatus) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.logTime, that.logTime) + && Objects.equals(this.jobId, that.jobId); + } + + public static class Builder { + + private final String jobId; + private long modelBytes; + private long totalByFieldCount; + private long totalOverFieldCount; + private long totalPartitionFieldCount; + private long bucketAllocationFailuresCount; + private MemoryStatus memoryStatus; + private Date timestamp; + private Date logTime; + + public Builder(String jobId) { + this.jobId = jobId; + memoryStatus = MemoryStatus.OK; + logTime = new Date(); + } + + public Builder(ModelSizeStats modelSizeStats) { + this.jobId = modelSizeStats.jobId; + this.modelBytes = modelSizeStats.modelBytes; + this.totalByFieldCount = modelSizeStats.totalByFieldCount; + this.totalOverFieldCount = modelSizeStats.totalOverFieldCount; + this.totalPartitionFieldCount = modelSizeStats.totalPartitionFieldCount; + this.bucketAllocationFailuresCount = modelSizeStats.bucketAllocationFailuresCount; + this.memoryStatus = modelSizeStats.memoryStatus; + this.timestamp = modelSizeStats.timestamp; + this.logTime = modelSizeStats.logTime; + } + + public Builder setModelBytes(long modelBytes) { + this.modelBytes = modelBytes; + return this; + } + + public Builder setTotalByFieldCount(long totalByFieldCount) { + this.totalByFieldCount = totalByFieldCount; + return this; + } + + public Builder setTotalPartitionFieldCount(long totalPartitionFieldCount) { + this.totalPartitionFieldCount = totalPartitionFieldCount; + return this; + } + + public Builder setTotalOverFieldCount(long totalOverFieldCount) { + this.totalOverFieldCount = totalOverFieldCount; + return this; + } + + public Builder setBucketAllocationFailuresCount(long bucketAllocationFailuresCount) { + this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; + return this; + } + + public Builder setMemoryStatus(MemoryStatus memoryStatus) { + Objects.requireNonNull(memoryStatus, "[" + MEMORY_STATUS_FIELD.getPreferredName() + "] must not be null"); + this.memoryStatus = memoryStatus; + return this; + } + + public Builder setTimestamp(Date timestamp) { + this.timestamp = timestamp; + return this; + } + + public Builder setLogTime(Date logTime) { + this.logTime = logTime; + return this; + } + + public ModelSizeStats build() { + return new ModelSizeStats(jobId, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, + bucketAllocationFailuresCount, memoryStatus, timestamp, logTime); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java new file mode 100644 index 00000000000..ddf6a7984bf --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshot.java @@ -0,0 +1,330 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * ModelSnapshot Result POJO + */ +public class ModelSnapshot implements ToXContentObject { + /** + * Field Names + */ + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField SNAPSHOT_DOC_COUNT = new ParseField("snapshot_doc_count"); + public static final ParseField LATEST_RECORD_TIME = new ParseField("latest_record_time_stamp"); + public static final ParseField LATEST_RESULT_TIME = new ParseField("latest_result_time_stamp"); + public static final ParseField QUANTILES = new ParseField("quantiles"); + public static final ParseField RETAIN = new ParseField("retain"); + public static final ParseField MIN_VERSION = new ParseField("min_version"); + public static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id"); + + public static final ObjectParser PARSER = new ObjectParser<>("model_snapshot", true, Builder::new); + + static { + PARSER.declareString(Builder::setJobId, Job.ID); + PARSER.declareString(Builder::setMinVersion, MIN_VERSION); + PARSER.declareField(Builder::setTimestamp, + (p) -> TimeUtil.parseTimeField(p, TIMESTAMP.getPreferredName()), + TIMESTAMP, + ValueType.VALUE); + PARSER.declareString(Builder::setDescription, DESCRIPTION); + PARSER.declareString(Builder::setSnapshotId, SNAPSHOT_ID); + PARSER.declareInt(Builder::setSnapshotDocCount, SNAPSHOT_DOC_COUNT); + PARSER.declareObject(Builder::setModelSizeStats, ModelSizeStats.PARSER, + ModelSizeStats.RESULT_TYPE_FIELD); + PARSER.declareField(Builder::setLatestRecordTimeStamp, + (p) -> TimeUtil.parseTimeField(p, LATEST_RECORD_TIME.getPreferredName()), + LATEST_RECORD_TIME, + ValueType.VALUE); + PARSER.declareField(Builder::setLatestResultTimeStamp, + (p) -> TimeUtil.parseTimeField(p, LATEST_RESULT_TIME.getPreferredName()), + LATEST_RESULT_TIME, + ValueType.VALUE); + PARSER.declareObject(Builder::setQuantiles, Quantiles.PARSER, QUANTILES); + PARSER.declareBoolean(Builder::setRetain, RETAIN); + } + + + private final String jobId; + + /** + * The minimum version a node should have to be able + * to read this model snapshot. + */ + private final Version minVersion; + + private final Date timestamp; + private final String description; + private final String snapshotId; + private final int snapshotDocCount; + private final ModelSizeStats modelSizeStats; + private final Date latestRecordTimeStamp; + private final Date latestResultTimeStamp; + private final Quantiles quantiles; + private final boolean retain; + + + private ModelSnapshot(String jobId, Version minVersion, Date timestamp, String description, String snapshotId, int snapshotDocCount, + ModelSizeStats modelSizeStats, Date latestRecordTimeStamp, Date latestResultTimeStamp, Quantiles quantiles, + boolean retain) { + this.jobId = jobId; + this.minVersion = minVersion; + this.timestamp = timestamp; + this.description = description; + this.snapshotId = snapshotId; + this.snapshotDocCount = snapshotDocCount; + this.modelSizeStats = modelSizeStats; + this.latestRecordTimeStamp = latestRecordTimeStamp; + this.latestResultTimeStamp = latestResultTimeStamp; + this.quantiles = quantiles; + this.retain = retain; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(MIN_VERSION.getPreferredName(), minVersion); + if (timestamp != null) { + builder.timeField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + } + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + if (snapshotId != null) { + builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); + } + builder.field(SNAPSHOT_DOC_COUNT.getPreferredName(), snapshotDocCount); + if (modelSizeStats != null) { + builder.field(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName(), modelSizeStats); + } + if (latestRecordTimeStamp != null) { + builder.timeField(LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string", + latestRecordTimeStamp.getTime()); + } + if (latestResultTimeStamp != null) { + builder.timeField(LATEST_RESULT_TIME.getPreferredName(), LATEST_RESULT_TIME.getPreferredName() + "_string", + latestResultTimeStamp.getTime()); + } + if (quantiles != null) { + builder.field(QUANTILES.getPreferredName(), quantiles); + } + builder.field(RETAIN.getPreferredName(), retain); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public Version getMinVersion() { + return minVersion; + } + + public Date getTimestamp() { + return timestamp; + } + + public String getDescription() { + return description; + } + + public String getSnapshotId() { + return snapshotId; + } + + public int getSnapshotDocCount() { + return snapshotDocCount; + } + + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + public Quantiles getQuantiles() { + return quantiles; + } + + public Date getLatestRecordTimeStamp() { + return latestRecordTimeStamp; + } + + public Date getLatestResultTimeStamp() { + return latestResultTimeStamp; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, minVersion, timestamp, description, snapshotId, quantiles, snapshotDocCount, modelSizeStats, + latestRecordTimeStamp, latestResultTimeStamp, retain); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + ModelSnapshot that = (ModelSnapshot) other; + + return Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.minVersion, that.minVersion) + && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.description, that.description) + && Objects.equals(this.snapshotId, that.snapshotId) + && this.snapshotDocCount == that.snapshotDocCount + && Objects.equals(this.modelSizeStats, that.modelSizeStats) + && Objects.equals(this.quantiles, that.quantiles) + && Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) + && Objects.equals(this.latestResultTimeStamp, that.latestResultTimeStamp) + && this.retain == that.retain; + } + + public static class Builder { + private String jobId; + + // Stored snapshot documents created prior to 6.3.0 will have no + // value for min_version. We default it to 5.5.0 as there were + // no model changes between 5.5.0 and 6.3.0. + private Version minVersion = Version.V_5_5_0; + + private Date timestamp; + private String description; + private String snapshotId; + private int snapshotDocCount; + private ModelSizeStats modelSizeStats; + private Date latestRecordTimeStamp; + private Date latestResultTimeStamp; + private Quantiles quantiles; + private boolean retain; + + + public Builder() { + } + + public Builder(String jobId) { + this.jobId = jobId; + } + + public Builder(ModelSnapshot modelSnapshot) { + this.jobId = modelSnapshot.jobId; + this.timestamp = modelSnapshot.timestamp; + this.description = modelSnapshot.description; + this.snapshotId = modelSnapshot.snapshotId; + this.snapshotDocCount = modelSnapshot.snapshotDocCount; + this.modelSizeStats = modelSnapshot.modelSizeStats; + this.latestRecordTimeStamp = modelSnapshot.latestRecordTimeStamp; + this.latestResultTimeStamp = modelSnapshot.latestResultTimeStamp; + this.quantiles = modelSnapshot.quantiles; + this.retain = modelSnapshot.retain; + this.minVersion = modelSnapshot.minVersion; + } + + public Builder setJobId(String jobId) { + this.jobId = jobId; + return this; + } + + Builder setMinVersion(Version minVersion) { + this.minVersion = minVersion; + return this; + } + + Builder setMinVersion(String minVersion) { + this.minVersion = Version.fromString(minVersion); + return this; + } + + public Builder setTimestamp(Date timestamp) { + this.timestamp = timestamp; + return this; + } + + public Builder setDescription(String description) { + this.description = description; + return this; + } + + public Builder setSnapshotId(String snapshotId) { + this.snapshotId = snapshotId; + return this; + } + + public Builder setSnapshotDocCount(int snapshotDocCount) { + this.snapshotDocCount = snapshotDocCount; + return this; + } + + public Builder setModelSizeStats(ModelSizeStats.Builder modelSizeStats) { + this.modelSizeStats = modelSizeStats.build(); + return this; + } + + public Builder setModelSizeStats(ModelSizeStats modelSizeStats) { + this.modelSizeStats = modelSizeStats; + return this; + } + + public Builder setLatestRecordTimeStamp(Date latestRecordTimeStamp) { + this.latestRecordTimeStamp = latestRecordTimeStamp; + return this; + } + + public Builder setLatestResultTimeStamp(Date latestResultTimeStamp) { + this.latestResultTimeStamp = latestResultTimeStamp; + return this; + } + + public Builder setQuantiles(Quantiles quantiles) { + this.quantiles = quantiles; + return this; + } + + public Builder setRetain(boolean value) { + this.retain = value; + return this; + } + + public ModelSnapshot build() { + return new ModelSnapshot(jobId, minVersion, timestamp, description, snapshotId, snapshotDocCount, modelSizeStats, + latestRecordTimeStamp, latestResultTimeStamp, quantiles, retain); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java new file mode 100644 index 00000000000..1c047d6c302 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/Quantiles.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Quantiles Result POJO + */ +public class Quantiles implements ToXContentObject { + + /** + * Field Names + */ + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField QUANTILE_STATE = new ParseField("quantile_state"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("quantiles", true, a -> new Quantiles((String) a[0], (Date) a[1], (String) a[2])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> new Date(p.longValue()), TIMESTAMP, ValueType.LONG); + PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); + } + + private final String jobId; + private final Date timestamp; + private final String quantileState; + + public Quantiles(String jobId, Date timestamp, String quantileState) { + this.jobId = jobId; + this.timestamp = Objects.requireNonNull(timestamp); + this.quantileState = Objects.requireNonNull(quantileState); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (quantileState != null) { + builder.field(QUANTILE_STATE.getPreferredName(), quantileState); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public Date getTimestamp() { + return timestamp; + } + + public String getQuantileState() { + return quantileState; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, quantileState); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + Quantiles that = (Quantiles) other; + + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.quantileState, that.quantileState); + } +} + diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/TimeUtil.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/TimeUtil.java new file mode 100644 index 00000000000..a52b99d0af7 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/process/TimeUtil.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.time.DateFormatters; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +final class TimeUtil { + + /** + * Parse out a Date object given the current parser and field name. + * + * @param parser current XContentParser + * @param fieldName the field's preferred name (utilized in exception) + * @return parsed Date object + * @throws IOException from XContentParser + */ + static Date parseTimeField(XContentParser parser, String fieldName) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return new Date(parser.longValue()); + } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli()); + } + throw new IllegalArgumentException( + "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]"); + } + +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java index 8289032634e..4747f3a48bd 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/AnomalyRecord.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import java.io.IOException; import java.time.format.DateTimeFormatter; @@ -88,7 +89,7 @@ public class AnomalyRecord implements ToXContentObject { static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Result.JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); @@ -159,7 +160,7 @@ public class AnomalyRecord implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Result.JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); builder.field(PROBABILITY.getPreferredName(), probability); builder.field(RECORD_SCORE.getPreferredName(), recordScore); diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java index dc56c7bd262..cbaf83abbad 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Bucket.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import java.io.IOException; import java.time.format.DateTimeFormatter; @@ -61,7 +62,7 @@ public class Bucket implements ToXContentObject { new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, a -> new Bucket((String) a[0], (Date) a[1], (long) a[2])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Result.JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); @@ -104,7 +105,7 @@ public class Bucket implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Result.JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); builder.timeField(Result.TIMESTAMP.getPreferredName(), Result.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java index c556737213e..29d8447cd6a 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/BucketInfluencer.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import java.io.IOException; import java.time.format.DateTimeFormatter; @@ -54,7 +55,7 @@ public class BucketInfluencer implements ToXContentObject { a -> new BucketInfluencer((String) a[0], (Date) a[1], (long) a[2])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Result.JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); @@ -93,7 +94,7 @@ public class BucketInfluencer implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Result.JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); if (influenceField != null) { builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField); diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java index 2b452eeb828..59b59006b33 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/CategoryDefinition.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import java.io.IOException; import java.util.ArrayList; @@ -49,7 +50,7 @@ public class CategoryDefinition implements ToXContentObject { new ConstructingObjectParser<>(TYPE.getPreferredName(), true, a -> new CategoryDefinition((String) a[0])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Result.JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareLong(CategoryDefinition::setCategoryId, CATEGORY_ID); PARSER.declareString(CategoryDefinition::setTerms, TERMS); PARSER.declareString(CategoryDefinition::setRegex, REGEX); @@ -130,7 +131,7 @@ public class CategoryDefinition implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Result.JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); builder.field(CATEGORY_ID.getPreferredName(), categoryId); builder.field(TERMS.getPreferredName(), terms); builder.field(REGEX.getPreferredName(), regex); diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java index ce3a032e54b..51c88883608 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Influencer.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import java.io.IOException; import java.time.format.DateTimeFormatter; @@ -57,7 +58,7 @@ public class Influencer implements ToXContentObject { a -> new Influencer((String) a[0], (String) a[1], (String) a[2], (Date) a[3], (long) a[4])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Result.JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUE); PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { @@ -98,7 +99,7 @@ public class Influencer implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Result.JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField); builder.field(INFLUENCER_FIELD_VALUE.getPreferredName(), influenceValue); diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java index 217f0bf5e21..4f13b4b2664 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/OverallBucket.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.protocol.xpack.ml.job.config.Job; import java.io.IOException; import java.time.format.DateTimeFormatter; @@ -158,7 +159,7 @@ public class OverallBucket implements ToXContentObject { new ConstructingObjectParser<>("job_info", true, a -> new JobInfo((String) a[0], (double) a[1])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Result.JOB_ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareDouble(ConstructingObjectParser.constructorArg(), MAX_ANOMALY_SCORE); } @@ -181,7 +182,7 @@ public class OverallBucket implements ToXContentObject { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(Result.JOB_ID.getPreferredName(), jobId); + builder.field(Job.ID.getPreferredName(), jobId); builder.field(MAX_ANOMALY_SCORE.getPreferredName(), maxAnomalyScore); builder.endObject(); return builder; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java index 0cd8a09da95..cce5fa65ebb 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/ml/job/results/Result.java @@ -28,7 +28,6 @@ public final class Result { /** * Serialisation fields */ - public static final ParseField JOB_ID = new ParseField("job_id"); public static final ParseField TYPE = new ParseField("result"); public static final ParseField RESULT_TYPE = new ParseField("result_type"); public static final ParseField TIMESTAMP = new ParseField("timestamp"); diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java new file mode 100644 index 00000000000..2232e8c88d9 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/DataCountsTests.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.joda.time.DateTime; + +import java.util.Date; + +import static org.hamcrest.Matchers.greaterThan; + +public class DataCountsTests extends AbstractXContentTestCase { + + public static DataCounts createTestInstance(String jobId) { + return new DataCounts(jobId, randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate(), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate(), + new DateTime(randomDateTimeZone()).toDate()); + } + + @Override + public DataCounts createTestInstance() { + return createTestInstance(randomAlphaOfLength(10)); + } + + @Override + protected DataCounts doParseInstance(XContentParser parser) { + return DataCounts.PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + public void testCountsEquals_GivenEqualCounts() { + DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + DataCounts counts2 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + + assertTrue(counts1.equals(counts2)); + assertTrue(counts2.equals(counts1)); + } + + public void testCountsHashCode_GivenEqualCounts() { + DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + DataCounts counts2 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + assertEquals(counts1.hashCode(), counts2.hashCode()); + } + + public void testCountsCopyConstructor() { + DataCounts counts1 = createCounts(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + DataCounts counts2 = new DataCounts(counts1); + + assertEquals(counts1.hashCode(), counts2.hashCode()); + } + + public void testCountCreatedZero() throws Exception { + DataCounts counts = new DataCounts(randomAlphaOfLength(16)); + assertAllFieldsEqualZero(counts); + } + + public void testCountCopyCreatedFieldsNotZero() throws Exception { + DataCounts counts1 = createCounts(1, 200, 400, 3, 4, 5, 6, 7, 8, 9, 1479211200000L, 1479384000000L, 13, 14, 15); + assertAllFieldsGreaterThanZero(counts1); + + DataCounts counts2 = new DataCounts(counts1); + assertAllFieldsGreaterThanZero(counts2); + } + + private void assertAllFieldsEqualZero(DataCounts stats) throws Exception { + assertEquals(0L, stats.getProcessedRecordCount()); + assertEquals(0L, stats.getProcessedFieldCount()); + assertEquals(0L, stats.getInputBytes()); + assertEquals(0L, stats.getInputFieldCount()); + assertEquals(0L, stats.getInputRecordCount()); + assertEquals(0L, stats.getInvalidDateCount()); + assertEquals(0L, stats.getMissingFieldCount()); + assertEquals(0L, stats.getOutOfOrderTimeStampCount()); + } + + private void assertAllFieldsGreaterThanZero(DataCounts stats) throws Exception { + assertThat(stats.getProcessedRecordCount(), greaterThan(0L)); + assertThat(stats.getProcessedFieldCount(), greaterThan(0L)); + assertThat(stats.getInputBytes(), greaterThan(0L)); + assertThat(stats.getInputFieldCount(), greaterThan(0L)); + assertThat(stats.getInputRecordCount(), greaterThan(0L)); + assertThat(stats.getInputRecordCount(), greaterThan(0L)); + assertThat(stats.getInvalidDateCount(), greaterThan(0L)); + assertThat(stats.getMissingFieldCount(), greaterThan(0L)); + assertThat(stats.getOutOfOrderTimeStampCount(), greaterThan(0L)); + assertThat(stats.getLatestRecordTimeStamp().getTime(), greaterThan(0L)); + } + + private static DataCounts createCounts( + long processedRecordCount, long processedFieldCount, long inputBytes, long inputFieldCount, + long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount, + long emptyBucketCount, long sparseBucketCount, long bucketCount, + long earliestRecordTime, long latestRecordTime, long lastDataTimeStamp, long latestEmptyBucketTimeStamp, + long latestSparseBucketTimeStamp) { + + DataCounts counts = new DataCounts("foo", processedRecordCount, processedFieldCount, inputBytes, + inputFieldCount, invalidDateCount, missingFieldCount, outOfOrderTimeStampCount, + emptyBucketCount, sparseBucketCount, bucketCount, + new Date(earliestRecordTime), new Date(latestRecordTime), + new Date(lastDataTimeStamp), new Date(latestEmptyBucketTimeStamp), new Date(latestSparseBucketTimeStamp)); + + return counts; + } + +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java new file mode 100644 index 00000000000..e3341123fb0 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSizeStatsTests.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; +import org.elasticsearch.protocol.xpack.ml.job.process.ModelSizeStats.MemoryStatus; + +import java.util.Date; + +public class ModelSizeStatsTests extends AbstractXContentTestCase { + + public void testDefaultConstructor() { + ModelSizeStats stats = new ModelSizeStats.Builder("foo").build(); + assertEquals(0, stats.getModelBytes()); + assertEquals(0, stats.getTotalByFieldCount()); + assertEquals(0, stats.getTotalOverFieldCount()); + assertEquals(0, stats.getTotalPartitionFieldCount()); + assertEquals(0, stats.getBucketAllocationFailuresCount()); + assertEquals(MemoryStatus.OK, stats.getMemoryStatus()); + } + + public void testSetMemoryStatus_GivenNull() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + + NullPointerException ex = expectThrows(NullPointerException.class, () -> stats.setMemoryStatus(null)); + + assertEquals("[memory_status] must not be null", ex.getMessage()); + } + + public void testSetMemoryStatus_GivenSoftLimit() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + + stats.setMemoryStatus(MemoryStatus.SOFT_LIMIT); + + assertEquals(MemoryStatus.SOFT_LIMIT, stats.build().getMemoryStatus()); + } + + @Override + protected ModelSizeStats createTestInstance() { + return createRandomized(); + } + + public static ModelSizeStats createRandomized() { + ModelSizeStats.Builder stats = new ModelSizeStats.Builder("foo"); + if (randomBoolean()) { + stats.setBucketAllocationFailuresCount(randomNonNegativeLong()); + } + if (randomBoolean()) { + stats.setModelBytes(randomNonNegativeLong()); + } + if (randomBoolean()) { + stats.setTotalByFieldCount(randomNonNegativeLong()); + } + if (randomBoolean()) { + stats.setTotalOverFieldCount(randomNonNegativeLong()); + } + if (randomBoolean()) { + stats.setTotalPartitionFieldCount(randomNonNegativeLong()); + } + if (randomBoolean()) { + stats.setLogTime(new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis())); + } + if (randomBoolean()) { + stats.setTimestamp(new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis())); + } + if (randomBoolean()) { + stats.setMemoryStatus(randomFrom(MemoryStatus.values())); + } + return stats.build(); + } + + @Override + protected ModelSizeStats doParseInstance(XContentParser parser) { + return ModelSizeStats.PARSER.apply(parser, null).build(); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java new file mode 100644 index 00000000000..8c6a9bd83c9 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/ModelSnapshotTests.java @@ -0,0 +1,186 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.Version; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.Date; + +public class ModelSnapshotTests extends AbstractXContentTestCase { + + private static final Date DEFAULT_TIMESTAMP = new Date(); + private static final String DEFAULT_DESCRIPTION = "a snapshot"; + private static final String DEFAULT_ID = "my_id"; + private static final int DEFAULT_DOC_COUNT = 7; + private static final Date DEFAULT_LATEST_RESULT_TIMESTAMP = new Date(12345678901234L); + private static final Date DEFAULT_LATEST_RECORD_TIMESTAMP = new Date(12345678904321L); + private static final boolean DEFAULT_RETAIN = true; + + public void testCopyBuilder() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = new ModelSnapshot.Builder(modelSnapshot1).build(); + assertEquals(modelSnapshot1, modelSnapshot2); + } + + public void testEquals_GivenSameObject() { + ModelSnapshot modelSnapshot = createFullyPopulated().build(); + assertTrue(modelSnapshot.equals(modelSnapshot)); + } + + public void testEquals_GivenObjectOfDifferentClass() { + ModelSnapshot modelSnapshot = createFullyPopulated().build(); + assertFalse(modelSnapshot.equals("a string")); + } + + public void testEquals_GivenEqualModelSnapshots() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated().build(); + + assertEquals(modelSnapshot1, modelSnapshot2); + assertEquals(modelSnapshot2, modelSnapshot1); + assertEquals(modelSnapshot1.hashCode(), modelSnapshot2.hashCode()); + } + + public void testEquals_GivenDifferentTimestamp() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated().setTimestamp( + new Date(modelSnapshot1.getTimestamp().getTime() + 1)).build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentDescription() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated() + .setDescription(modelSnapshot1.getDescription() + " blah").build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentId() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated() + .setSnapshotId(modelSnapshot1.getSnapshotId() + "_2").build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentDocCount() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated() + .setSnapshotDocCount(modelSnapshot1.getSnapshotDocCount() + 1).build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentModelSizeStats() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSizeStats.Builder modelSizeStats = new ModelSizeStats.Builder("foo"); + modelSizeStats.setModelBytes(42L); + ModelSnapshot modelSnapshot2 = createFullyPopulated().setModelSizeStats(modelSizeStats).build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentQuantiles() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated() + .setQuantiles(new Quantiles("foo", modelSnapshot1.getQuantiles().getTimestamp(), + "different state")).build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentLatestResultTimestamp() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated().setLatestResultTimeStamp( + new Date(modelSnapshot1.getLatestResultTimeStamp().getTime() + 1)).build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + public void testEquals_GivenDifferentLatestRecordTimestamp() { + ModelSnapshot modelSnapshot1 = createFullyPopulated().build(); + ModelSnapshot modelSnapshot2 = createFullyPopulated().setLatestRecordTimeStamp( + new Date(modelSnapshot1.getLatestRecordTimeStamp().getTime() + 1)).build(); + + assertFalse(modelSnapshot1.equals(modelSnapshot2)); + assertFalse(modelSnapshot2.equals(modelSnapshot1)); + } + + private static ModelSnapshot.Builder createFullyPopulated() { + ModelSnapshot.Builder modelSnapshot = new ModelSnapshot.Builder(); + modelSnapshot.setJobId("foo"); + modelSnapshot.setMinVersion(Version.CURRENT); + modelSnapshot.setTimestamp(DEFAULT_TIMESTAMP); + modelSnapshot.setDescription(DEFAULT_DESCRIPTION); + modelSnapshot.setSnapshotId(DEFAULT_ID); + modelSnapshot.setSnapshotDocCount(DEFAULT_DOC_COUNT); + ModelSizeStats.Builder modelSizeStatsBuilder = new ModelSizeStats.Builder("foo"); + modelSizeStatsBuilder.setLogTime(null); + modelSnapshot.setModelSizeStats(modelSizeStatsBuilder); + modelSnapshot.setLatestResultTimeStamp(DEFAULT_LATEST_RESULT_TIMESTAMP); + modelSnapshot.setLatestRecordTimeStamp(DEFAULT_LATEST_RECORD_TIMESTAMP); + modelSnapshot.setQuantiles(new Quantiles("foo", DEFAULT_TIMESTAMP, "state")); + modelSnapshot.setRetain(DEFAULT_RETAIN); + return modelSnapshot; + } + + @Override + protected ModelSnapshot createTestInstance() { + return createRandomized(); + } + + public static ModelSnapshot createRandomized() { + ModelSnapshot.Builder modelSnapshot = new ModelSnapshot.Builder(randomAlphaOfLengthBetween(1, 20)); + modelSnapshot.setMinVersion(Version.CURRENT); + modelSnapshot.setTimestamp(new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis())); + modelSnapshot.setDescription(randomAlphaOfLengthBetween(1, 20)); + modelSnapshot.setSnapshotId(randomAlphaOfLengthBetween(1, 20)); + modelSnapshot.setSnapshotDocCount(randomInt()); + modelSnapshot.setModelSizeStats(ModelSizeStatsTests.createRandomized()); + modelSnapshot.setLatestResultTimeStamp( + new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis())); + modelSnapshot.setLatestRecordTimeStamp( + new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis())); + modelSnapshot.setQuantiles(QuantilesTests.createRandomized()); + modelSnapshot.setRetain(randomBoolean()); + return modelSnapshot.build(); + } + + @Override + protected ModelSnapshot doParseInstance(XContentParser parser){ + return ModelSnapshot.PARSER.apply(parser, null).build(); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java new file mode 100644 index 00000000000..77ae21bc6f8 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/ml/job/process/QuantilesTests.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack.ml.job.process; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.Date; + + +public class QuantilesTests extends AbstractXContentTestCase { + + public void testEquals_GivenSameObject() { + Quantiles quantiles = new Quantiles("foo", new Date(0L), "foo"); + assertTrue(quantiles.equals(quantiles)); + } + + + public void testEquals_GivenDifferentClassObject() { + Quantiles quantiles = new Quantiles("foo", new Date(0L), "foo"); + assertFalse(quantiles.equals("not a quantiles object")); + } + + + public void testEquals_GivenEqualQuantilesObject() { + Quantiles quantiles1 = new Quantiles("foo", new Date(0L), "foo"); + + Quantiles quantiles2 = new Quantiles("foo", new Date(0L), "foo"); + + assertTrue(quantiles1.equals(quantiles2)); + assertTrue(quantiles2.equals(quantiles1)); + } + + + public void testEquals_GivenDifferentState() { + Quantiles quantiles1 = new Quantiles("foo", new Date(0L), "bar1"); + + Quantiles quantiles2 = new Quantiles("foo", new Date(0L), "bar2"); + + assertFalse(quantiles1.equals(quantiles2)); + assertFalse(quantiles2.equals(quantiles1)); + } + + + public void testHashCode_GivenEqualObject() { + Quantiles quantiles1 = new Quantiles("foo", new Date(0L), "foo"); + + Quantiles quantiles2 = new Quantiles("foo", new Date(0L), "foo"); + + assertEquals(quantiles1.hashCode(), quantiles2.hashCode()); + } + + + @Override + protected Quantiles createTestInstance() { + return createRandomized(); + } + + public static Quantiles createRandomized() { + return new Quantiles(randomAlphaOfLengthBetween(1, 20), + new Date(TimeValue.parseTimeValue(randomTimeValue(), "test").millis()), + randomAlphaOfLengthBetween(0, 1000)); + } + + @Override + protected Quantiles doParseInstance(XContentParser parser) { + return Quantiles.PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} From 2608012422b7bb3621cdcc9f12736139c1ae9bdb Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 7 Aug 2018 16:59:56 +0100 Subject: [PATCH 05/20] Add temporary directory cleanup workarounds (#32615) On some Linux distributions tmpfiles.d cleans files and directories under /tmp if they haven't been accessed for 10 days. This can cause problems for ML as ML is currently the only component that uses the temp directory more than a few seconds after startup. If you didn't open an ML job for 10 days and then tried to open one then the temp directory would have been deleted. This commit prevents the problem occurring in the case of Elasticsearch being managed by systemd, as systemd private temp directories are not subject to periodic cleanup (by default). Additionally there are now some docs to warn people about the risk and suggest a manual mitigation for .tar.gz users. --- .../src/common/systemd/elasticsearch.service | 1 + .../setup/important-settings.asciidoc | 1 + .../important-settings/es-tmpdir.asciidoc | 23 +++++++++++++++++++ .../resources/packaging/tests/60_systemd.bats | 5 +++- .../tests/module_and_plugin_test_cases.bash | 9 +++++--- .../test/resources/packaging/utils/utils.bash | 5 +++- 6 files changed, 39 insertions(+), 5 deletions(-) create mode 100644 docs/reference/setup/important-settings/es-tmpdir.asciidoc diff --git a/distribution/packages/src/common/systemd/elasticsearch.service b/distribution/packages/src/common/systemd/elasticsearch.service index 409f04f76d0..a4d67d8830a 100644 --- a/distribution/packages/src/common/systemd/elasticsearch.service +++ b/distribution/packages/src/common/systemd/elasticsearch.service @@ -6,6 +6,7 @@ After=network-online.target [Service] RuntimeDirectory=elasticsearch +PrivateTmp=true Environment=ES_HOME=/usr/share/elasticsearch Environment=ES_PATH_CONF=${path.conf} Environment=PID_DIR=/var/run/elasticsearch diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index b9b99b70803..6968014db4f 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -14,6 +14,7 @@ The following settings *must* be considered before going to production: * <> * <> * <> +* <> include::important-settings/path-settings.asciidoc[] diff --git a/docs/reference/setup/important-settings/es-tmpdir.asciidoc b/docs/reference/setup/important-settings/es-tmpdir.asciidoc new file mode 100644 index 00000000000..20959d969b8 --- /dev/null +++ b/docs/reference/setup/important-settings/es-tmpdir.asciidoc @@ -0,0 +1,23 @@ +[[es-tmpdir]] +=== Temp directory + +By default, Elasticsearch uses a private temporary directory that the startup +script creates immediately below the system temporary directory. + +On some Linux distributions a system utility will clean files and directories +from `/tmp` if they have not been recently accessed. This can lead to the +private temporary directory being removed while Elasticsearch is running if +features that require the temporary directory are not used for a long time. +This causes problems if a feature that requires the temporary directory is +subsequently used. + +If you install Elasticsearch using the `.deb` or `.rpm` packages and run it +under `systemd` then the private temporary directory that Elasticsearch uses +is excluded from periodic cleanup. + +However, if you intend to run the `.tar.gz` distribution on Linux for an +extended period then you should consider creating a dedicated temporary +directory for Elasticsearch that is not under a path that will have old files +and directories cleaned from it. This directory should have permissions set +so that only the user that Elasticsearch runs as can access it. Then set the +`$ES_TMPDIR` environment variable to point to it before starting Elasticsearch. diff --git a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats index cb9e6658d3d..a7628d08bba 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats +++ b/qa/vagrant/src/test/resources/packaging/tests/60_systemd.bats @@ -189,7 +189,10 @@ setup() { @test "[SYSTEMD] start Elasticsearch with custom JVM options" { assert_file_exist $ESENVFILE - local temp=`mktemp -d` + # The custom config directory is not under /tmp or /var/tmp because + # systemd's private temp directory functionaly means different + # processes can have different views of what's in these directories + local temp=`mktemp -p /etc -d` cp "$ESCONFIG"/elasticsearch.yml "$temp" cp "$ESCONFIG"/log4j2.properties "$temp" touch "$temp/jvm.options" diff --git a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash index e258c4db5e6..8fd6bd9ad3f 100644 --- a/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/tests/module_and_plugin_test_cases.bash @@ -92,11 +92,14 @@ fi @test "[$GROUP] install a sample plugin with a symlinked plugins path" { # Clean up after the last time this test was run - rm -rf /tmp/plugins.* - rm -rf /tmp/old_plugins.* + rm -rf /var/plugins.* + rm -rf /var/old_plugins.* rm -rf "$ESPLUGINS" - local es_plugins=$(mktemp -d -t 'plugins.XXXX') + # The custom plugins directory is not under /tmp or /var/tmp because + # systemd's private temp directory functionaly means different + # processes can have different views of what's in these directories + local es_plugins=$(mktemp -p /var -d -t 'plugins.XXXX') chown -R elasticsearch:elasticsearch "$es_plugins" ln -s "$es_plugins" "$ESPLUGINS" diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash index c07037a5f27..cb71e9e6ec1 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash @@ -555,7 +555,10 @@ run_elasticsearch_tests() { # Move the config directory to another directory and properly chown it. move_config() { local oldConfig="$ESCONFIG" - export ESCONFIG="${1:-$(mktemp -d -t 'config.XXXX')}" + # The custom config directory is not under /tmp or /var/tmp because + # systemd's private temp directory functionaly means different + # processes can have different views of what's in these directories + export ESCONFIG="${1:-$(mktemp -p /etc -d -t 'config.XXXX')}" echo "Moving configuration directory from $oldConfig to $ESCONFIG" # Move configuration files to the new configuration directory From 733f84c49bf0161164fbe64b087046c4080b4438 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Tue, 7 Aug 2018 17:49:48 +0100 Subject: [PATCH 06/20] [DOCS] Add missing docs include --- docs/reference/setup/important-settings.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/setup/important-settings.asciidoc b/docs/reference/setup/important-settings.asciidoc index 6968014db4f..8a9b59480a0 100644 --- a/docs/reference/setup/important-settings.asciidoc +++ b/docs/reference/setup/important-settings.asciidoc @@ -32,4 +32,6 @@ include::important-settings/heap-dump-path.asciidoc[] include::important-settings/gc-logging.asciidoc[] +include::important-settings/es-tmpdir.asciidoc[] + include::important-settings/error-file.asciidoc[] From 2e65bac5ddb5d5fe5714d9288ef31deee8a663f7 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 7 Aug 2018 18:54:42 +0200 Subject: [PATCH 07/20] [Rollup] Remove builders from RollupJobConfig (#32669) --- .../rollup/action/GetRollupJobsAction.java | 2 +- .../rollup/action/PutRollupJobAction.java | 5 +- .../xpack/core/rollup/job/MetricConfig.java | 2 +- .../xpack/core/rollup/job/RollupJob.java | 2 +- .../core/rollup/job/RollupJobConfig.java | 380 ++++++------------ .../xpack/core/rollup/ConfigTestHelpers.java | 60 ++- .../job/JobWrapperSerializingTests.java | 2 +- .../core/rollup/job/RollupJobConfigTests.java | 164 ++++++-- .../xpack/core/rollup/job/RollupJobTests.java | 6 +- .../xpack/rollup/action/RollupIndexCaps.java | 4 +- .../rollup/rest/RestPutRollupJobAction.java | 2 +- .../rollup/RollupJobIdentifierUtilTests.java | 176 ++++---- .../rollup/RollupRequestTranslationTests.java | 18 - .../action/GetJobsActionRequestTests.java | 8 +- .../GetRollupCapsActionRequestTests.java | 8 +- .../GetRollupIndexCapsActionRequestTests.java | 15 +- .../action/PutJobActionRequestTests.java | 9 +- .../action/PutJobStateMachineTests.java | 30 +- .../rollup/action/RollupIndexCapsTests.java | 8 +- .../rollup/action/SearchActionTests.java | 147 +++---- .../action/TransportTaskHelperTests.java | 8 +- .../xpack/rollup/config/ConfigTests.java | 115 +----- .../job/RollupIndexerIndexingTests.java | 11 +- .../rollup/job/RollupIndexerStateTests.java | 36 +- .../xpack/rollup/job/RollupJobTaskTests.java | 36 +- 25 files changed, 527 insertions(+), 727 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java index d5a5e7a07fa..50f79315085 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/GetRollupJobsAction.java @@ -212,7 +212,7 @@ public class GetRollupJobsAction extends Action { (RollupJobStats) a[1], (RollupJobStatus)a[2])); static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.PARSER.apply(p,c).build(), CONFIG); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.fromXContent(p, null), CONFIG); PARSER.declareObject(ConstructingObjectParser.constructorArg(), RollupJobStats.PARSER::apply, STATS); PARSER.declareObject(ConstructingObjectParser.constructorArg(), RollupJobStatus.PARSER::apply, STATUS); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java index 9c3767d4188..962f4cceb6a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/action/PutRollupJobAction.java @@ -52,9 +52,8 @@ public class PutRollupJobAction extends Action { } - public static Request parseRequest(String id, XContentParser parser) { - RollupJobConfig.Builder config = RollupJobConfig.Builder.fromXContent(id, parser); - return new Request(config.build()); + public static Request fromXContent(final XContentParser parser, final String id) throws IOException { + return new Request(RollupJobConfig.fromXContent(parser, id)); } public RollupJobConfig getConfig() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java index 1843db5b3c3..cc673c4ed0d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java @@ -60,7 +60,7 @@ public class MetricConfig implements Writeable, ToXContentObject { private static final ParseField AVG = new ParseField("avg"); private static final ParseField VALUE_COUNT = new ParseField("value_count"); - private static final String NAME = "metrics"; + static final String NAME = "metrics"; private static final String FIELD = "field"; private static final String METRICS = "metrics"; private static final ConstructingObjectParser PARSER; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java index 7afcdb71b11..94306966a34 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJob.java @@ -41,7 +41,7 @@ public class RollupJob extends AbstractDiffable implements XPackPlugi = new ConstructingObjectParser<>(NAME, a -> new RollupJob((RollupJobConfig) a[0], (Map) a[1])); static { - PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.PARSER.apply(p,c).build(), CONFIG); + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> RollupJobConfig.fromXContent(p, null), CONFIG); PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> p.mapStrings(), HEADERS); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java index b876aa251cc..27461c62b67 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java @@ -7,20 +7,19 @@ package org.elasticsearch.xpack.core.rollup.job; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.rollup.RollupField; import org.elasticsearch.xpack.core.scheduler.Cron; import java.io.IOException; @@ -30,61 +29,112 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; /** * This class holds the configuration details of a rollup job, such as the groupings, metrics, what * index to rollup and where to roll them to. */ public class RollupJobConfig implements NamedWriteable, ToXContentObject { + private static final String NAME = "xpack/rollup/jobconfig"; + private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(20); + private static final String ID = "id"; + private static final String TIMEOUT = "timeout"; + private static final String CRON = "cron"; + private static final String PAGE_SIZE = "page_size"; + private static final String INDEX_PATTERN = "index_pattern"; + private static final String ROLLUP_INDEX = "rollup_index"; - public static final ParseField TIMEOUT = new ParseField("timeout"); - public static final ParseField CURRENT = new ParseField("current"); - public static final ParseField CRON = new ParseField("cron"); - public static final ParseField PAGE_SIZE = new ParseField("page_size"); - - private static final ParseField INDEX_PATTERN = new ParseField("index_pattern"); - private static final ParseField ROLLUP_INDEX = new ParseField("rollup_index"); - private static final ParseField GROUPS = new ParseField("groups"); - private static final ParseField METRICS = new ParseField("metrics"); - - private String id; - private String indexPattern; - private String rollupIndex; - private GroupConfig groupConfig; - private List metricsConfig = Collections.emptyList(); - private TimeValue timeout = TimeValue.timeValueSeconds(20); - private String cron; - private int pageSize; - - public static final ObjectParser PARSER = new ObjectParser<>(NAME, false, RollupJobConfig.Builder::new); + private final String id; + private final String indexPattern; + private final String rollupIndex; + private final GroupConfig groupConfig; + private final List metricsConfig; + private final TimeValue timeout; + private final String cron; + private final int pageSize; + private static final ConstructingObjectParser PARSER; static { - PARSER.declareString(RollupJobConfig.Builder::setId, RollupField.ID); - PARSER.declareObject(RollupJobConfig.Builder::setGroupConfig, (p, c) -> GroupConfig.fromXContent(p), GROUPS); - PARSER.declareObjectArray(RollupJobConfig.Builder::setMetricsConfig, (p, c) -> MetricConfig.fromXContent(p), METRICS); - PARSER.declareString((params, val) -> - params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); - PARSER.declareString(RollupJobConfig.Builder::setIndexPattern, INDEX_PATTERN); - PARSER.declareString(RollupJobConfig.Builder::setRollupIndex, ROLLUP_INDEX); - PARSER.declareString(RollupJobConfig.Builder::setCron, CRON); - PARSER.declareInt(RollupJobConfig.Builder::setPageSize, PAGE_SIZE); + PARSER = new ConstructingObjectParser<>(NAME, false, (args, optionalId) -> { + String id = args[0] != null ? (String) args[0] : optionalId; + String indexPattern = (String) args[1]; + String rollupIndex = (String) args[2]; + GroupConfig groupConfig = (GroupConfig) args[3]; + @SuppressWarnings("unchecked") + List metricsConfig = (List) args[4]; + TimeValue timeout = (TimeValue) args[5]; + String cron = (String) args[6]; + int pageSize = (int) args[7]; + return new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groupConfig, metricsConfig, timeout); + }); + PARSER.declareString(optionalConstructorArg(), new ParseField(ID)); + PARSER.declareString(constructorArg(), new ParseField(INDEX_PATTERN)); + PARSER.declareString(constructorArg(), new ParseField(ROLLUP_INDEX)); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> GroupConfig.fromXContent(p), new ParseField(GroupConfig.NAME)); + PARSER.declareObjectArray(optionalConstructorArg(), (p, c) -> MetricConfig.fromXContent(p), new ParseField(MetricConfig.NAME)); + PARSER.declareField(optionalConstructorArg(), (p, c) -> TimeValue.parseTimeValue(p.textOrNull(), TIMEOUT), + new ParseField(TIMEOUT), ObjectParser.ValueType.STRING_OR_NULL); + PARSER.declareString(constructorArg(), new ParseField(CRON)); + PARSER.declareInt(constructorArg(), new ParseField(PAGE_SIZE)); } - RollupJobConfig(String id, String indexPattern, String rollupIndex, String cron, int pageSize, GroupConfig groupConfig, - List metricsConfig, TimeValue timeout) { + public RollupJobConfig(final String id, + final String indexPattern, + final String rollupIndex, + final String cron, + final int pageSize, + final GroupConfig groupConfig, + final List metricsConfig, + final @Nullable TimeValue timeout) { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("Id must be a non-null, non-empty string"); + } + if (indexPattern == null || indexPattern.isEmpty()) { + throw new IllegalArgumentException("Index pattern must be a non-null, non-empty string"); + } + if (Regex.isMatchAllPattern(indexPattern)) { + throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index"); + } + if (Regex.isSimpleMatchPattern(indexPattern)) { + if (Regex.simpleMatch(indexPattern, rollupIndex)) { + throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed"); + } + } + if (indexPattern.equals(rollupIndex)) { + throw new IllegalArgumentException("Rollup index may not be the same as the index pattern"); + } + if (rollupIndex == null || rollupIndex.isEmpty()) { + throw new IllegalArgumentException("Rollup index must be a non-null, non-empty string"); + } + if (cron == null || cron.isEmpty()) { + throw new IllegalArgumentException("Cron schedule must be a non-null, non-empty string"); + } + if (pageSize <= 0) { + throw new IllegalArgumentException("Page size is mandatory and must be a positive long"); + } + // Cron doesn't have a parse helper method to see if the cron is valid, + // so just construct a temporary cron object and if the cron is bad, it'll + // throw an exception + Cron testCron = new Cron(cron); + if (groupConfig == null && (metricsConfig == null || metricsConfig.isEmpty())) { + throw new IllegalArgumentException("At least one grouping or metric must be configured"); + } + this.id = id; this.indexPattern = indexPattern; this.rollupIndex = rollupIndex; this.groupConfig = groupConfig; - this.metricsConfig = metricsConfig; - this.timeout = timeout; + this.metricsConfig = metricsConfig != null ? metricsConfig : Collections.emptyList(); + this.timeout = timeout != null ? timeout : DEFAULT_TIMEOUT; this.cron = cron; this.pageSize = pageSize; } - public RollupJobConfig(StreamInput in) throws IOException { + public RollupJobConfig(final StreamInput in) throws IOException { id = in.readString(); indexPattern = in.readString(); rollupIndex = in.readString(); @@ -95,8 +145,6 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { pageSize = in.readInt(); } - public RollupJobConfig() {} - public String getId() { return id; } @@ -135,13 +183,20 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { } public Set getAllFields() { - Set fields = new HashSet<>(groupConfig.getAllFields()); - fields.addAll(metricsConfig.stream().map(MetricConfig::getField).collect(Collectors.toSet())); - return fields; + final Set fields = new HashSet<>(); + if (groupConfig != null) { + fields.addAll(groupConfig.getAllFields()); + } + if (metricsConfig != null) { + for (MetricConfig metric : metricsConfig) { + fields.add(metric.getField()); + } + } + return Collections.unmodifiableSet(fields); } - public void validateMappings(Map> fieldCapsResponse, - ActionRequestValidationException validationException) { + public void validateMappings(final Map> fieldCapsResponse, + final ActionRequestValidationException validationException) { groupConfig.validateMappings(fieldCapsResponse, validationException); for (MetricConfig m : metricsConfig) { m.validateMappings(fieldCapsResponse, validationException); @@ -149,32 +204,34 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { } @Override - public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); - builder.field(RollupField.ID.getPreferredName(), id); - builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); - builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex); - builder.field(CRON.getPreferredName(), cron); - if (groupConfig != null) { - builder.field(GROUPS.getPreferredName(), groupConfig); - } - if (metricsConfig != null) { - builder.startArray(METRICS.getPreferredName()); - for (MetricConfig metric : metricsConfig) { - metric.toXContent(builder, params); + { + builder.field(ID, id); + builder.field(INDEX_PATTERN, indexPattern); + builder.field(ROLLUP_INDEX, rollupIndex); + builder.field(CRON, cron); + if (groupConfig != null) { + builder.field(GroupConfig.NAME, groupConfig); } - builder.endArray(); + if (metricsConfig != null) { + builder.startArray(MetricConfig.NAME); + for (MetricConfig metric : metricsConfig) { + metric.toXContent(builder, params); + } + builder.endArray(); + } + if (timeout != null) { + builder.field(TIMEOUT, timeout); + } + builder.field(PAGE_SIZE, pageSize); } - if (timeout != null) { - builder.field(TIMEOUT.getPreferredName(), timeout); - } - builder.field(PAGE_SIZE.getPreferredName(), pageSize); builder.endObject(); return builder; } @Override - public void writeTo(StreamOutput out) throws IOException { + public void writeTo(final StreamOutput out) throws IOException { out.writeString(id); out.writeString(indexPattern); out.writeString(rollupIndex); @@ -190,13 +247,11 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { if (this == other) { return true; } - if (other == null || getClass() != other.getClass()) { return false; } - RollupJobConfig that = (RollupJobConfig) other; - + final RollupJobConfig that = (RollupJobConfig) other; return Objects.equals(this.id, that.id) && Objects.equals(this.indexPattern, that.indexPattern) && Objects.equals(this.rollupIndex, that.rollupIndex) @@ -209,8 +264,7 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { @Override public int hashCode() { - return Objects.hash(id, indexPattern, rollupIndex, cron, groupConfig, - metricsConfig, timeout, pageSize); + return Objects.hash(id, indexPattern, rollupIndex, cron, groupConfig, metricsConfig, timeout, pageSize); } @Override @@ -225,193 +279,7 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { return toString(); } - public static class Builder implements Writeable, ToXContentObject { - private String id; - private String indexPattern; - private String rollupIndex; - private GroupConfig groupConfig; - private List metricsConfig = Collections.emptyList(); - private TimeValue timeout = TimeValue.timeValueSeconds(20); - private String cron; - private int pageSize = 0; - - public Builder(RollupJobConfig job) { - this.id = job.getId(); - this.indexPattern = job.getIndexPattern(); - this.rollupIndex = job.getRollupIndex(); - this.groupConfig = job.getGroupConfig(); - this.metricsConfig = job.getMetricsConfig(); - this.timeout = job.getTimeout(); - this.cron = job.getCron(); - this.pageSize = job.getPageSize(); - } - - public static RollupJobConfig.Builder fromXContent(String id, XContentParser parser) { - RollupJobConfig.Builder config = RollupJobConfig.PARSER.apply(parser, null); - if (id != null) { - config.setId(id); - } - return config; - } - - public Builder() {} - - public String getId() { - return id; - } - - public RollupJobConfig.Builder setId(String id) { - this.id = id; - return this; - } - - public String getIndexPattern() { - return indexPattern; - } - - public RollupJobConfig.Builder setIndexPattern(String indexPattern) { - this.indexPattern = indexPattern; - return this; - } - - public String getRollupIndex() { - return rollupIndex; - } - - public RollupJobConfig.Builder setRollupIndex(String rollupIndex) { - this.rollupIndex = rollupIndex; - return this; - } - - public GroupConfig getGroupConfig() { - return groupConfig; - } - - public RollupJobConfig.Builder setGroupConfig(GroupConfig groupConfig) { - this.groupConfig = groupConfig; - return this; - } - - public List getMetricsConfig() { - return metricsConfig; - } - - public RollupJobConfig.Builder setMetricsConfig(List metricsConfig) { - this.metricsConfig = metricsConfig; - return this; - } - - public TimeValue getTimeout() { - return timeout; - } - - public RollupJobConfig.Builder setTimeout(TimeValue timeout) { - this.timeout = timeout; - return this; - } - - public String getCron() { - return cron; - } - - public RollupJobConfig.Builder setCron(String cron) { - this.cron = cron; - return this; - } - - public int getPageSize() { - return pageSize; - } - - public RollupJobConfig.Builder setPageSize(int pageSize) { - this.pageSize = pageSize; - return this; - } - - public RollupJobConfig build() { - if (id == null || id.isEmpty()) { - throw new IllegalArgumentException("An ID is mandatory."); - } - if (indexPattern == null || indexPattern.isEmpty()) { - throw new IllegalArgumentException("An index pattern is mandatory."); - } - if (Regex.isMatchAllPattern(indexPattern)) { - throw new IllegalArgumentException("Index pattern must not match all indices (as it would match it's own rollup index"); - } - if (Regex.isSimpleMatchPattern(indexPattern)) { - if (Regex.simpleMatch(indexPattern, rollupIndex)) { - throw new IllegalArgumentException("Index pattern would match rollup index name which is not allowed."); - } - } - if (indexPattern.equals(rollupIndex)) { - throw new IllegalArgumentException("Rollup index may not be the same as the index pattern."); - } - if (rollupIndex == null || rollupIndex.isEmpty()) { - throw new IllegalArgumentException("A rollup index name is mandatory."); - } - if (cron == null || cron.isEmpty()) { - throw new IllegalArgumentException("A cron schedule is mandatory."); - } - if (pageSize <= 0) { - throw new IllegalArgumentException("Parameter [" + PAGE_SIZE.getPreferredName() - + "] is mandatory and must be a positive long."); - } - // Cron doesn't have a parse helper method to see if the cron is valid, - // so just construct a temporary cron object and if the cron is bad, it'll - // throw an exception - Cron testCron = new Cron(cron); - if (groupConfig == null && (metricsConfig == null || metricsConfig.isEmpty())) { - throw new IllegalArgumentException("At least one grouping or metric must be configured."); - } - return new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groupConfig, - metricsConfig, timeout); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - if (id != null) { - builder.field(RollupField.ID.getPreferredName(), id); - } - if (indexPattern != null) { - builder.field(INDEX_PATTERN.getPreferredName(), indexPattern); - } - if (indexPattern != null) { - builder.field(ROLLUP_INDEX.getPreferredName(), rollupIndex); - } - if (cron != null) { - builder.field(CRON.getPreferredName(), cron); - } - if (groupConfig != null) { - builder.field(GROUPS.getPreferredName(), groupConfig); - } - if (metricsConfig != null) { - builder.startArray(METRICS.getPreferredName()); - for (MetricConfig config : metricsConfig) { - builder.startObject(); - config.toXContent(builder, params); - builder.endObject(); - } - builder.endArray(); - } - if (timeout != null) { - builder.field(TIMEOUT.getPreferredName(), timeout); - } - builder.field(PAGE_SIZE.getPreferredName(), pageSize); - builder.endObject(); - return builder; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(id); - out.writeOptionalString(indexPattern); - out.writeOptionalString(rollupIndex); - out.writeOptionalString(cron); - out.writeOptionalWriteable(groupConfig); - out.writeList(metricsConfig); - out.writeTimeValue(timeout); - out.writeInt(pageSize); - } + public static RollupJobConfig fromXContent(final XContentParser parser, @Nullable final String optionalJobId) throws IOException { + return PARSER.parse(parser, optionalJobId); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java index 6713fc75032..d892eb550a1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/ConfigTestHelpers.java @@ -17,32 +17,47 @@ import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Random; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.carrotsearch.randomizedtesting.generators.RandomNumbers.randomIntBetween; +import static com.carrotsearch.randomizedtesting.generators.RandomPicks.randomFrom; import static com.carrotsearch.randomizedtesting.generators.RandomStrings.randomAsciiAlphanumOfLengthBetween; import static org.elasticsearch.test.ESTestCase.randomDateTimeZone; public class ConfigTestHelpers { - public static RollupJobConfig.Builder getRollupJob(String jobId) { - RollupJobConfig.Builder builder = new RollupJobConfig.Builder(); - builder.setId(jobId); - builder.setCron(getCronString()); - builder.setTimeout(new TimeValue(ESTestCase.randomIntBetween(1,100))); - String indexPattern = ESTestCase.randomAlphaOfLengthBetween(1,10); - builder.setIndexPattern(indexPattern); - builder.setRollupIndex("rollup_" + indexPattern); // to ensure the index pattern != rollup index - builder.setGroupConfig(ConfigTestHelpers.randomGroupConfig(ESTestCase.random())); - builder.setPageSize(ESTestCase.randomIntBetween(1,10)); - if (ESTestCase.randomBoolean()) { - builder.setMetricsConfig(randomMetricsConfigs(ESTestCase.random())); - } - return builder; + private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"}; + + private ConfigTestHelpers() { + } + + public static RollupJobConfig randomRollupJobConfig(final Random random) { + return randomRollupJobConfig(random, randomAsciiAlphanumOfLengthBetween(random, 5, 20)); + } + public static RollupJobConfig randomRollupJobConfig(final Random random, final String id) { + return randomRollupJobConfig(random, id, randomAsciiAlphanumOfLengthBetween(random, 5, 20)); + } + + public static RollupJobConfig randomRollupJobConfig(final Random random, final String id, final String indexPattern) { + return randomRollupJobConfig(random, id, indexPattern, "rollup_" + indexPattern); + } + + public static RollupJobConfig randomRollupJobConfig(final Random random, + final String id, + final String indexPattern, + final String rollupIndex) { + final String cron = randomCron(); + final int pageSize = randomIntBetween(random, 1, 10); + final TimeValue timeout = random.nextBoolean() ? null : randomTimeout(random); + final GroupConfig groups = randomGroupConfig(random); + final List metrics = random.nextBoolean() ? null : randomMetricsConfigs(random); + return new RollupJobConfig(id, indexPattern, rollupIndex, cron, pageSize, groups, metrics, timeout); } public static GroupConfig randomGroupConfig(final Random random) { @@ -52,11 +67,6 @@ public class ConfigTestHelpers { return new GroupConfig(dateHistogram, histogram, terms); } - private static final String[] TIME_SUFFIXES = new String[]{"d", "h", "ms", "s", "m"}; - public static String randomPositiveTimeValue() { - return ESTestCase.randomIntBetween(1, 1000) + ESTestCase.randomFrom(TIME_SUFFIXES); - } - public static DateHistogramGroupConfig randomDateHistogramGroupConfig(final Random random) { final String field = randomField(random); final DateHistogramInterval interval = randomInterval(); @@ -71,7 +81,7 @@ public class ConfigTestHelpers { .collect(Collectors.toList()); } - public static String getCronString() { + public static String randomCron() { return (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 59))) + //second " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 59))) + //minute " " + (ESTestCase.randomBoolean() ? "*" : String.valueOf(ESTestCase.randomIntBetween(0, 23))) + //hour @@ -135,6 +145,10 @@ public class ConfigTestHelpers { return randomAsciiAlphanumOfLengthBetween(random, 5, 10); } + private static String randomPositiveTimeValue() { + return ESTestCase.randomIntBetween(1, 1000) + ESTestCase.randomFrom(TIME_SUFFIXES); + } + public static DateHistogramInterval randomInterval() { return new DateHistogramInterval(randomPositiveTimeValue()); } @@ -142,4 +156,10 @@ public class ConfigTestHelpers { private static long randomInterval(final Random random) { return RandomNumbers.randomLongBetween(random, 1L, Long.MAX_VALUE); } + + public static TimeValue randomTimeout(final Random random) { + return new TimeValue(randomIntBetween(random, 0, 60), + randomFrom(random, Arrays.asList(TimeUnit.MILLISECONDS, TimeUnit.SECONDS, TimeUnit.MINUTES))); + } + } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java index fa9767b51a3..a0df63bc38d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/JobWrapperSerializingTests.java @@ -39,7 +39,7 @@ public class JobWrapperSerializingTests extends AbstractSerializingTestCase { + private String jobId; + + @Before + public void setUpOptionalId() { + jobId = randomAlphaOfLengthBetween(1, 10); + } + @Override protected RollupJobConfig createTestInstance() { - return ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1,10)).build(); + return randomRollupJobConfig(random(), jobId); } @Override @@ -27,43 +37,139 @@ public class RollupJobConfigTests extends AbstractSerializingTestCase + new RollupJobConfig(sample.getId(), null, sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Index pattern must be a non-null, non-empty string")); + + e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), "", sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Index pattern must be a non-null, non-empty string")); } public void testEmptyCron() { - RollupJobConfig.Builder builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10)); - builder.setCron(null); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); - assertThat(e.getMessage(), equalTo("A cron schedule is mandatory.")); + final RollupJobConfig sample = randomRollupJobConfig(random()); - builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10)); - builder.setCron(""); - e = expectThrows(IllegalArgumentException.class, builder::build); - assertThat(e.getMessage(), equalTo("A cron schedule is mandatory.")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), null, sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Cron schedule must be a non-null, non-empty string")); + + e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), "", sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Cron schedule must be a non-null, non-empty string")); } public void testEmptyID() { - RollupJobConfig.Builder builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10)); - builder.setId(null); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); - assertThat(e.getMessage(), equalTo("An ID is mandatory.")); + final RollupJobConfig sample = randomRollupJobConfig(random()); - builder = ConfigTestHelpers.getRollupJob(randomAlphaOfLengthBetween(1, 10)); - builder.setId(""); - e = expectThrows(IllegalArgumentException.class, builder::build); - assertThat(e.getMessage(), equalTo("An ID is mandatory.")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(null, sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Id must be a non-null, non-empty string")); + + e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig("", sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Id must be a non-null, non-empty string")); + } + + public void testBadCron() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), "0 * * *", sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("invalid cron expression [0 * * *]")); + } + + public void testMatchAllIndexPattern() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), "*", sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index")); + } + + public void testMatchOwnRollupPatternPrefix() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), "foo-*", "foo-rollup", sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed")); + } + + public void testMatchOwnRollupPatternSuffix() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), "*-rollup", "foo-rollup", sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed")); + } + + public void testIndexPatternIdenticalToRollup() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), "foo", "foo", sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern")); + } + + public void testEmptyRollupIndex() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), "", sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Rollup index must be a non-null, non-empty string")); + + e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), null, sample.getCron(), sample.getPageSize(), + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Rollup index must be a non-null, non-empty string")); + } + + public void testBadSize() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), -1, + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Page size is mandatory and must be a positive long")); + + e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), 0, + sample.getGroupConfig(), sample.getMetricsConfig(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("Page size is mandatory and must be a positive long")); + } + + public void testEmptyGroupAndMetrics() { + final RollupJobConfig sample = randomRollupJobConfig(random()); + + Exception e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + null, null, sample.getTimeout())); + assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured")); + + e = expectThrows(IllegalArgumentException.class, () -> + new RollupJobConfig(sample.getId(), sample.getIndexPattern(), sample.getRollupIndex(), sample.getCron(), sample.getPageSize(), + null, emptyList(), sample.getTimeout())); + assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured")); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java index 915cfc2fe35..3d2367f12bf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/RollupJobTests.java @@ -37,7 +37,7 @@ public class RollupJobTests extends AbstractDiffableSerializationTestCase { @Override protected Writeable createTestInstance() { if (randomBoolean()) { - return new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), null); + return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), null); } Map headers = Collections.emptyMap(); @@ -45,7 +45,7 @@ public class RollupJobTests extends AbstractDiffableSerializationTestCase { headers = new HashMap<>(1); headers.put("foo", "bar"); } - return new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), headers); + return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers); } @Override @@ -60,7 +60,7 @@ public class RollupJobTests extends AbstractDiffableSerializationTestCase { return new RollupJob(other.getConfig(), null); } } else { - return new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), other.getHeaders()); + return new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), other.getHeaders()); } } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java index 22bf9ff06c2..88c29865747 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupIndexCaps.java @@ -75,7 +75,7 @@ public class RollupIndexCaps implements Writeable, ToXContentFragment { // "job-1" while (parser.nextToken().equals(XContentParser.Token.END_OBJECT) == false) { - jobs.add(RollupJobConfig.PARSER.apply(parser, aVoid).build()); + jobs.add(RollupJobConfig.fromXContent(parser, null)); } } } @@ -167,4 +167,4 @@ public class RollupIndexCaps implements Writeable, ToXContentFragment { public int hashCode() { return Objects.hash(rollupIndexName, jobCaps); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java index f41074e8e65..231e382827e 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestPutRollupJobAction.java @@ -32,7 +32,7 @@ public class RestPutRollupJobAction extends BaseRestHandler { String id = restRequest.param(ID.getPreferredName()); XContentParser parser = restRequest.contentParser(); - PutRollupJobAction.Request request = PutRollupJobAction.Request.parseRequest(id, parser); + PutRollupJobAction.Request request = PutRollupJobAction.Request.fromXContent(parser, id); return channel -> client.execute(PutRollupJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java index 24cb1dab0fa..3235d0c39e2 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupJobIdentifierUtilTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig; import org.elasticsearch.xpack.core.rollup.job.GroupConfig; @@ -27,18 +26,19 @@ import org.joda.time.DateTimeZone; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; +import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; public class RollupJobIdentifierUtilTests extends ESTestCase { public void testOneMatch() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = singletonSet(cap); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -49,10 +49,9 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testBiggerButCompatibleInterval() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = singletonSet(cap); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -63,10 +62,9 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testIncompatibleInterval() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = singletonSet(cap); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -78,10 +76,9 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testBadTimeZone() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"), null, "EST")); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = singletonSet(cap); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -94,11 +91,10 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testMetricOnlyAgg() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - job.setMetricsConfig(singletonList(new MetricConfig("bar", singletonList("max")))); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final List metrics = singletonList(new MetricConfig("bar", singletonList("max"))); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, metrics, null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = singletonSet(cap); MaxAggregationBuilder max = new MaxAggregationBuilder("the_max").field("bar"); @@ -108,10 +104,9 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testOneOfTwoMatchingCaps() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = singletonSet(cap); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -124,18 +119,16 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testTwoJobsSameRollupIndex() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = new HashSet<>(2); caps.add(cap); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2"); final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job2.setGroupConfig(group); - job2.setRollupIndex(job.getRollupIndex()); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, group2, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); caps.add(cap2); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -148,19 +141,16 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testTwoJobsButBothPartialMatches() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - job.setMetricsConfig(singletonList(new MetricConfig("bar", singletonList("max")))); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final List metrics = singletonList(new MetricConfig("bar", singletonList("max"))); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, metrics, null); + RollupJobCaps cap = new RollupJobCaps(job); Set caps = new HashSet<>(2); caps.add(cap); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2"); - final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job2.setGroupConfig(group); - job.setMetricsConfig(singletonList(new MetricConfig("bar", singletonList("min")))); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + // TODO Is it what we really want to test? + final RollupJobConfig job2 = new RollupJobConfig("foo2", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); caps.add(cap2); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") @@ -174,15 +164,14 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testComparableDifferentDateIntervals() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex()); final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"))); - job2.setGroupConfig(group2); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, group2, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") .dateHistogramInterval(new DateHistogramInterval("1d")); @@ -197,15 +186,14 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testComparableDifferentDateIntervalsOnlyOneWorks() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex()); final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"))); - job2.setGroupConfig(group2); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, group2, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") .dateHistogramInterval(new DateHistogramInterval("1h")); @@ -220,16 +208,15 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testComparableNoHistoVsHisto() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex()); final HistogramGroupConfig histoConfig = new HistogramGroupConfig(100L, "bar"); final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), histoConfig, null); - job2.setGroupConfig(group2); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, group2, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") .dateHistogramInterval(new DateHistogramInterval("1h")) @@ -245,16 +232,15 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { } public void testComparableNoTermsVsTerms() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex()); final TermsGroupConfig termsConfig = new TermsGroupConfig("bar"); final GroupConfig group2 = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, termsConfig); - job2.setGroupConfig(group2); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, group2, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder("foo").field("foo") .dateHistogramInterval(new DateHistogramInterval("1h")) @@ -276,16 +262,16 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig( + final GroupConfig group = new GroupConfig( // NOTE same name but wrong type new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()), new HistogramGroupConfig(1L, "baz"), // <-- NOTE right type but wrong name null - )) - .setMetricsConfig( - Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) - .build(); + ); + final List metrics = + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg"))); + + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, metrics, null); Set caps = singletonSet(new RollupJobCaps(job)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps)); @@ -300,13 +286,13 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig( + final GroupConfig group = new GroupConfig( new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()) - )) - .setMetricsConfig( - Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) - .build(); + ); + final List metrics = + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg"))); + + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, metrics, null); Set caps = singletonSet(new RollupJobCaps(job)); Exception e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo,caps)); @@ -321,12 +307,11 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig( + final GroupConfig group = new GroupConfig( // interval in job is much higher than agg interval above new DateHistogramGroupConfig("foo", new DateHistogramInterval("100d"), null, DateTimeZone.UTC.getID()) - )) - .build(); + ); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); Set caps = singletonSet(new RollupJobCaps(job)); Exception e = expectThrows(RuntimeException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps)); @@ -341,14 +326,14 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig( + final GroupConfig group = new GroupConfig( // NOTE different field from the one in the query new DateHistogramGroupConfig("bar", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()) - )) - .setMetricsConfig( - Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) - .build(); + ); + final List metrics = + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg"))); + + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, metrics, null); Set caps = singletonSet(new RollupJobCaps(job)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps)); @@ -363,15 +348,15 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig( + final GroupConfig group = new GroupConfig( new DateHistogramGroupConfig("bar", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()), new HistogramGroupConfig(1L, "baz"), // <-- NOTE right type but wrong name null - )) - .setMetricsConfig( - Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg")))) - .build(); + ); + final List metrics = + Arrays.asList(new MetricConfig("max_field", singletonList("max")), new MetricConfig("avg_field", singletonList("avg"))); + + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, metrics, null); Set caps = singletonSet(new RollupJobCaps(job)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> RollupJobIdentifierUtils.findBestJobs(histo, caps)); @@ -386,13 +371,12 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { .subAggregation(new MaxAggregationBuilder("the_max").field("max_field")) .subAggregation(new AvgAggregationBuilder("the_avg").field("avg_field")); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig( + final GroupConfig group = new GroupConfig( new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID()), new HistogramGroupConfig(1L, "baz"), // <-- NOTE right type but wrong name null - )) - .build(); + ); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); Set caps = singletonSet(new RollupJobCaps(job)); Exception e = expectThrows(RuntimeException.class, @@ -404,10 +388,10 @@ public class RollupJobIdentifierUtilTests extends ESTestCase { public void testMissingMetric() { int i = ESTestCase.randomIntBetween(0, 3); - Set caps = singletonSet(new RollupJobCaps(ConfigTestHelpers - .getRollupJob("foo") - .setMetricsConfig(singletonList(new MetricConfig("foo", Arrays.asList("avg", "max", "min", "sum")))) - .build())); + final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); + final List metrics = singletonList(new MetricConfig("foo", Arrays.asList("avg", "max", "min", "sum"))); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, group, emptyList(), null); + Set caps = singletonSet(new RollupJobCaps(job)); String aggType; Exception e; diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java index 08663eb9bba..a618e8b4e6f 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/RollupRequestTranslationTests.java @@ -27,25 +27,18 @@ import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; -import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; -import org.elasticsearch.xpack.core.rollup.job.MetricConfig; import org.hamcrest.Matchers; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.rollup.RollupRequestTranslator.translateAggregation; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.IsInstanceOf.instanceOf; @@ -153,11 +146,6 @@ public class RollupRequestTranslationTests extends ESTestCase { } public void testUnsupportedMetric() { - Set caps = singletonSet(new RollupJobCaps(ConfigTestHelpers - .getRollupJob("foo") - .setMetricsConfig(singletonList(new MetricConfig("foo", Arrays.asList("avg", "max", "min", "sum")))) - .build())); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> translateAggregation(new StatsAggregationBuilder("test_metric") .field("foo"), Collections.emptyList(), namedWriteableRegistry)); @@ -384,10 +372,4 @@ public class RollupRequestTranslationTests extends ESTestCase { assertThat(e.getMessage(), equalTo("Unable to translate aggregation tree into Rollup. Aggregation [test_geo] is of type " + "[GeoDistanceAggregationBuilder] which is currently unsupported.")); } - - private Set singletonSet(RollupJobCaps cap) { - Set caps = new HashSet<>(); - caps.add(cap); - return caps; - } } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java index 419feb6f19c..bf73ceb8811 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetJobsActionRequestTests.java @@ -70,7 +70,7 @@ public class GetJobsActionRequestTests extends AbstractStreamableTestCase> tasks = Collections.singletonMap("foo", new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupJob.NAME, job, 1, null)); ClusterState state = ClusterState.builder(new ClusterName("_name")) @@ -83,7 +83,7 @@ public class GetJobsActionRequestTests extends AbstractStreamableTestCase> tasks = Collections.singletonMap("foo", new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupJob.NAME, job, 1, null)); ClusterState state = ClusterState.builder(new ClusterName("_name")) @@ -96,8 +96,8 @@ public class GetJobsActionRequestTests extends AbstractStreamableTestCase> tasks = new HashMap<>(2); tasks.put("foo", new PersistentTasksCustomMetaData.PersistentTask<>("foo", RollupJob.NAME, job, 1, null)); tasks.put("bar", new PersistentTasksCustomMetaData.PersistentTask<>("bar", RollupJob.NAME, job2, 1, null)); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java index e3a45dbd66b..9068bcfce36 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupCapsActionRequestTests.java @@ -89,7 +89,7 @@ public class GetRollupCapsActionRequestTests extends AbstractStreamableTestCase< public void testOneJob() throws IOException { String indexPattern = randomBoolean() ? randomAlphaOfLength(10) : randomAlphaOfLength(10) + "-*"; String jobName = randomAlphaOfLength(5); - RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build(); + RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -113,7 +113,7 @@ public class GetRollupCapsActionRequestTests extends AbstractStreamableTestCase< Map jobs = new HashMap<>(num); for (int i = 0; i < num; i++) { String jobName = randomAlphaOfLength(5); - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).build()); + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName)); } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, @@ -147,7 +147,7 @@ public class GetRollupCapsActionRequestTests extends AbstractStreamableTestCase< String jobName = randomAlphaOfLength(10); String indexName = Integer.toString(indexCounter); indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setIndexPattern(indexName).build()); + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName)); } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, @@ -179,7 +179,7 @@ public class GetRollupCapsActionRequestTests extends AbstractStreamableTestCase< Map jobs = new HashMap<>(num); for (int i = 0; i < num; i++) { String jobName = randomAlphaOfLength(5); - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setIndexPattern(indexName).build()); + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName)); } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java index 2066d664996..e9d5d6153b1 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/GetRollupIndexCapsActionRequestTests.java @@ -60,7 +60,7 @@ public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTest String jobName = randomAlphaOfLength(10); String indexName = Integer.toString(indexCounter); indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName).setRollupIndex("foo").build()); + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName, "foo")); } MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, @@ -89,10 +89,7 @@ public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTest String jobName = randomAlphaOfLength(10); String indexName = Integer.toString(indexCounter); indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) - .setIndexPattern(indexName) - .setRollupIndex("rollup_" + indexName).build()); - + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, indexName, "rollup_" + indexName)); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -120,9 +117,7 @@ public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTest String jobName = randomAlphaOfLength(10); String indexName = Integer.toString(indexCounter); indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) - .setIndexPattern("foo_" + indexName) - .setRollupIndex("rollup_" + indexName).build()); + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, "foo_" + indexName, "rollup_" + indexName)); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -151,9 +146,7 @@ public class GetRollupIndexCapsActionRequestTests extends AbstractStreamableTest String jobName = randomAlphaOfLength(10); String indexName = Integer.toString(indexCounter); indexCounter += 1; - jobs.put(jobName, ConfigTestHelpers.getRollupJob(jobName) - .setIndexPattern("foo_" + indexName) - .setRollupIndex("rollup_foo").build()); + jobs.put(jobName, ConfigTestHelpers.randomRollupJobConfig(random(), jobName, "foo_" + indexName, "rollup_foo")); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java index 254c2a8c811..848bd5f13dd 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobActionRequestTests.java @@ -12,6 +12,8 @@ import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction.Request; import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.junit.Before; +import java.io.IOException; + public class PutJobActionRequestTests extends AbstractStreamableXContentTestCase { private String jobId; @@ -23,7 +25,7 @@ public class PutJobActionRequestTests extends AbstractStreamableXContentTestCase @Override protected Request createTestInstance() { - return new Request(ConfigTestHelpers.getRollupJob(jobId).build()); + return new Request(ConfigTestHelpers.randomRollupJobConfig(random(), jobId)); } @Override @@ -37,9 +39,8 @@ public class PutJobActionRequestTests extends AbstractStreamableXContentTestCase } @Override - protected Request doParseInstance(XContentParser parser) { - return Request.parseRequest(jobId, parser); + protected Request doParseInstance(final XContentParser parser) throws IOException { + return Request.fromXContent(parser, jobId); } - } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java index d7bc2786646..d9caad5147d 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java @@ -50,7 +50,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testCreateIndexException() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -76,7 +76,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testIndexAlreadyExists() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -108,7 +108,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testIndexMetaData() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -151,7 +151,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testGetMappingFails() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -175,7 +175,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testNoMetadataInMapping() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -208,7 +208,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testNoMappingVersion() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -245,7 +245,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testJobAlreadyInMapping() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -282,12 +282,12 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testAddJobToMapping() { - RollupJobConfig unrelatedJob = ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLength(10)) - .setIndexPattern("foo").setRollupIndex("rollup_index_foo").build(); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo") - .setIndexPattern("foo") - .setRollupIndex("rollup_index_foo") - .build(), Collections.emptyMap()); + final RollupJobConfig unrelatedJob = + ConfigTestHelpers.randomRollupJobConfig(random(), ESTestCase.randomAlphaOfLength(10), "foo", "rollup_index_foo"); + + final RollupJobConfig config = + ConfigTestHelpers.randomRollupJobConfig(random(), ESTestCase.randomAlphaOfLength(10), "foo", "rollup_index_foo"); + RollupJob job = new RollupJob(config, Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); }, e -> { @@ -331,7 +331,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testTaskAlreadyExists() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random(), "foo"), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); @@ -354,7 +354,7 @@ public class PutJobStateMachineTests extends ESTestCase { @SuppressWarnings("unchecked") public void testStartTask() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob("foo").build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); ActionListener testListener = ActionListener.wrap(response -> { fail("Listener success should not have been triggered."); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java index a1e4dba0fff..78b1e1e0d2d 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/RollupIndexCapsTests.java @@ -30,8 +30,8 @@ public class RollupIndexCapsTests extends ESTestCase { public void testGetAllJobs() { List jobs = new ArrayList<>(2); - jobs.add(ConfigTestHelpers.getRollupJob("foo").build()); - jobs.add(ConfigTestHelpers.getRollupJob("bar").build()); + jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "foo")); + jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "bar")); RollupIndexCaps caps = new RollupIndexCaps(ESTestCase.randomAlphaOfLength(10), jobs); assertTrue(caps.hasCaps()); @@ -45,8 +45,8 @@ public class RollupIndexCapsTests extends ESTestCase { public void testFilterGetJobs() { List jobs = new ArrayList<>(2); - jobs.add(ConfigTestHelpers.getRollupJob("foo").setIndexPattern("foo_index_pattern").build()); - jobs.add(ConfigTestHelpers.getRollupJob("bar").build()); + jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "foo", "foo_index_pattern")); + jobs.add(ConfigTestHelpers.randomRollupJobConfig(random(), "bar")); RollupIndexCaps caps = new RollupIndexCaps(ESTestCase.randomAlphaOfLength(10), jobs); assertTrue(caps.hasCaps()); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java index 6aec4d4f438..069e23e4093 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/SearchActionTests.java @@ -16,9 +16,6 @@ import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.common.util.MockBigArrays; -import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -30,8 +27,6 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; -import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.aggregations.Aggregations; @@ -54,10 +49,12 @@ import org.elasticsearch.xpack.core.rollup.RollupField; import org.elasticsearch.xpack.core.rollup.action.RollupJobCaps; import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig; import org.elasticsearch.xpack.core.rollup.job.GroupConfig; +import org.elasticsearch.xpack.core.rollup.job.MetricConfig; import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig; import org.elasticsearch.xpack.rollup.Rollup; import org.hamcrest.core.IsEqual; +import org.joda.time.DateTimeZone; import org.junit.Before; import org.mockito.Mockito; @@ -71,6 +68,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import static java.util.Collections.emptyList; +import static java.util.Collections.singleton; import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig; import static org.elasticsearch.xpack.core.rollup.RollupField.COUNT_FIELD; import static org.hamcrest.Matchers.equalTo; @@ -82,12 +81,11 @@ public class SearchActionTests extends ESTestCase { private NamedWriteableRegistry namedWriteableRegistry; - @Override @Before public void setUp() throws Exception { super.setUp(); - IndicesModule indicesModule = new IndicesModule(Collections.emptyList()); - SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + IndicesModule indicesModule = new IndicesModule(emptyList()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); List entries = new ArrayList<>(); entries.addAll(indicesModule.getNamedWriteables()); entries.addAll(searchModule.getNamedWriteables()); @@ -121,10 +119,9 @@ public class SearchActionTests extends ESTestCase { } public void testRange() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(groupConfig); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new RangeQueryBuilder("foo").gt(1).timeZone("UTC"), caps); @@ -133,10 +130,9 @@ public class SearchActionTests extends ESTestCase { } public void testRangeNullTimeZone() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"), null, null)); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new RangeQueryBuilder("foo").gt(1), caps); @@ -145,10 +141,9 @@ public class SearchActionTests extends ESTestCase { } public void testRangeWrongTZ() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"), null, "UTC")); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); Exception e = expectThrows(IllegalArgumentException.class, @@ -158,11 +153,10 @@ public class SearchActionTests extends ESTestCase { } public void testTermQuery() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final TermsGroupConfig termsConfig = new TermsGroupConfig("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("date", new DateHistogramInterval("1h")), null, termsConfig); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final TermsGroupConfig terms = new TermsGroupConfig("foo"); + final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("boo", new DateHistogramInterval("1h")), null, terms); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new TermQueryBuilder("foo", "bar"), caps); @@ -171,16 +165,14 @@ public class SearchActionTests extends ESTestCase { } public void testTermsQuery() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final TermsGroupConfig termsConfig = new TermsGroupConfig("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("date", new DateHistogramInterval("1h")), null, termsConfig); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final TermsGroupConfig terms = new TermsGroupConfig("foo"); + final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("boo", new DateHistogramInterval("1h")), null, terms); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); QueryBuilder original = new TermsQueryBuilder("foo", Arrays.asList("bar", "baz")); - QueryBuilder rewritten = - TransportRollupSearchAction.rewriteQuery(original, caps); + QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(original, caps); assertThat(rewritten, instanceOf(TermsQueryBuilder.class)); assertNotSame(rewritten, original); assertThat(((TermsQueryBuilder)rewritten).fieldName(), equalTo("foo.terms.value")); @@ -188,10 +180,9 @@ public class SearchActionTests extends ESTestCase { } public void testCompounds() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(groupConfig); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); @@ -203,10 +194,9 @@ public class SearchActionTests extends ESTestCase { } public void testMatchAll() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(groupConfig); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); QueryBuilder rewritten = TransportRollupSearchAction.rewriteQuery(new MatchAllQueryBuilder(), caps); @@ -214,11 +204,10 @@ public class SearchActionTests extends ESTestCase { } public void testAmbiguousResolution() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final TermsGroupConfig termsConfig = new TermsGroupConfig("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, termsConfig); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final TermsGroupConfig terms = new TermsGroupConfig("foo"); + final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, terms); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); Set caps = new HashSet<>(); caps.add(cap); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, @@ -364,11 +353,10 @@ public class SearchActionTests extends ESTestCase { } public void testGood() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(groupConfig); - RollupJobCaps cap = new RollupJobCaps(job.build()); - Set caps = singletonSet(cap); + final RollupJobConfig config = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(config); + Set caps = singleton(cap); String[] normalIndices = new String[]{ESTestCase.randomAlphaOfLength(10)}; String[] rollupIndices = new String[]{ESTestCase.randomAlphaOfLength(10)}; @@ -381,7 +369,7 @@ public class SearchActionTests extends ESTestCase { source.query(getQueryBuilder(1)); source.size(0); source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo") - .dateHistogramInterval(job.getGroupConfig().getDateHistogram().getInterval())); + .dateHistogramInterval(config.getGroupConfig().getDateHistogram().getInterval())); SearchRequest request = new SearchRequest(combinedIndices, source); MultiSearchRequest msearch = TransportRollupSearchAction.createMSearchRequest(request, namedWriteableRegistry, ctx); @@ -409,10 +397,10 @@ public class SearchActionTests extends ESTestCase { source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo").dateHistogramInterval(new DateHistogramInterval("1d"))); SearchRequest request = new SearchRequest(combinedIndices, source); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo") - .setGroupConfig(new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, "UTC"))) - .build(); - Set caps = singletonSet(new RollupJobCaps(job)); + final GroupConfig groupConfig = + new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1d"), null, DateTimeZone.UTC.getID())); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + Set caps = singleton(new RollupJobCaps(job)); TransportRollupSearchAction.RollupSearchContext ctx = new TransportRollupSearchAction.RollupSearchContext(normalIndices, rollupIndices, caps); @@ -432,17 +420,15 @@ public class SearchActionTests extends ESTestCase { } public void testTwoMatchingJobs() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); - - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex()); - job2.setGroupConfig(group); + final GroupConfig groupConfig = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, null); + final RollupJobConfig job = new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); // so that the jobs aren't exactly equal - job2.setMetricsConfig(ConfigTestHelpers.randomMetricsConfigs(random())); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final List metricConfigs = ConfigTestHelpers.randomMetricsConfigs(random()); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, groupConfig, metricConfigs, null); + RollupJobCaps cap2 = new RollupJobCaps(job2); Set caps = new HashSet<>(2); caps.add(cap); @@ -479,15 +465,17 @@ public class SearchActionTests extends ESTestCase { } public void testTwoMatchingJobsOneBetter() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - final GroupConfig group = new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h"))); - job.setGroupConfig(group); - RollupJobCaps cap = new RollupJobCaps(job.build()); + final GroupConfig groupConfig = + new GroupConfig(new DateHistogramGroupConfig("foo", new DateHistogramInterval("1h")), null, null); + final RollupJobConfig job = + new RollupJobConfig("foo", "index", "rollup", "*/5 * * * * ?", 10, groupConfig, emptyList(), null); + RollupJobCaps cap = new RollupJobCaps(job); - RollupJobConfig.Builder job2 = ConfigTestHelpers.getRollupJob("foo2").setRollupIndex(job.getRollupIndex()); - final GroupConfig group2 = new GroupConfig(group.getDateHistogram(), randomHistogramGroupConfig(random()), null); - job2.setGroupConfig(group2); - RollupJobCaps cap2 = new RollupJobCaps(job2.build()); + final GroupConfig groupConfig2 = + new GroupConfig(groupConfig.getDateHistogram(), randomHistogramGroupConfig(random()), null); + final RollupJobConfig job2 = + new RollupJobConfig("foo2", "index", job.getRollupIndex(), "*/5 * * * * ?", 10, groupConfig2, emptyList(), null); + RollupJobCaps cap2 = new RollupJobCaps(job2); Set caps = new HashSet<>(2); caps.add(cap); @@ -504,7 +492,7 @@ public class SearchActionTests extends ESTestCase { source.query(getQueryBuilder(1)); source.size(0); source.aggregation(new DateHistogramAggregationBuilder("foo").field("foo") - .dateHistogramInterval(job.getGroupConfig().getDateHistogram().getInterval())); + .dateHistogramInterval(job.getGroupConfig().getDateHistogram().getInterval())); SearchRequest request = new SearchRequest(combinedIndices, source); MultiSearchRequest msearch = TransportRollupSearchAction.createMSearchRequest(request, namedWriteableRegistry, ctx); @@ -572,7 +560,7 @@ public class SearchActionTests extends ESTestCase { String[] indices = new String[]{"foo"}; String jobName = randomAlphaOfLength(5); - RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build(); + RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -616,7 +604,7 @@ public class SearchActionTests extends ESTestCase { String[] indices = new String[]{"foo"}; String jobName = randomAlphaOfLength(5); - RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build(); + RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -680,7 +668,7 @@ public class SearchActionTests extends ESTestCase { String[] indices = new String[]{"foo", "bar"}; String jobName = randomAlphaOfLength(5); - RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build(); + RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -715,7 +703,7 @@ public class SearchActionTests extends ESTestCase { String[] indices = new String[]{"foo", "bar"}; String jobName = randomAlphaOfLength(5); - RollupJobConfig job = ConfigTestHelpers.getRollupJob(jobName).build(); + RollupJobConfig job = ConfigTestHelpers.randomRollupJobConfig(random(), jobName); MappingMetaData mappingMeta = new MappingMetaData(RollupField.TYPE_NAME, Collections.singletonMap(RollupField.TYPE_NAME, @@ -745,7 +733,7 @@ public class SearchActionTests extends ESTestCase { SearchResponse protoResponse = mock(SearchResponse.class); when(protoResponse.getTook()).thenReturn(new TimeValue(100)); List protoAggTree = new ArrayList<>(1); - InternalAvg internalAvg = new InternalAvg("foo", 10, 2, DocValueFormat.RAW, Collections.emptyList(), null); + InternalAvg internalAvg = new InternalAvg("foo", 10, 2, DocValueFormat.RAW, emptyList(), null); protoAggTree.add(internalAvg); Aggregations protoMockAggs = new InternalAggregations(protoAggTree); when(protoResponse.getAggregations()).thenReturn(protoMockAggs); @@ -785,14 +773,9 @@ public class SearchActionTests extends ESTestCase { MultiSearchResponse msearchResponse = new MultiSearchResponse(new MultiSearchResponse.Item[]{unrolledResponse, rolledResponse}, 123); - - BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); - ScriptService scriptService = mock(ScriptService.class); - SearchResponse response = TransportRollupSearchAction.processResponses(separateIndices, msearchResponse, mock(InternalAggregation.ReduceContext.class)); - assertNotNull(response); Aggregations responseAggs = response.getAggregations(); assertNotNull(responseAggs); @@ -800,10 +783,4 @@ public class SearchActionTests extends ESTestCase { assertThat(avg.getValue(), IsEqual.equalTo(5.0)); } - - private Set singletonSet(RollupJobCaps cap) { - Set caps = new HashSet<>(); - caps.add(cap); - return caps; - } } diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java index b27953e4e0a..a156585b609 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportTaskHelperTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.tasks.TaskManager; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; import org.elasticsearch.xpack.rollup.job.RollupJobTask; @@ -18,6 +17,7 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; +import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomRollupJobConfig; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -25,7 +25,7 @@ import static org.mockito.Mockito.when; public class TransportTaskHelperTests extends ESTestCase { public void testProcessRequestOneMatching() { - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo").build(); + RollupJobConfig job = randomRollupJobConfig(random(), "foo"); TaskManager taskManager = mock(TaskManager.class); RollupJobTask task = mock(RollupJobTask.class); when(task.getDescription()).thenReturn("rollup_foo"); @@ -58,13 +58,13 @@ public class TransportTaskHelperTests extends ESTestCase { Map tasks = getRandomTasks(); when(taskManager.getTasks()).thenReturn(tasks); - RollupJobConfig job = ConfigTestHelpers.getRollupJob("foo").build(); + RollupJobConfig job = randomRollupJobConfig(random(), "foo"); RollupJobTask task = mock(RollupJobTask.class); when(task.getDescription()).thenReturn("rollup_foo"); when(task.getConfig()).thenReturn(job); tasks.put(1L, task); - RollupJobConfig job2 = ConfigTestHelpers.getRollupJob("foo").build(); + RollupJobConfig job2 = randomRollupJobConfig(random(), "foo"); RollupJobTask task2 = mock(RollupJobTask.class); when(task2.getDescription()).thenReturn("rollup_foo"); when(task2.getConfig()).thenReturn(job2); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java index 6c4f2cabfa9..86891eda669 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/config/ConfigTests.java @@ -7,13 +7,11 @@ package org.elasticsearch.xpack.rollup.config; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.rollup.ConfigTestHelpers; import org.elasticsearch.xpack.core.rollup.job.DateHistogramGroupConfig; import org.elasticsearch.xpack.core.rollup.job.GroupConfig; import org.elasticsearch.xpack.core.rollup.job.HistogramGroupConfig; import org.elasticsearch.xpack.core.rollup.job.MetricConfig; import org.elasticsearch.xpack.core.rollup.job.RollupJob; -import org.elasticsearch.xpack.core.rollup.job.RollupJobConfig; import org.elasticsearch.xpack.core.rollup.job.TermsGroupConfig; import org.joda.time.DateTimeZone; @@ -23,6 +21,7 @@ import java.util.Map; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomHistogramGroupConfig; +import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomRollupJobConfig; import static org.elasticsearch.xpack.core.rollup.ConfigTestHelpers.randomTermsGroupConfig; import static org.hamcrest.Matchers.equalTo; //TODO split this into dedicated unit test classes (one for each config object) @@ -55,115 +54,6 @@ public class ConfigTests extends ESTestCase { assertThat(e.getMessage(), equalTo("Date histogram must not be null")); } - public void testEmptyGroupAndMetrics() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setGroupConfig(null); - job.setMetricsConfig(null); - - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("At least one grouping or metric must be configured.")); - } - - public void testEmptyJobID() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob(null); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("An ID is mandatory.")); - - job = ConfigTestHelpers.getRollupJob(""); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("An ID is mandatory.")); - - job.setId(""); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("An ID is mandatory.")); - - job.setId(null); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("An ID is mandatory.")); - } - - public void testEmptyCron() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setCron(""); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("A cron schedule is mandatory.")); - - job.setCron(null); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("A cron schedule is mandatory.")); - } - - public void testBadCron() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setCron("0 * * *"); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("invalid cron expression [0 * * *]")); - } - - public void testEmptyIndexPattern() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setIndexPattern(""); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("An index pattern is mandatory.")); - - job.setIndexPattern(null); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("An index pattern is mandatory.")); - } - - public void testMatchAllIndexPattern() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setIndexPattern("*"); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("Index pattern must not match all indices (as it would match it's own rollup index")); - } - - public void testMatchOwnRollupPatternPrefix() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setIndexPattern("foo-*"); - job.setRollupIndex("foo-rollup"); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed.")); - } - - public void testMatchOwnRollupPatternSuffix() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setIndexPattern("*-rollup"); - job.setRollupIndex("foo-rollup"); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("Index pattern would match rollup index name which is not allowed.")); - } - - public void testIndexPatternIdenticalToRollup() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setIndexPattern("foo"); - job.setRollupIndex("foo"); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("Rollup index may not be the same as the index pattern.")); - } - - public void testEmptyRollupIndex() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setRollupIndex(""); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("A rollup index name is mandatory.")); - - job.setRollupIndex(null); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("A rollup index name is mandatory.")); - } - - public void testBadSize() { - RollupJobConfig.Builder job = ConfigTestHelpers.getRollupJob("foo"); - job.setPageSize(-1); - Exception e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("Parameter [page_size] is mandatory and must be a positive long.")); - - job.setPageSize(0); - e = expectThrows(IllegalArgumentException.class, job::build); - assertThat(e.getMessage(), equalTo("Parameter [page_size] is mandatory and must be a positive long.")); - } - public void testEmptyDateHistoField() { Exception e = expectThrows(IllegalArgumentException.class, () -> new DateHistogramGroupConfig(null, DateHistogramInterval.HOUR)); @@ -225,8 +115,7 @@ public class ConfigTests extends ESTestCase { Map headers = new HashMap<>(1); headers.put("es-security-runas-user", "foo"); headers.put("_xpack_security_authentication", "bar"); - RollupJobConfig config = ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(); - RollupJob job = new RollupJob(config, headers); + RollupJob job = new RollupJob(randomRollupJobConfig(random()), headers); String json = job.toString(); assertFalse(json.contains("authentication")); assertFalse(json.contains("security")); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 21a834f4b57..6d29ee9f9ba 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -434,15 +434,8 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { } private RollupJobConfig createJob(String rollupIndex, GroupConfig groupConfig, List metricConfigs) { - return new RollupJobConfig.Builder() - .setId(randomAlphaOfLength(10)) - .setIndexPattern(randomAlphaOfLength(10)) - .setRollupIndex(rollupIndex) - .setGroupConfig(groupConfig) - .setMetricsConfig(metricConfigs) - .setCron(ConfigTestHelpers.getCronString()) - .setPageSize(randomIntBetween(1, 100)) - .build(); + return new RollupJobConfig(randomAlphaOfLength(10), randomAlphaOfLength(10), rollupIndex, ConfigTestHelpers.randomCron(), + randomIntBetween(1, 100), groupConfig, metricConfigs, ConfigTestHelpers.randomTimeout(random())); } static Map asMap(Object... fields) { diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java index 733a784b843..955dcbc2beb 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerStateTests.java @@ -216,8 +216,7 @@ public class RollupIndexerStateTests extends ESTestCase { } public void testStarted() throws Exception { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); try { @@ -236,8 +235,7 @@ public class RollupIndexerStateTests extends ESTestCase { } public void testIndexing() throws Exception { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); try { @@ -304,8 +302,7 @@ public class RollupIndexerStateTests extends ESTestCase { public void testAbortDuringSearch() throws Exception { final AtomicBoolean aborted = new AtomicBoolean(false); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); final CountDownLatch latch = new CountDownLatch(1); @@ -349,8 +346,7 @@ public class RollupIndexerStateTests extends ESTestCase { public void testAbortAfterCompletion() throws Exception { final AtomicBoolean aborted = new AtomicBoolean(false); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); @@ -435,8 +431,7 @@ public class RollupIndexerStateTests extends ESTestCase { } public void testStopIndexing() throws Exception { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); try { @@ -458,8 +453,7 @@ public class RollupIndexerStateTests extends ESTestCase { } public void testAbortIndexing() throws Exception { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); try { @@ -486,8 +480,7 @@ public class RollupIndexerStateTests extends ESTestCase { } public void testAbortStarted() throws Exception { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); try { @@ -513,8 +506,7 @@ public class RollupIndexerStateTests extends ESTestCase { } public void testMultipleJobTriggering() throws Exception { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final ExecutorService executor = Executors.newFixedThreadPool(1); try { @@ -554,8 +546,7 @@ public class RollupIndexerStateTests extends ESTestCase { // deal with it everyhwere public void testUnknownKey() throws Exception { AtomicBoolean isFinished = new AtomicBoolean(false); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); Function searchFunction = searchRequest -> { Aggregations aggs = new Aggregations(Collections.singletonList(new CompositeAggregation() { @@ -659,8 +650,7 @@ public class RollupIndexerStateTests extends ESTestCase { public void testFailureWhileStopping() throws Exception { AtomicBoolean isFinished = new AtomicBoolean(false); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); Function searchFunction = searchRequest -> { Aggregations aggs = new Aggregations(Collections.singletonList(new CompositeAggregation() { @@ -762,8 +752,7 @@ public class RollupIndexerStateTests extends ESTestCase { public void testSearchShardFailure() throws Exception { AtomicBoolean isFinished = new AtomicBoolean(false); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); Function searchFunction = searchRequest -> { ShardSearchFailure[] failures = new ShardSearchFailure[]{new ShardSearchFailure(new RuntimeException("failed"))}; @@ -806,8 +795,7 @@ public class RollupIndexerStateTests extends ESTestCase { public void testBulkFailure() throws Exception { AtomicBoolean isFinished = new AtomicBoolean(false); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(ESTestCase.randomAlphaOfLengthBetween(1, 10)).build(), - Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); Function searchFunction = searchRequest -> { Aggregations aggs = new Aggregations(Collections.singletonList(new CompositeAggregation() { diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java index df7c12f47fa..13290f09e8e 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupJobTaskTests.java @@ -58,7 +58,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testInitialStatusStopped() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -71,7 +71,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testInitialStatusAborting() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.ABORTING, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -84,7 +84,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testInitialStatusStopping() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPING, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -97,7 +97,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testInitialStatusStarted() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STARTED, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -110,7 +110,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testInitialStatusIndexingOldID() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.INDEXING, Collections.singletonMap("foo", "bar"), false); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -124,7 +124,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testInitialStatusIndexingNewID() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.INDEXING, Collections.singletonMap("foo", "bar"), true); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -138,7 +138,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testNoInitialStatus() { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); SchedulerEngine schedulerEngine = new SchedulerEngine(Clock.systemUTC()); @@ -150,7 +150,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testStartWhenStarted() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STARTED, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -179,7 +179,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testStartWhenStopping() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); when(client.threadPool()).thenReturn(pool); @@ -258,7 +258,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testStartWhenStopped() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -296,7 +296,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testTriggerUnrelated() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, Collections.singletonMap("foo", "bar"), randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -337,7 +337,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testTrigger() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); when(client.threadPool()).thenReturn(pool); @@ -380,7 +380,7 @@ public class RollupJobTaskTests extends ESTestCase { @SuppressWarnings("unchecked") public void testTriggerWithoutHeaders() throws InterruptedException { final ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -465,7 +465,7 @@ public class RollupJobTaskTests extends ESTestCase { Map headers = new HashMap<>(1); headers.put("es-security-runas-user", "foo"); headers.put("_xpack_security_authentication", "bar"); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), headers); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -553,7 +553,7 @@ public class RollupJobTaskTests extends ESTestCase { Map headers = new HashMap<>(1); headers.put("es-security-runas-user", "foo"); headers.put("_xpack_security_authentication", "bar"); - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), headers); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), headers); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -637,7 +637,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testStopWhenStopped() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null, randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); @@ -663,7 +663,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testStopWhenStopping() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); when(client.threadPool()).thenReturn(pool); @@ -744,7 +744,7 @@ public class RollupJobTaskTests extends ESTestCase { } public void testStopWhenAborting() throws InterruptedException { - RollupJob job = new RollupJob(ConfigTestHelpers.getRollupJob(randomAlphaOfLength(5)).build(), Collections.emptyMap()); + RollupJob job = new RollupJob(ConfigTestHelpers.randomRollupJobConfig(random()), Collections.emptyMap()); RollupJobStatus status = new RollupJobStatus(IndexerState.STOPPED, null, randomBoolean()); Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); From 3d5e9114e38af11324f6117947aa270da0d1d691 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Tue, 7 Aug 2018 12:52:28 -0600 Subject: [PATCH 08/20] Reduce connections used by MockNioTransport (#32620) The MockNioTransport (similar to the MockTcpTransport) is used for integ tests. The MockTcpTransport has always only opened a single for all of its work. The MockNioTransport has awlays opened the default number of connections (13). This means that every test where two transports connect requires 26 connections. This is more than is necessary. This commit modifies the MockNioTransport to only require 3 connections. --- .../elasticsearch/transport/TcpTransport.java | 2 +- .../transport/nio/MockNioTransport.java | 32 +++++++++++++++++++ .../nio/SimpleMockNioTransportTests.java | 5 +++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java index b2294ce5893..83c8399c3f8 100644 --- a/server/src/main/java/org/elasticsearch/transport/TcpTransport.java +++ b/server/src/main/java/org/elasticsearch/transport/TcpTransport.java @@ -588,7 +588,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements * takes a {@link ConnectionProfile} that have been passed as a parameter to the public methods * and resolves it to a fully specified (i.e., no nulls) profile */ - static ConnectionProfile resolveConnectionProfile(@Nullable ConnectionProfile connectionProfile, + protected static ConnectionProfile resolveConnectionProfile(@Nullable ConnectionProfile connectionProfile, ConnectionProfile defaultConnectionProfile) { Objects.requireNonNull(defaultConnectionProfile); if (connectionProfile == null) { diff --git a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java index 3eca4818c4a..dc5305d951b 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java @@ -40,9 +40,11 @@ import org.elasticsearch.nio.NioServerSocketChannel; import org.elasticsearch.nio.NioSocketChannel; import org.elasticsearch.nio.ServerChannelContext; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ConnectionProfile; import org.elasticsearch.transport.TcpChannel; import org.elasticsearch.transport.TcpServerChannel; import org.elasticsearch.transport.TcpTransport; +import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.Transports; import java.io.IOException; @@ -51,6 +53,8 @@ import java.net.StandardSocketOptions; import java.nio.ByteBuffer; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.util.HashSet; +import java.util.Set; import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.function.Supplier; @@ -128,6 +132,34 @@ public class MockNioTransport extends TcpTransport { profileToChannelFactory.clear(); } + @Override + protected ConnectionProfile resolveConnectionProfile(ConnectionProfile connectionProfile) { + ConnectionProfile resolvedProfile = resolveConnectionProfile(connectionProfile, defaultConnectionProfile); + if (resolvedProfile.getNumConnections() <= 3) { + return resolvedProfile; + } + ConnectionProfile.Builder builder = new ConnectionProfile.Builder(); + Set allTypesWithConnection = new HashSet<>(); + Set allTypesWithoutConnection = new HashSet<>(); + for (TransportRequestOptions.Type type : TransportRequestOptions.Type.values()) { + int numConnections = resolvedProfile.getNumConnectionsPerType(type); + if (numConnections > 0) { + allTypesWithConnection.add(type); + } else { + allTypesWithoutConnection.add(type); + } + } + + // make sure we maintain at least the types that are supported by this profile even if we only use a single channel for them. + builder.addConnections(3, allTypesWithConnection.toArray(new TransportRequestOptions.Type[0])); + if (allTypesWithoutConnection.isEmpty() == false) { + builder.addConnections(0, allTypesWithoutConnection.toArray(new TransportRequestOptions.Type[0])); + } + builder.setHandshakeTimeout(resolvedProfile.getHandshakeTimeout()); + builder.setConnectTimeout(resolvedProfile.getConnectTimeout()); + return builder.build(); + } + private void exceptionCaught(NioSocketChannel channel, Exception exception) { onException((TcpChannel) channel, exception); } diff --git a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java index 108411dee5b..3a78f366bd8 100644 --- a/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java +++ b/test/framework/src/test/java/org/elasticsearch/transport/nio/SimpleMockNioTransportTests.java @@ -94,6 +94,11 @@ public class SimpleMockNioTransportTests extends AbstractSimpleTransportTestCase return transportService; } + @Override + protected int channelsPerNodeConnection() { + return 3; + } + @Override protected void closeConnectionChannel(Transport transport, Transport.Connection connection) throws IOException { TcpTransport.NodeChannels channels = (TcpTransport.NodeChannels) connection; From dcc816427ef62a60690d80ff14f4caec4b471c72 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Tue, 7 Aug 2018 15:10:09 -0400 Subject: [PATCH 09/20] Expose whether or not the global checkpoint updated (#32659) It will be useful for future efforts to know if the global checkpoint was updated. To this end, we need to expose whether or not the global checkpoint was updated when the state of the replication tracker updates. For this, we add to the tracker a callback that is invoked whenever the global checkpoint is updated. For primaries this will be invoked when the computed global checkpoint is updated based on state changes to the tracker. For replicas this will be invoked when the local knowledge of the global checkpoint is advanced from the primary. --- .../index/seqno/ReplicationTracker.java | 24 +++- .../elasticsearch/index/shard/IndexShard.java | 5 +- .../index/seqno/ReplicationTrackerTests.java | 114 +++++++++++++++--- .../index/engine/EngineTestCase.java | 2 +- 4 files changed, 117 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java index e868da5e82a..b406621e978 100644 --- a/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java +++ b/server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java @@ -39,6 +39,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.OptionalLong; import java.util.Set; import java.util.function.Function; @@ -127,6 +128,13 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L */ final Map checkpoints; + /** + * A callback invoked when the global checkpoint is updated. For primary mode this occurs if the computed global checkpoint advances on + * the basis of state changes tracked here. For non-primary mode this occurs if the local knowledge of the global checkpoint advances + * due to an update from the primary. + */ + private final LongConsumer onGlobalCheckpointUpdated; + /** * This set contains allocation IDs for which there is a thread actively waiting for the local checkpoint to advance to at least the * current global checkpoint. @@ -391,7 +399,8 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L final ShardId shardId, final String allocationId, final IndexSettings indexSettings, - final long globalCheckpoint) { + final long globalCheckpoint, + final LongConsumer onGlobalCheckpointUpdated) { super(shardId, indexSettings); assert globalCheckpoint >= SequenceNumbers.UNASSIGNED_SEQ_NO : "illegal initial global checkpoint: " + globalCheckpoint; this.shardAllocationId = allocationId; @@ -400,6 +409,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L this.appliedClusterStateVersion = -1L; this.checkpoints = new HashMap<>(1 + indexSettings.getNumberOfReplicas()); checkpoints.put(allocationId, new CheckpointState(SequenceNumbers.UNASSIGNED_SEQ_NO, globalCheckpoint, false, false)); + this.onGlobalCheckpointUpdated = Objects.requireNonNull(onGlobalCheckpointUpdated); this.pendingInSync = new HashSet<>(); this.routingTable = null; this.replicationGroup = null; @@ -456,7 +466,10 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L updateGlobalCheckpoint( shardAllocationId, globalCheckpoint, - current -> logger.trace("updating global checkpoint from [{}] to [{}] due to [{}]", current, globalCheckpoint, reason)); + current -> { + logger.trace("updated global checkpoint from [{}] to [{}] due to [{}]", current, globalCheckpoint, reason); + onGlobalCheckpointUpdated.accept(globalCheckpoint); + }); assert invariant(); } @@ -474,7 +487,7 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L allocationId, globalCheckpoint, current -> logger.trace( - "updating local knowledge for [{}] on the primary of the global checkpoint from [{}] to [{}]", + "updated local knowledge for [{}] on the primary of the global checkpoint from [{}] to [{}]", allocationId, current, globalCheckpoint)); @@ -485,8 +498,8 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L final CheckpointState cps = checkpoints.get(allocationId); assert !this.shardAllocationId.equals(allocationId) || cps != null; if (cps != null && globalCheckpoint > cps.globalCheckpoint) { - ifUpdated.accept(cps.globalCheckpoint); cps.globalCheckpoint = globalCheckpoint; + ifUpdated.accept(cps.globalCheckpoint); } } @@ -737,8 +750,9 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L assert computedGlobalCheckpoint >= globalCheckpoint : "new global checkpoint [" + computedGlobalCheckpoint + "] is lower than previous one [" + globalCheckpoint + "]"; if (globalCheckpoint != computedGlobalCheckpoint) { - logger.trace("global checkpoint updated to [{}]", computedGlobalCheckpoint); cps.globalCheckpoint = computedGlobalCheckpoint; + logger.trace("updated global checkpoint to [{}]", computedGlobalCheckpoint); + onGlobalCheckpointUpdated.accept(computedGlobalCheckpoint); } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 3aa281da101..08a0111fb4d 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -297,8 +297,9 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP); this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays); - this.replicationTracker = new ReplicationTracker(shardId, shardRouting.allocationId().getId(), indexSettings, - SequenceNumbers.UNASSIGNED_SEQ_NO); + final String aId = shardRouting.allocationId().getId(); + this.replicationTracker = + new ReplicationTracker(shardId, aId, indexSettings, SequenceNumbers.UNASSIGNED_SEQ_NO, globalCheckpoint -> {}); // the query cache is a node-level thing, however we want the most popular filters // to be computed on a per-shard basis if (IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.get(settings)) { diff --git a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java index e001f82809b..3948da9c111 100644 --- a/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java +++ b/server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerTests.java @@ -47,7 +47,9 @@ import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; +import java.util.function.LongConsumer; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -60,7 +62,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.not; public class ReplicationTrackerTests extends ESTestCase { - + public void testEmptyShards() { final ReplicationTracker tracker = newTracker(AllocationId.newInitializing()); assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO)); @@ -99,6 +101,11 @@ public class ReplicationTrackerTests extends ESTestCase { return allocationIds.stream().map(AllocationId::getId).collect(Collectors.toSet()); } + private void updateLocalCheckpoint(final ReplicationTracker tracker, final String allocationId, final long localCheckpoint) { + tracker.updateLocalCheckpoint(allocationId, localCheckpoint); + assertThat(updatedGlobalCheckpoint.get(), equalTo(tracker.getGlobalCheckpoint())); + } + public void testGlobalCheckpointUpdate() { final long initialClusterStateVersion = randomNonNegativeLong(); Map allocations = new HashMap<>(); @@ -137,14 +144,14 @@ public class ReplicationTrackerTests extends ESTestCase { assertThat(tracker.getReplicationGroup().getReplicationTargets().size(), equalTo(1)); initializing.forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED)); assertThat(tracker.getReplicationGroup().getReplicationTargets().size(), equalTo(1 + initializing.size())); - allocations.keySet().forEach(aId -> tracker.updateLocalCheckpoint(aId.getId(), allocations.get(aId))); + allocations.keySet().forEach(aId -> updateLocalCheckpoint(tracker, aId.getId(), allocations.get(aId))); assertThat(tracker.getGlobalCheckpoint(), equalTo(minLocalCheckpoint)); // increment checkpoints active.forEach(aId -> allocations.put(aId, allocations.get(aId) + 1 + randomInt(4))); initializing.forEach(aId -> allocations.put(aId, allocations.get(aId) + 1 + randomInt(4))); - allocations.keySet().forEach(aId -> tracker.updateLocalCheckpoint(aId.getId(), allocations.get(aId))); + allocations.keySet().forEach(aId -> updateLocalCheckpoint(tracker, aId.getId(), allocations.get(aId))); final long minLocalCheckpointAfterUpdates = allocations.entrySet().stream().map(Map.Entry::getValue).min(Long::compareTo).orElse(UNASSIGNED_SEQ_NO); @@ -153,7 +160,7 @@ public class ReplicationTrackerTests extends ESTestCase { final AllocationId extraId = AllocationId.newInitializing(); // first check that adding it without the master blessing doesn't change anything. - tracker.updateLocalCheckpoint(extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4)); + updateLocalCheckpoint(tracker, extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4)); assertNull(tracker.checkpoints.get(extraId)); expectThrows(IllegalStateException.class, () -> tracker.initiateTracking(extraId.getId())); @@ -165,7 +172,7 @@ public class ReplicationTrackerTests extends ESTestCase { // now notify for the new id if (randomBoolean()) { - tracker.updateLocalCheckpoint(extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4)); + updateLocalCheckpoint(tracker, extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4)); markAsTrackingAndInSyncQuietly(tracker, extraId.getId(), randomInt((int) minLocalCheckpointAfterUpdates)); } else { markAsTrackingAndInSyncQuietly(tracker, extraId.getId(), minLocalCheckpointAfterUpdates + 1 + randomInt(4)); @@ -175,6 +182,64 @@ public class ReplicationTrackerTests extends ESTestCase { assertThat(tracker.getGlobalCheckpoint(), greaterThan(minLocalCheckpoint)); } + public void testUpdateGlobalCheckpointOnReplica() { + final AllocationId active = AllocationId.newInitializing(); + final ReplicationTracker tracker = newTracker(active); + final long globalCheckpoint = randomLongBetween(NO_OPS_PERFORMED, Long.MAX_VALUE - 1); + tracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "test"); + assertThat(updatedGlobalCheckpoint.get(), equalTo(globalCheckpoint)); + final long nonUpdate = randomLongBetween(NO_OPS_PERFORMED, globalCheckpoint); + updatedGlobalCheckpoint.set(UNASSIGNED_SEQ_NO); + tracker.updateGlobalCheckpointOnReplica(nonUpdate, "test"); + assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO)); + final long update = randomLongBetween(globalCheckpoint, Long.MAX_VALUE); + tracker.updateGlobalCheckpointOnReplica(update, "test"); + assertThat(updatedGlobalCheckpoint.get(), equalTo(update)); + } + + public void testMarkAllocationIdAsInSync() throws BrokenBarrierException, InterruptedException { + final long initialClusterStateVersion = randomNonNegativeLong(); + Map activeWithCheckpoints = randomAllocationsWithLocalCheckpoints(1, 1); + Set active = new HashSet<>(activeWithCheckpoints.keySet()); + Map initializingWithCheckpoints = randomAllocationsWithLocalCheckpoints(1, 1); + Set initializing = new HashSet<>(initializingWithCheckpoints.keySet()); + final AllocationId primaryId = active.iterator().next(); + final AllocationId replicaId = initializing.iterator().next(); + final ReplicationTracker tracker = newTracker(primaryId); + tracker.updateFromMaster(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId), emptySet()); + final long localCheckpoint = randomLongBetween(0, Long.MAX_VALUE - 1); + tracker.activatePrimaryMode(localCheckpoint); + tracker.initiateTracking(replicaId.getId()); + final CyclicBarrier barrier = new CyclicBarrier(2); + final Thread thread = new Thread(() -> { + try { + barrier.await(); + tracker.markAllocationIdAsInSync( + replicaId.getId(), + randomLongBetween(NO_OPS_PERFORMED, localCheckpoint - 1)); + barrier.await(); + } catch (BrokenBarrierException | InterruptedException e) { + throw new AssertionError(e); + } + }); + thread.start(); + barrier.await(); + awaitBusy(tracker::pendingInSync); + final long updatedLocalCheckpoint = randomLongBetween(1 + localCheckpoint, Long.MAX_VALUE); + // there is a shard copy pending in sync, the global checkpoint can not advance + updatedGlobalCheckpoint.set(UNASSIGNED_SEQ_NO); + tracker.updateLocalCheckpoint(primaryId.getId(), updatedLocalCheckpoint); + assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO)); + // we are implicitly marking the pending in sync copy as in sync with the current global checkpoint, no advancement should occur + tracker.updateLocalCheckpoint(replicaId.getId(), localCheckpoint); + assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO)); + barrier.await(); + thread.join(); + // now we expect that the global checkpoint would advance + tracker.markAllocationIdAsInSync(replicaId.getId(), updatedLocalCheckpoint); + assertThat(updatedGlobalCheckpoint.get(), equalTo(updatedLocalCheckpoint)); + } + public void testMissingActiveIdsPreventAdvance() { final Map active = randomAllocationsWithLocalCheckpoints(2, 5); final Map initializing = randomAllocationsWithLocalCheckpoints(0, 5); @@ -191,14 +256,16 @@ public class ReplicationTrackerTests extends ESTestCase { .entrySet() .stream() .filter(e -> !e.getKey().equals(missingActiveID)) - .forEach(e -> tracker.updateLocalCheckpoint(e.getKey().getId(), e.getValue())); + .forEach(e -> updateLocalCheckpoint(tracker, e.getKey().getId(), e.getValue())); if (missingActiveID.equals(primaryId) == false) { assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO)); + assertThat(updatedGlobalCheckpoint.get(), equalTo(UNASSIGNED_SEQ_NO)); } // now update all knowledge of all shards - assigned.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP)); + assigned.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP)); assertThat(tracker.getGlobalCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO))); + assertThat(updatedGlobalCheckpoint.get(), not(equalTo(UNASSIGNED_SEQ_NO))); } public void testMissingInSyncIdsPreventAdvance() { @@ -213,13 +280,15 @@ public class ReplicationTrackerTests extends ESTestCase { randomSubsetOf(randomIntBetween(1, initializing.size() - 1), initializing.keySet()).forEach(aId -> markAsTrackingAndInSyncQuietly(tracker, aId.getId(), NO_OPS_PERFORMED)); - active.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP)); + active.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP)); assertThat(tracker.getGlobalCheckpoint(), equalTo(NO_OPS_PERFORMED)); + assertThat(updatedGlobalCheckpoint.get(), equalTo(NO_OPS_PERFORMED)); // update again - initializing.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP)); + initializing.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP)); assertThat(tracker.getGlobalCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO))); + assertThat(updatedGlobalCheckpoint.get(), not(equalTo(UNASSIGNED_SEQ_NO))); } public void testInSyncIdsAreIgnoredIfNotValidatedByMaster() { @@ -236,7 +305,7 @@ public class ReplicationTrackerTests extends ESTestCase { List> allocations = Arrays.asList(active, initializing, nonApproved); Collections.shuffle(allocations, random()); - allocations.forEach(a -> a.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP))); + allocations.forEach(a -> a.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP))); assertThat(tracker.getGlobalCheckpoint(), not(equalTo(UNASSIGNED_SEQ_NO))); } @@ -271,7 +340,7 @@ public class ReplicationTrackerTests extends ESTestCase { initializing.forEach(k -> markAsTrackingAndInSyncQuietly(tracker, k.getId(), NO_OPS_PERFORMED)); } if (randomBoolean()) { - allocations.forEach((aid, localCP) -> tracker.updateLocalCheckpoint(aid.getId(), localCP)); + allocations.forEach((aid, localCP) -> updateLocalCheckpoint(tracker, aid.getId(), localCP)); } // now remove shards @@ -281,9 +350,9 @@ public class ReplicationTrackerTests extends ESTestCase { ids(activeToStay.keySet()), routingTable(initializingToStay.keySet(), primaryId), emptySet()); - allocations.forEach((aid, ckp) -> tracker.updateLocalCheckpoint(aid.getId(), ckp + 10L)); + allocations.forEach((aid, ckp) -> updateLocalCheckpoint(tracker, aid.getId(), ckp + 10L)); } else { - allocations.forEach((aid, ckp) -> tracker.updateLocalCheckpoint(aid.getId(), ckp + 10L)); + allocations.forEach((aid, ckp) -> updateLocalCheckpoint(tracker, aid.getId(), ckp + 10L)); tracker.updateFromMaster( initialClusterStateVersion + 2, ids(activeToStay.keySet()), @@ -331,7 +400,7 @@ public class ReplicationTrackerTests extends ESTestCase { final List elements = IntStream.rangeClosed(0, globalCheckpoint - 1).boxed().collect(Collectors.toList()); Randomness.shuffle(elements); for (int i = 0; i < elements.size(); i++) { - tracker.updateLocalCheckpoint(trackingAllocationId.getId(), elements.get(i)); + updateLocalCheckpoint(tracker, trackingAllocationId.getId(), elements.get(i)); assertFalse(complete.get()); assertFalse(tracker.getTrackedLocalCheckpointForShard(trackingAllocationId.getId()).inSync); assertBusy(() -> assertTrue(tracker.pendingInSync.contains(trackingAllocationId.getId()))); @@ -339,7 +408,7 @@ public class ReplicationTrackerTests extends ESTestCase { if (randomBoolean()) { // normal path, shard catches up - tracker.updateLocalCheckpoint(trackingAllocationId.getId(), randomIntBetween(globalCheckpoint, 64)); + updateLocalCheckpoint(tracker, trackingAllocationId.getId(), randomIntBetween(globalCheckpoint, 64)); // synchronize with the waiting thread to mark that it is complete barrier.await(); assertTrue(complete.get()); @@ -355,13 +424,16 @@ public class ReplicationTrackerTests extends ESTestCase { assertFalse(tracker.pendingInSync.contains(trackingAllocationId.getId())); thread.join(); } + + private AtomicLong updatedGlobalCheckpoint = new AtomicLong(UNASSIGNED_SEQ_NO); private ReplicationTracker newTracker(final AllocationId allocationId) { return new ReplicationTracker( new ShardId("test", "_na_", 0), allocationId.getId(), IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), - UNASSIGNED_SEQ_NO); + UNASSIGNED_SEQ_NO, + updatedGlobalCheckpoint::set); } public void testWaitForAllocationIdToBeInSyncCanBeInterrupted() throws BrokenBarrierException, InterruptedException { @@ -488,10 +560,10 @@ public class ReplicationTrackerTests extends ESTestCase { // the tracking allocation IDs should play no role in determining the global checkpoint final Map activeLocalCheckpoints = newActiveAllocationIds.stream().collect(Collectors.toMap(Function.identity(), a -> randomIntBetween(1, 1024))); - activeLocalCheckpoints.forEach((a, l) -> tracker.updateLocalCheckpoint(a.getId(), l)); + activeLocalCheckpoints.forEach((a, l) -> updateLocalCheckpoint(tracker, a.getId(), l)); final Map initializingLocalCheckpoints = newInitializingAllocationIds.stream().collect(Collectors.toMap(Function.identity(), a -> randomIntBetween(1, 1024))); - initializingLocalCheckpoints.forEach((a, l) -> tracker.updateLocalCheckpoint(a.getId(), l)); + initializingLocalCheckpoints.forEach((a, l) -> updateLocalCheckpoint(tracker, a.getId(), l)); assertTrue( activeLocalCheckpoints .entrySet() @@ -504,6 +576,7 @@ public class ReplicationTrackerTests extends ESTestCase { .allMatch(e -> tracker.getTrackedLocalCheckpointForShard(e.getKey().getId()).getLocalCheckpoint() == e.getValue())); final long minimumActiveLocalCheckpoint = (long) activeLocalCheckpoints.values().stream().min(Integer::compareTo).get(); assertThat(tracker.getGlobalCheckpoint(), equalTo(minimumActiveLocalCheckpoint)); + assertThat(updatedGlobalCheckpoint.get(), equalTo(minimumActiveLocalCheckpoint)); final long minimumInitailizingLocalCheckpoint = (long) initializingLocalCheckpoints.values().stream().min(Integer::compareTo).get(); // now we are going to add a new allocation ID and bring it in sync which should move it to the in-sync allocation IDs @@ -635,10 +708,11 @@ public class ReplicationTrackerTests extends ESTestCase { FakeClusterState clusterState = initialState(); final AllocationId primaryAllocationId = clusterState.routingTable.primaryShard().allocationId(); + final LongConsumer onUpdate = updatedGlobalCheckpoint -> {}; ReplicationTracker oldPrimary = - new ReplicationTracker(shardId, primaryAllocationId.getId(), indexSettings, UNASSIGNED_SEQ_NO); + new ReplicationTracker(shardId, primaryAllocationId.getId(), indexSettings, UNASSIGNED_SEQ_NO, onUpdate); ReplicationTracker newPrimary = - new ReplicationTracker(shardId, primaryAllocationId.getRelocationId(), indexSettings, UNASSIGNED_SEQ_NO); + new ReplicationTracker(shardId, primaryAllocationId.getRelocationId(), indexSettings, UNASSIGNED_SEQ_NO, onUpdate); Set allocationIds = new HashSet<>(Arrays.asList(oldPrimary.shardAllocationId, newPrimary.shardAllocationId)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index f2652224549..2a84a8f4246 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -460,7 +460,7 @@ public abstract class EngineTestCase extends ESTestCase { TimeValue.timeValueMinutes(5), refreshListenerList, Collections.emptyList(), indexSort, handler, new NoneCircuitBreakerService(), globalCheckpointSupplier == null ? - new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED) : + new ReplicationTracker(shardId, allocationId.getId(), indexSettings, SequenceNumbers.NO_OPS_PERFORMED, update -> {}) : globalCheckpointSupplier, primaryTerm::get); return config; } From 0b7fb4e7b943c4e48d8dde7a5be0839231062fdf Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 7 Aug 2018 12:26:57 -0700 Subject: [PATCH 10/20] Painless: Clean up FunctionRef (#32644) This change consolidates all the logic for generating a FunctionReference (renamed from FunctionRef) from several arbitrary constructors to a single static function that is used at both compile-time and run-time. This increases long-term maintainability as it is much easier to follow when and how a function reference is being generated. It moves most of the duplicated logic out of the ECapturingFuncRef, EFuncRef and ELambda nodes and Def as well. --- .../java/org/elasticsearch/painless/Def.java | 77 ++-- .../elasticsearch/painless/FunctionRef.java | 421 ++++++++---------- .../elasticsearch/painless/MethodWriter.java | 15 + .../painless/lookup/PainlessClass.java | 6 +- .../painless/lookup/PainlessClassBuilder.java | 6 +- .../painless/lookup/PainlessLookup.java | 40 ++ .../lookup/PainlessLookupBuilder.java | 2 +- .../painless/node/ECapturingFunctionRef.java | 34 +- .../painless/node/EFunctionRef.java | 52 +-- .../elasticsearch/painless/node/ELambda.java | 39 +- .../painless/FunctionRefTests.java | 23 +- .../elasticsearch/painless/LambdaTests.java | 4 +- 12 files changed, 304 insertions(+), 415 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 1db1e4f7cac..4752c2b2685 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -279,10 +279,6 @@ public final class Def { String type = signature.substring(1, separator); String call = signature.substring(separator+1, separator2); int numCaptures = Integer.parseInt(signature.substring(separator2+1)); - Class captures[] = new Class[numCaptures]; - for (int capture = 0; capture < captures.length; capture++) { - captures[capture] = callSiteType.parameterType(i + 1 + capture); - } MethodHandle filter; Class interfaceType = method.typeParameters.get(i - 1 - replaced); if (signature.charAt(0) == 'S') { @@ -294,11 +290,15 @@ public final class Def { interfaceType, type, call, - captures); + numCaptures); } else if (signature.charAt(0) == 'D') { // the interface type is now known, but we need to get the implementation. // this is dynamically based on the receiver type (and cached separately, underneath // this cache). It won't blow up since we never nest here (just references) + Class captures[] = new Class[numCaptures]; + for (int capture = 0; capture < captures.length; capture++) { + captures[capture] = callSiteType.parameterType(i + 1 + capture); + } MethodType nestedType = MethodType.methodType(interfaceType, captures); CallSite nested = DefBootstrap.bootstrap(painlessLookup, localMethods, @@ -331,57 +331,34 @@ public final class Def { */ static MethodHandle lookupReference(PainlessLookup painlessLookup, Map localMethods, MethodHandles.Lookup methodHandlesLookup, String interfaceClass, Class receiverClass, String name) throws Throwable { - Class interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass); - PainlessMethod interfaceMethod = painlessLookup.lookupPainlessClass(interfaceType).functionalMethod; - if (interfaceMethod == null) { - throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface"); - } - int arity = interfaceMethod.typeParameters.size(); - PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity); + Class interfaceType = painlessLookup.canonicalTypeNameToType(interfaceClass); + PainlessMethod interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(interfaceType); + if (interfaceMethod == null) { + throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface"); + } + int arity = interfaceMethod.typeParameters.size(); + PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity); return lookupReferenceInternal(painlessLookup, localMethods, methodHandlesLookup, interfaceType, PainlessLookupUtility.typeToCanonicalTypeName(implMethod.targetClass), - implMethod.javaMethod.getName(), receiverClass); + implMethod.javaMethod.getName(), 1); } /** Returns a method handle to an implementation of clazz, given method reference signature. */ private static MethodHandle lookupReferenceInternal(PainlessLookup painlessLookup, Map localMethods, - MethodHandles.Lookup methodHandlesLookup, Class clazz, String type, String call, Class... captures) throws Throwable { - final FunctionRef ref; - if ("this".equals(type)) { - // user written method - PainlessMethod interfaceMethod = painlessLookup.lookupPainlessClass(clazz).functionalMethod; - if (interfaceMethod == null) { - throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + PainlessLookupUtility.typeToCanonicalTypeName(clazz) + "], not a functional interface"); - } - int arity = interfaceMethod.typeParameters.size() + captures.length; - LocalMethod localMethod = localMethods.get(Locals.buildLocalMethodKey(call, arity)); - if (localMethod == null) { - // is it a synthetic method? If we generated the method ourselves, be more helpful. It can only fail - // because the arity does not match the expected interface type. - if (call.contains("$")) { - throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.javaMethod.getName() + - "] in [" + clazz + "]"); - } - throw new IllegalArgumentException("Unknown call [" + call + "] with [" + arity + "] arguments."); - } - ref = new FunctionRef(clazz, interfaceMethod, call, localMethod.methodType, captures.length); - } else { - // whitelist lookup - ref = FunctionRef.resolveFromLookup(painlessLookup, clazz, type, call, captures.length); - } - final CallSite callSite = LambdaBootstrap.lambdaBootstrap( - methodHandlesLookup, - ref.interfaceMethodName, - ref.factoryMethodType, - ref.interfaceMethodType, - ref.delegateClassName, - ref.delegateInvokeType, - ref.delegateMethodName, - ref.delegateMethodType, - ref.isDelegateInterface ? 1 : 0 - ); - return callSite.dynamicInvoker().asType(MethodType.methodType(clazz, captures)); + MethodHandles.Lookup methodHandlesLookup, Class clazz, String type, String call, int captures) throws Throwable { + final FunctionRef ref = FunctionRef.create(painlessLookup, localMethods, null, clazz, type, call, captures); + final CallSite callSite = LambdaBootstrap.lambdaBootstrap( + methodHandlesLookup, + ref.interfaceMethodName, + ref.factoryMethodType, + ref.interfaceMethodType, + ref.delegateClassName, + ref.delegateInvokeType, + ref.delegateMethodName, + ref.delegateMethodType, + ref.isDelegateInterface ? 1 : 0 + ); + return callSite.dynamicInvoker().asType(MethodType.methodType(clazz, ref.factoryMethodType.parameterArray())); } /** diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index cc558489446..065f63dc3f5 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -20,17 +20,17 @@ package org.elasticsearch.painless; import org.elasticsearch.painless.Locals.LocalMethod; -import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.lookup.PainlessConstructor; import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.lookup.PainlessLookupUtility; import org.elasticsearch.painless.lookup.PainlessMethod; -import org.objectweb.asm.Type; import java.lang.invoke.MethodType; -import java.lang.reflect.Constructor; import java.lang.reflect.Modifier; +import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Objects; import static org.elasticsearch.painless.WriterConstants.CLASS_NAME; import static org.objectweb.asm.Opcodes.H_INVOKEINTERFACE; @@ -39,251 +39,210 @@ import static org.objectweb.asm.Opcodes.H_INVOKEVIRTUAL; import static org.objectweb.asm.Opcodes.H_NEWINVOKESPECIAL; /** - * Reference to a function or lambda. - *

- * Once you have created one of these, you have "everything you need" to call {@link LambdaBootstrap} - * either statically from bytecode with invokedynamic, or at runtime from Java. + * Contains all the values necessary to write the instruction to initiate a + * {@link LambdaBootstrap} for either a function reference or a user-defined + * lambda function. */ public class FunctionRef { + /** + * Creates a new FunctionRef which will resolve {@code type::call} from the whitelist. + * @param painlessLookup the whitelist against which this script is being compiled + * @param localMethods user-defined and synthetic methods generated directly on the script class + * @param location the character number within the script at compile-time + * @param targetClass functional interface type to implement. + * @param typeName the left hand side of a method reference expression + * @param methodName the right hand side of a method reference expression + * @param numberOfCaptures number of captured arguments + */ + public static FunctionRef create(PainlessLookup painlessLookup, Map localMethods, Location location, + Class targetClass, String typeName, String methodName, int numberOfCaptures) { + + Objects.requireNonNull(painlessLookup); + Objects.requireNonNull(targetClass); + Objects.requireNonNull(typeName); + Objects.requireNonNull(methodName); + + String targetClassName = PainlessLookupUtility.typeToCanonicalTypeName(targetClass); + PainlessMethod interfaceMethod; + + try { + try { + interfaceMethod = painlessLookup.lookupFunctionalInterfacePainlessMethod(targetClass); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException("cannot convert function reference [" + typeName + "::" + methodName + "] " + + "to a non-functional interface [" + targetClassName + "]", iae); + } + + String interfaceMethodName = interfaceMethod.javaMethod.getName(); + MethodType interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1); + String delegateClassName; + boolean isDelegateInterface; + int delegateInvokeType; + String delegateMethodName; + MethodType delegateMethodType; + + Class delegateMethodReturnType; + List> delegateMethodParameters; + int interfaceTypeParametersSize = interfaceMethod.typeParameters.size(); + + if ("this".equals(typeName)) { + Objects.requireNonNull(localMethods); + + if (numberOfCaptures < 0) { + throw new IllegalStateException("internal error"); + } + + String localMethodKey = Locals.buildLocalMethodKey(methodName, numberOfCaptures + interfaceTypeParametersSize); + LocalMethod localMethod = localMethods.get(localMethodKey); + + if (localMethod == null) { + throw new IllegalArgumentException("function reference [this::" + localMethodKey + "] " + + "matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " + + "not found" + (localMethodKey.contains("$") ? " due to an incorrect number of arguments" : "") + ); + } + + delegateClassName = CLASS_NAME; + isDelegateInterface = false; + delegateInvokeType = H_INVOKESTATIC; + delegateMethodName = localMethod.name; + delegateMethodType = localMethod.methodType; + + delegateMethodReturnType = localMethod.returnType; + delegateMethodParameters = localMethod.typeParameters; + } else if ("new".equals(methodName)) { + if (numberOfCaptures != 0) { + throw new IllegalStateException("internal error"); + } + + PainlessConstructor painlessConstructor; + + try { + painlessConstructor = painlessLookup.lookupPainlessConstructor(typeName, interfaceTypeParametersSize); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException("function reference [" + typeName + "::new/" + interfaceTypeParametersSize + "] " + + "matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " + + "not found", iae); + } + + delegateClassName = painlessConstructor.javaConstructor.getDeclaringClass().getName(); + isDelegateInterface = false; + delegateInvokeType = H_NEWINVOKESPECIAL; + delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME; + delegateMethodType = painlessConstructor.methodType; + + delegateMethodReturnType = painlessConstructor.javaConstructor.getDeclaringClass(); + delegateMethodParameters = painlessConstructor.typeParameters; + } else { + if (numberOfCaptures != 0 && numberOfCaptures != 1) { + throw new IllegalStateException("internal error"); + } + + boolean captured = numberOfCaptures == 1; + PainlessMethod painlessMethod; + + try { + painlessMethod = painlessLookup.lookupPainlessMethod(typeName, true, methodName, interfaceTypeParametersSize); + + if (captured) { + throw new IllegalStateException("internal error"); + } + } catch (IllegalArgumentException staticIAE) { + try { + painlessMethod = painlessLookup.lookupPainlessMethod(typeName, false, methodName, + captured ? interfaceTypeParametersSize : interfaceTypeParametersSize - 1); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException( + "function reference " + "[" + typeName + "::" + methodName + "/" + interfaceTypeParametersSize + "] " + + "matching [" + targetClassName + ", " + interfaceMethodName + "/" + interfaceTypeParametersSize + "] " + + "not found", iae); + } + } + + delegateClassName = painlessMethod.javaMethod.getDeclaringClass().getName(); + isDelegateInterface = painlessMethod.javaMethod.getDeclaringClass().isInterface(); + + if (Modifier.isStatic(painlessMethod.javaMethod.getModifiers())) { + delegateInvokeType = H_INVOKESTATIC; + } else if (isDelegateInterface) { + delegateInvokeType = H_INVOKEINTERFACE; + } else { + delegateInvokeType = H_INVOKEVIRTUAL; + } + + delegateMethodName = painlessMethod.javaMethod.getName(); + delegateMethodType = painlessMethod.methodType; + + delegateMethodReturnType = painlessMethod.returnType; + + if (delegateMethodType.parameterList().size() > painlessMethod.typeParameters.size()) { + delegateMethodParameters = new ArrayList<>(painlessMethod.typeParameters); + delegateMethodParameters.add(0, delegateMethodType.parameterType(0)); + } else { + delegateMethodParameters = painlessMethod.typeParameters; + } + } + + if (location != null) { + for (int typeParameter = 0; typeParameter < interfaceTypeParametersSize; ++typeParameter) { + Class from = interfaceMethod.typeParameters.get(typeParameter); + Class to = delegateMethodParameters.get(numberOfCaptures + typeParameter); + AnalyzerCaster.getLegalCast(location, from, to, false, true); + } + + if (interfaceMethod.returnType != void.class) { + AnalyzerCaster.getLegalCast(location, delegateMethodReturnType, interfaceMethod.returnType, false, true); + } + } + + MethodType factoryMethodType = MethodType.methodType(targetClass, + delegateMethodType.dropParameterTypes(numberOfCaptures, delegateMethodType.parameterCount())); + delegateMethodType = delegateMethodType.dropParameterTypes(0, numberOfCaptures); + + return new FunctionRef(interfaceMethodName, interfaceMethodType, + delegateClassName, isDelegateInterface, delegateInvokeType, delegateMethodName, delegateMethodType, + factoryMethodType + ); + } catch (IllegalArgumentException iae) { + if (location != null) { + throw location.createError(iae); + } + + throw iae; + } + } + /** functional interface method name */ public final String interfaceMethodName; - /** factory (CallSite) method signature */ - public final MethodType factoryMethodType; /** functional interface method signature */ public final MethodType interfaceMethodType; /** class of the delegate method to be called */ public final String delegateClassName; + /** whether a call is made on a delegate interface */ + public final boolean isDelegateInterface; /** the invocation type of the delegate method */ public final int delegateInvokeType; /** the name of the delegate method */ public final String delegateMethodName; /** delegate method signature */ public final MethodType delegateMethodType; + /** factory (CallSite) method signature */ + public final MethodType factoryMethodType; - /** interface method */ - public final PainlessMethod interfaceMethod; - /** delegate method type parameters */ - public final List> delegateTypeParameters; - /** delegate method return type */ - public final Class delegateReturnType; + private FunctionRef( + String interfaceMethodName, MethodType interfaceMethodType, + String delegateClassName, boolean isDelegateInterface, + int delegateInvokeType, String delegateMethodName, MethodType delegateMethodType, + MethodType factoryMethodType) { - /** factory method type descriptor */ - public final String factoryDescriptor; - /** functional interface method as type */ - public final Type interfaceType; - /** delegate method type method as type */ - public final Type delegateType; - - /** whether a call is made on a delegate interface */ - public final boolean isDelegateInterface; - - /** - * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist. - * @param painlessLookup the whitelist against which this script is being compiled - * @param expected functional interface type to implement. - * @param type the left hand side of a method reference expression - * @param call the right hand side of a method reference expression - * @param numCaptures number of captured arguments - */ - public static FunctionRef resolveFromLookup( - PainlessLookup painlessLookup, Class expected, String type, String call, int numCaptures) { - - if ("new".equals(call)) { - return new FunctionRef(expected, painlessLookup.lookupPainlessClass(expected).functionalMethod, - lookup(painlessLookup, expected, type), numCaptures); - } else { - return new FunctionRef(expected, painlessLookup.lookupPainlessClass(expected).functionalMethod, - lookup(painlessLookup, expected, type, call, numCaptures > 0), numCaptures); - } - } - - /** - * Creates a new FunctionRef (already resolved) - * @param expected functional interface type to implement - * @param interfaceMethod functional interface method - * @param delegateConstructor implementation constructor - * @param numCaptures number of captured arguments - */ - public FunctionRef(Class expected, PainlessMethod interfaceMethod, PainlessConstructor delegateConstructor, int numCaptures) { - Constructor javaConstructor = delegateConstructor.javaConstructor; - MethodType delegateMethodType = delegateConstructor.methodType; - - this.interfaceMethodName = interfaceMethod.javaMethod.getName(); - this.factoryMethodType = MethodType.methodType(expected, - delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount())); - this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1); - - this.delegateClassName = javaConstructor.getDeclaringClass().getName(); - this.isDelegateInterface = false; - this.delegateInvokeType = H_NEWINVOKESPECIAL; - this.delegateMethodName = PainlessLookupUtility.CONSTRUCTOR_NAME; - this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures); - - this.interfaceMethod = interfaceMethod; - this.delegateTypeParameters = delegateConstructor.typeParameters; - this.delegateReturnType = void.class; - - this.factoryDescriptor = factoryMethodType.toMethodDescriptorString(); - this.interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString()); - this.delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString()); - } - - /** - * Creates a new FunctionRef (already resolved) - * @param expected functional interface type to implement - * @param interfaceMethod functional interface method - * @param delegateMethod implementation method - * @param numCaptures number of captured arguments - */ - public FunctionRef(Class expected, PainlessMethod interfaceMethod, PainlessMethod delegateMethod, int numCaptures) { - MethodType delegateMethodType = delegateMethod.methodType; - - this.interfaceMethodName = interfaceMethod.javaMethod.getName(); - this.factoryMethodType = MethodType.methodType(expected, - delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount())); - this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1); - - this.delegateClassName = delegateMethod.javaMethod.getDeclaringClass().getName(); - this.isDelegateInterface = delegateMethod.javaMethod.getDeclaringClass().isInterface(); - - if (Modifier.isStatic(delegateMethod.javaMethod.getModifiers())) { - this.delegateInvokeType = H_INVOKESTATIC; - } else if (delegateMethod.javaMethod.getDeclaringClass().isInterface()) { - this.delegateInvokeType = H_INVOKEINTERFACE; - } else { - this.delegateInvokeType = H_INVOKEVIRTUAL; - } - - this.delegateMethodName = delegateMethod.javaMethod.getName(); - this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures); - - this.interfaceMethod = interfaceMethod; - this.delegateTypeParameters = delegateMethod.typeParameters; - this.delegateReturnType = delegateMethod.returnType; - - this.factoryDescriptor = factoryMethodType.toMethodDescriptorString(); - this.interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString()); - this.delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString()); - } - - /** - * Creates a new FunctionRef (already resolved) - * @param expected functional interface type to implement - * @param interfaceMethod functional interface method - * @param delegateMethod implementation method - * @param numCaptures number of captured arguments - */ - public FunctionRef(Class expected, PainlessMethod interfaceMethod, LocalMethod delegateMethod, int numCaptures) { - MethodType delegateMethodType = delegateMethod.methodType; - - this.interfaceMethodName = interfaceMethod.javaMethod.getName(); - this.factoryMethodType = MethodType.methodType(expected, - delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount())); - this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1); - - this.delegateClassName = CLASS_NAME; - this.isDelegateInterface = false; - this.delegateInvokeType = H_INVOKESTATIC; - - this.delegateMethodName = delegateMethod.name; - this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures); - - this.interfaceMethod = interfaceMethod; - this.delegateTypeParameters = delegateMethod.typeParameters; - this.delegateReturnType = delegateMethod.returnType; - - this.factoryDescriptor = factoryMethodType.toMethodDescriptorString(); - this.interfaceType = Type.getMethodType(interfaceMethodType.toMethodDescriptorString()); - this.delegateType = Type.getMethodType(this.delegateMethodType.toMethodDescriptorString()); - } - - /** - * Creates a new FunctionRef (low level). - * It is for runtime use only. - */ - public FunctionRef(Class expected, - PainlessMethod interfaceMethod, String delegateMethodName, MethodType delegateMethodType, int numCaptures) { - this.interfaceMethodName = interfaceMethod.javaMethod.getName(); - this.factoryMethodType = MethodType.methodType(expected, - delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount())); - this.interfaceMethodType = interfaceMethod.methodType.dropParameterTypes(0, 1); - - this.delegateClassName = CLASS_NAME; - this.delegateInvokeType = H_INVOKESTATIC; + this.interfaceMethodName = interfaceMethodName; + this.interfaceMethodType = interfaceMethodType; + this.delegateClassName = delegateClassName; + this.isDelegateInterface = isDelegateInterface; + this.delegateInvokeType = delegateInvokeType; this.delegateMethodName = delegateMethodName; - this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures); - this.isDelegateInterface = false; - - this.interfaceMethod = null; - this.delegateTypeParameters = null; - this.delegateReturnType = null; - - this.factoryDescriptor = null; - this.interfaceType = null; - this.delegateType = null; - } - - /** - * Looks up {@code type} from the whitelist, and returns a matching constructor. - */ - private static PainlessConstructor lookup(PainlessLookup painlessLookup, Class expected, String type) { - // check its really a functional interface - // for e.g. Comparable - PainlessMethod method = painlessLookup.lookupPainlessClass(expected).functionalMethod; - if (method == null) { - throw new IllegalArgumentException("Cannot convert function reference [" + type + "::new] " + - "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"); - } - - // lookup requested constructor - PainlessClass struct = painlessLookup.lookupPainlessClass(painlessLookup.canonicalTypeNameToType(type)); - PainlessConstructor impl = struct.constructors.get(PainlessLookupUtility.buildPainlessConstructorKey(method.typeParameters.size())); - - if (impl == null) { - throw new IllegalArgumentException("Unknown reference [" + type + "::new] matching [" + expected + "]"); - } - - return impl; - } - - /** - * Looks up {@code type::call} from the whitelist, and returns a matching method. - */ - private static PainlessMethod lookup(PainlessLookup painlessLookup, Class expected, - String type, String call, boolean receiverCaptured) { - // check its really a functional interface - // for e.g. Comparable - PainlessMethod method = painlessLookup.lookupPainlessClass(expected).functionalMethod; - if (method == null) { - throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"); - } - - // lookup requested method - PainlessClass struct = painlessLookup.lookupPainlessClass(painlessLookup.canonicalTypeNameToType(type)); - final PainlessMethod impl; - // look for a static impl first - PainlessMethod staticImpl = - struct.staticMethods.get(PainlessLookupUtility.buildPainlessMethodKey(call, method.typeParameters.size())); - if (staticImpl == null) { - // otherwise a virtual impl - final int arity; - if (receiverCaptured) { - // receiver captured - arity = method.typeParameters.size(); - } else { - // receiver passed - arity = method.typeParameters.size() - 1; - } - impl = struct.methods.get(PainlessLookupUtility.buildPainlessMethodKey(call, arity)); - } else { - impl = staticImpl; - } - if (impl == null) { - throw new IllegalArgumentException("Unknown reference [" + type + "::" + call + "] matching " + - "[" + expected + "]"); - } - return impl; + this.delegateMethodType = delegateMethodType; + this.factoryMethodType = factoryMethodType; } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java index 72435562a3b..df5f7966c35 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java @@ -56,6 +56,7 @@ import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_EXPLICIT; import static org.elasticsearch.painless.WriterConstants.DEF_TO_SHORT_IMPLICIT; import static org.elasticsearch.painless.WriterConstants.DEF_UTIL_TYPE; import static org.elasticsearch.painless.WriterConstants.INDY_STRING_CONCAT_BOOTSTRAP_HANDLE; +import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; import static org.elasticsearch.painless.WriterConstants.MAX_INDY_STRING_CONCAT_ARGS; import static org.elasticsearch.painless.WriterConstants.PAINLESS_ERROR_TYPE; import static org.elasticsearch.painless.WriterConstants.STRINGBUILDER_APPEND_BOOLEAN; @@ -439,4 +440,18 @@ public final class MethodWriter extends GeneratorAdapter { invokeVirtual(type, method); } } + + public void invokeLambdaCall(FunctionRef functionRef) { + invokeDynamic( + functionRef.interfaceMethodName, + functionRef.factoryMethodType.toMethodDescriptorString(), + LAMBDA_BOOTSTRAP_HANDLE, + Type.getMethodType(functionRef.interfaceMethodType.toMethodDescriptorString()), + functionRef.delegateClassName, + functionRef.delegateInvokeType, + functionRef.delegateMethodName, + Type.getMethodType(functionRef.delegateMethodType.toMethodDescriptorString()), + functionRef.isDelegateInterface ? 1 : 0 + ); + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java index 835bfb5c505..50bb79dcfbd 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java @@ -35,13 +35,13 @@ public final class PainlessClass { public final Map getterMethodHandles; public final Map setterMethodHandles; - public final PainlessMethod functionalMethod; + public final PainlessMethod functionalInterfaceMethod; PainlessClass(Map constructors, Map staticMethods, Map methods, Map staticFields, Map fields, Map getterMethodHandles, Map setterMethodHandles, - PainlessMethod functionalMethod) { + PainlessMethod functionalInterfaceMethod) { this.constructors = Collections.unmodifiableMap(constructors); @@ -54,6 +54,6 @@ public final class PainlessClass { this.getterMethodHandles = Collections.unmodifiableMap(getterMethodHandles); this.setterMethodHandles = Collections.unmodifiableMap(setterMethodHandles); - this.functionalMethod = functionalMethod; + this.functionalInterfaceMethod = functionalInterfaceMethod; } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java index 866f711ba0f..a61215e9ed7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClassBuilder.java @@ -35,7 +35,7 @@ final class PainlessClassBuilder { final Map getterMethodHandles; final Map setterMethodHandles; - PainlessMethod functionalMethod; + PainlessMethod functionalInterfaceMethod; PainlessClassBuilder() { constructors = new HashMap<>(); @@ -49,11 +49,11 @@ final class PainlessClassBuilder { getterMethodHandles = new HashMap<>(); setterMethodHandles = new HashMap<>(); - functionalMethod = null; + functionalInterfaceMethod = null; } PainlessClass build() { return new PainlessClass(constructors, staticMethods, methods, staticFields, fields, - getterMethodHandles, setterMethodHandles, functionalMethod); + getterMethodHandles, setterMethodHandles, functionalInterfaceMethod); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 786248f7269..adaf45aaa0b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -62,6 +62,14 @@ public final class PainlessLookup { return classesToPainlessClasses.get(targetClass); } + public PainlessConstructor lookupPainlessConstructor(String targetClassName, int constructorArity) { + Objects.requireNonNull(targetClassName); + + Class targetClass = canonicalTypeNameToType(targetClassName); + + return lookupPainlessConstructor(targetClass, constructorArity); + } + public PainlessConstructor lookupPainlessConstructor(Class targetClass, int constructorArity) { Objects.requireNonNull(targetClass); @@ -83,6 +91,14 @@ public final class PainlessLookup { return painlessConstructor; } + public PainlessMethod lookupPainlessMethod(String targetClassName, boolean isStatic, String methodName, int methodArity) { + Objects.requireNonNull(targetClassName); + + Class targetClass = canonicalTypeNameToType(targetClassName); + + return lookupPainlessMethod(targetClass, isStatic, methodName, methodArity); + } + public PainlessMethod lookupPainlessMethod(Class targetClass, boolean isStatic, String methodName, int methodArity) { Objects.requireNonNull(targetClass); Objects.requireNonNull(methodName); @@ -111,6 +127,14 @@ public final class PainlessLookup { return painlessMethod; } + public PainlessField lookupPainlessField(String targetClassName, boolean isStatic, String fieldName) { + Objects.requireNonNull(targetClassName); + + Class targetClass = canonicalTypeNameToType(targetClassName); + + return lookupPainlessField(targetClass, isStatic, fieldName); + } + public PainlessField lookupPainlessField(Class targetClass, boolean isStatic, String fieldName) { Objects.requireNonNull(targetClass); Objects.requireNonNull(fieldName); @@ -134,4 +158,20 @@ public final class PainlessLookup { return painlessField; } + + public PainlessMethod lookupFunctionalInterfacePainlessMethod(Class targetClass) { + PainlessClass targetPainlessClass = classesToPainlessClasses.get(targetClass); + + if (targetPainlessClass == null) { + throw new IllegalArgumentException("target class [" + typeToCanonicalTypeName(targetClass) + "] not found"); + } + + PainlessMethod functionalInterfacePainlessMethod = targetPainlessClass.functionalInterfaceMethod; + + if (functionalInterfacePainlessMethod == null) { + throw new IllegalArgumentException("target class [" + typeToCanonicalTypeName(targetClass) + "] is not a functional interface"); + } + + return functionalInterfacePainlessMethod; + } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index 799650c2c5d..45a5e188db3 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -875,7 +875,7 @@ public final class PainlessLookupBuilder { } else if (javaMethods.size() == 1) { java.lang.reflect.Method javaMethod = javaMethods.get(0); String painlessMethodKey = buildPainlessMethodKey(javaMethod.getName(), javaMethod.getParameterCount()); - painlessClassBuilder.functionalMethod = painlessClassBuilder.methods.get(painlessMethodKey); + painlessClassBuilder.functionalInterfaceMethod = painlessClassBuilder.methods.get(painlessMethodKey); } } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java index e78b3c67210..a649fa7611c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java @@ -19,7 +19,6 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Globals; @@ -35,8 +34,6 @@ import org.objectweb.asm.Type; import java.util.Objects; import java.util.Set; -import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; - /** * Represents a capturing function reference. */ @@ -76,23 +73,8 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda defPointer = null; // static case if (captured.clazz != def.class) { - try { - ref = FunctionRef.resolveFromLookup(locals.getPainlessLookup(), expected, - PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1); - - // check casts between the interface method and the delegate method are legal - for (int i = 0; i < ref.interfaceMethod.typeParameters.size(); ++i) { - Class from = ref.interfaceMethod.typeParameters.get(i); - Class to = ref.delegateTypeParameters.get(i); - AnalyzerCaster.getLegalCast(location, from, to, false, true); - } - - if (ref.interfaceMethod.returnType != void.class) { - AnalyzerCaster.getLegalCast(location, ref.delegateReturnType, ref.interfaceMethod.returnType, false, true); - } - } catch (IllegalArgumentException e) { - throw createError(e); - } + ref = FunctionRef.create(locals.getPainlessLookup(), locals.getMethods(), location, + expected, PainlessLookupUtility.typeToCanonicalTypeName(captured.clazz), call, 1); } actual = expected; } @@ -114,17 +96,7 @@ public final class ECapturingFunctionRef extends AExpression implements ILambda } else { // typed interface, typed implementation writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot()); - writer.invokeDynamic( - ref.interfaceMethodName, - ref.factoryDescriptor, - LAMBDA_BOOTSTRAP_HANDLE, - ref.interfaceType, - ref.delegateClassName, - ref.delegateInvokeType, - ref.delegateMethodName, - ref.delegateType, - ref.isDelegateInterface ? 1 : 0 - ); + writer.invokeLambdaCall(ref); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java index 692581d8118..c97cc66c7c7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java @@ -19,22 +19,16 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; -import org.elasticsearch.painless.Locals.LocalMethod; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; -import org.elasticsearch.painless.lookup.PainlessLookupUtility; -import org.elasticsearch.painless.lookup.PainlessMethod; import org.objectweb.asm.Type; import java.util.Objects; import java.util.Set; -import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; - /** * Represents a function reference. */ @@ -63,39 +57,7 @@ public final class EFunctionRef extends AExpression implements ILambda { defPointer = "S" + type + "." + call + ",0"; } else { defPointer = null; - try { - if ("this".equals(type)) { - // user's own function - PainlessMethod interfaceMethod = locals.getPainlessLookup().lookupPainlessClass(expected).functionalMethod; - if (interfaceMethod == null) { - throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface"); - } - LocalMethod delegateMethod = locals.getMethod(call, interfaceMethod.typeParameters.size()); - if (delegateMethod == null) { - throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], function not found"); - } - ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0); - - // check casts between the interface method and the delegate method are legal - for (int i = 0; i < interfaceMethod.typeParameters.size(); ++i) { - Class from = interfaceMethod.typeParameters.get(i); - Class to = delegateMethod.typeParameters.get(i); - AnalyzerCaster.getLegalCast(location, from, to, false, true); - } - - if (interfaceMethod.returnType != void.class) { - AnalyzerCaster.getLegalCast(location, delegateMethod.returnType, interfaceMethod.returnType, false, true); - } - } else { - // whitelist lookup - ref = FunctionRef.resolveFromLookup(locals.getPainlessLookup(), expected, type, call, 0); - } - - } catch (IllegalArgumentException e) { - throw createError(e); - } + ref = FunctionRef.create(locals.getPainlessLookup(), locals.getMethods(), location, expected, type, call, 0); actual = expected; } } @@ -104,17 +66,7 @@ public final class EFunctionRef extends AExpression implements ILambda { void write(MethodWriter writer, Globals globals) { if (ref != null) { writer.writeDebugInfo(location); - writer.invokeDynamic( - ref.interfaceMethodName, - ref.factoryDescriptor, - LAMBDA_BOOTSTRAP_HANDLE, - ref.interfaceType, - ref.delegateClassName, - ref.delegateInvokeType, - ref.delegateMethodName, - ref.delegateType, - ref.isDelegateInterface ? 1 : 0 - ); + writer.invokeLambdaCall(ref); } else { // TODO: don't do this: its just to cutover :) writer.push((String)null); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java index ecd11ce1bf7..af906416ca7 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java @@ -19,11 +19,9 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; -import org.elasticsearch.painless.Locals.LocalMethod; import org.elasticsearch.painless.Locals.Variable; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; @@ -40,8 +38,6 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import static org.elasticsearch.painless.WriterConstants.LAMBDA_BOOTSTRAP_HANDLE; - /** * Lambda expression node. *

@@ -122,7 +118,7 @@ public final class ELambda extends AExpression implements ILambda { } else { // we know the method statically, infer return type and any unknown/def types - interfaceMethod = locals.getPainlessLookup().lookupPainlessClass(expected).functionalMethod; + interfaceMethod = locals.getPainlessLookup().lookupFunctionalInterfacePainlessMethod(expected); if (interfaceMethod == null) { throw createError(new IllegalArgumentException("Cannot pass lambda to " + "[" + PainlessLookupUtility.typeToCanonicalTypeName(expected) + "], not a functional interface")); @@ -184,25 +180,8 @@ public final class ELambda extends AExpression implements ILambda { defPointer = "Sthis." + name + "," + captures.size(); } else { defPointer = null; - try { - LocalMethod localMethod = - new LocalMethod(desugared.name, desugared.returnType, desugared.typeParameters, desugared.methodType); - ref = new FunctionRef(expected, interfaceMethod, localMethod, captures.size()); - } catch (IllegalArgumentException e) { - throw createError(e); - } - - // check casts between the interface method and the delegate method are legal - for (int i = 0; i < interfaceMethod.typeParameters.size(); ++i) { - Class from = interfaceMethod.typeParameters.get(i); - Class to = desugared.parameters.get(i + captures.size()).clazz; - AnalyzerCaster.getLegalCast(location, from, to, false, true); - } - - if (interfaceMethod.returnType != void.class) { - AnalyzerCaster.getLegalCast(location, desugared.returnType, interfaceMethod.returnType, false, true); - } - + ref = FunctionRef.create( + locals.getPainlessLookup(), locals.getMethods(), location, expected, "this", desugared.name, captures.size()); actual = expected; } } @@ -218,17 +197,7 @@ public final class ELambda extends AExpression implements ILambda { writer.visitVarInsn(MethodWriter.getType(capture.clazz).getOpcode(Opcodes.ILOAD), capture.getSlot()); } - writer.invokeDynamic( - ref.interfaceMethodName, - ref.factoryDescriptor, - LAMBDA_BOOTSTRAP_HANDLE, - ref.interfaceType, - ref.delegateClassName, - ref.delegateInvokeType, - ref.delegateMethodName, - ref.delegateType, - ref.isDelegateInterface ? 1 : 0 - ); + writer.invokeLambdaCall(ref); } else { // placeholder writer.push((String)null); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java index fd47db6b83d..5829593f524 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java @@ -27,7 +27,6 @@ import java.lang.invoke.LambdaConversionException; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.startsWith; public class FunctionRefTests extends ScriptTestCase { @@ -193,14 +192,15 @@ public class FunctionRefTests extends ScriptTestCase { Exception e = expectScriptThrows(IllegalArgumentException.class, () -> { exec("List l = [2, 1]; l.sort(Integer::bogus); return l.get(0);"); }); - assertThat(e.getMessage(), startsWith("Unknown reference")); + assertThat(e.getMessage(), containsString("function reference [Integer::bogus/2] matching [java.util.Comparator")); } public void testQualifiedMethodMissing() { Exception e = expectScriptThrows(IllegalArgumentException.class, () -> { exec("List l = [2, 1]; l.sort(org.joda.time.ReadableDateTime::bogus); return l.get(0);", false); }); - assertThat(e.getMessage(), startsWith("Unknown reference")); + assertThat(e.getMessage(), + containsString("function reference [org.joda.time.ReadableDateTime::bogus/2] matching [java.util.Comparator")); } public void testClassMissing() { @@ -223,11 +223,12 @@ public class FunctionRefTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("List l = new ArrayList(); l.add(2); l.add(1); l.add(Integer::bogus); return l.get(0);"); }); - assertThat(expected.getMessage(), containsString("Cannot convert function reference")); + assertThat(expected.getMessage(), + containsString("cannot convert function reference [Integer::bogus] to a non-functional interface [def]")); } public void testIncompatible() { - expectScriptThrows(BootstrapMethodError.class, () -> { + expectScriptThrows(ClassCastException.class, () -> { exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::startsWith); return l.get(0);"); }); } @@ -236,28 +237,32 @@ public class FunctionRefTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("Optional.empty().orElseGet(String::startsWith);"); }); - assertThat(expected.getMessage(), containsString("Unknown reference")); + assertThat(expected.getMessage(), + containsString("function reference [String::startsWith/0] matching [java.util.function.Supplier")); } public void testWrongArityNotEnough() { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("List l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);"); }); - assertTrue(expected.getMessage().contains("Unknown reference")); + assertThat(expected.getMessage(), containsString( + "function reference [String::isEmpty/2] matching [java.util.Comparator")); } public void testWrongArityDef() { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("def y = Optional.empty(); return y.orElseGet(String::startsWith);"); }); - assertThat(expected.getMessage(), containsString("Unknown reference")); + assertThat(expected.getMessage(), + containsString("function reference [String::startsWith/0] matching [java.util.function.Supplier")); } public void testWrongArityNotEnoughDef() { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("def l = new ArrayList(); l.add(2); l.add(1); l.sort(String::isEmpty);"); }); - assertThat(expected.getMessage(), containsString("Unknown reference")); + assertThat(expected.getMessage(), + containsString("function reference [String::isEmpty/2] matching [java.util.Comparator")); } public void testReturnVoid() { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java index 20e257e5747..1f1a6f95b36 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LambdaTests.java @@ -184,7 +184,7 @@ public class LambdaTests extends ScriptTestCase { IllegalArgumentException expected = expectScriptThrows(IllegalArgumentException.class, () -> { exec("def y = Optional.empty(); return y.orElseGet(x -> x);"); }); - assertTrue(expected.getMessage(), expected.getMessage().contains("Incorrect number of parameters")); + assertTrue(expected.getMessage(), expected.getMessage().contains("due to an incorrect number of arguments")); } public void testWrongArityNotEnough() { @@ -200,7 +200,7 @@ public class LambdaTests extends ScriptTestCase { exec("def l = new ArrayList(); l.add(1); l.add(1); " + "return l.stream().mapToInt(() -> 5).sum();"); }); - assertTrue(expected.getMessage().contains("Incorrect number of parameters")); + assertTrue(expected.getMessage(), expected.getMessage().contains("due to an incorrect number of arguments")); } public void testLambdaInFunction() { From 8bfb0f3f8d9e9032fdc10ce3fcd1473933a59019 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 7 Aug 2018 13:31:00 -0700 Subject: [PATCH 11/20] serialize suggestion responses as named writeables (#30284) Suggestion responses were previously serialized as streamables which made writing suggesters in plugins with custom suggestion response types impossible. This commit makes them serialized as named writeables and provides a facility for registering a reader for suggestion responses when registering a suggester. This also makes Suggestion responses abstract, requiring a suggester implementation to provide its own types. Suggesters which do not need anything additional to what is defined in Suggest.Suggestion should provide a minimal subclass. The existing plugin suggester integration tests are removed and replaced with an equivalent implementation as an example plugin. --- .../client/RestHighLevelClient.java | 9 +- .../release-notes/7.0.0-alpha1.asciidoc | 8 +- .../examples/custom-suggester/build.gradle | 33 ++ .../customsuggester/CustomSuggester.java | 62 ++++ .../CustomSuggesterPlugin.java | 40 ++ .../customsuggester/CustomSuggestion.java | 227 ++++++++++++ .../CustomSuggestionBuilder.java | 143 +++++++ .../CustomSuggestionContext.java | 35 ++ .../CustomSuggesterClientYamlTestSuiteIT.java | 37 ++ .../test/custom-suggester/10_basic.yml | 13 + .../test/custom-suggester/20_suggest.yml | 55 +++ .../elasticsearch/plugins/SearchPlugin.java | 55 ++- .../elasticsearch/search/SearchModule.java | 19 +- .../internal/InternalSearchResponse.java | 4 +- .../search/query/QuerySearchResult.java | 2 +- .../elasticsearch/search/suggest/Suggest.java | 348 +++++++++--------- .../completion/CompletionSuggestion.java | 83 +++-- .../CompletionSuggestionBuilder.java | 4 +- .../suggest/phrase/PhraseSuggester.java | 4 +- .../suggest/phrase/PhraseSuggestion.java | 98 ++++- .../phrase/PhraseSuggestionBuilder.java | 2 +- .../search/suggest/term/TermSuggestion.java | 86 +++-- .../suggest/term/TermSuggestionBuilder.java | 2 +- .../search/SearchModuleTests.java | 104 +++++- .../search/suggest/CustomSuggester.java | 63 ---- .../suggest/CustomSuggesterSearchIT.java | 212 ----------- .../search/suggest/SuggestTests.java | 56 ++- .../search/suggest/SuggestionEntryTests.java | 9 +- .../search/suggest/SuggestionOptionTests.java | 7 +- .../search/suggest/SuggestionTests.java | 11 +- 30 files changed, 1269 insertions(+), 562 deletions(-) create mode 100644 plugins/examples/custom-suggester/build.gradle create mode 100644 plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java create mode 100644 plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java create mode 100644 plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java create mode 100644 plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionBuilder.java create mode 100644 plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestionContext.java create mode 100644 plugins/examples/custom-suggester/src/test/java/org/elasticsearch/example/customsuggester/CustomSuggesterClientYamlTestSuiteIT.java create mode 100644 plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/10_basic.yml create mode 100644 plugins/examples/custom-suggester/src/test/resources/rest-api-spec/test/custom-suggester/20_suggest.yml delete mode 100644 server/src/test/java/org/elasticsearch/search/suggest/CustomSuggester.java delete mode 100644 server/src/test/java/org/elasticsearch/search/suggest/CustomSuggesterSearchIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 816631ff94f..268603d3ce7 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -163,8 +163,11 @@ import org.elasticsearch.search.aggregations.pipeline.derivative.DerivativePipel import org.elasticsearch.search.aggregations.pipeline.derivative.ParsedDerivative; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; +import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder; import org.elasticsearch.search.suggest.phrase.PhraseSuggestion; +import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestion; +import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; import java.io.Closeable; import java.io.IOException; @@ -1141,11 +1144,11 @@ public class RestHighLevelClient implements Closeable { List entries = map.entrySet().stream() .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) .collect(Collectors.toList()); - entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestion.NAME), + entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(TermSuggestionBuilder.SUGGESTION_NAME), (parser, context) -> TermSuggestion.fromXContent(parser, (String)context))); - entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestion.NAME), + entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(PhraseSuggestionBuilder.SUGGESTION_NAME), (parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context))); - entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestion.NAME), + entries.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField(CompletionSuggestionBuilder.SUGGESTION_NAME), (parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context))); return entries; } diff --git a/docs/reference/release-notes/7.0.0-alpha1.asciidoc b/docs/reference/release-notes/7.0.0-alpha1.asciidoc index cf2e1e30be0..c3a03d77f81 100644 --- a/docs/reference/release-notes/7.0.0-alpha1.asciidoc +++ b/docs/reference/release-notes/7.0.0-alpha1.asciidoc @@ -21,4 +21,10 @@ Aggregations:: * The Percentiles and PercentileRanks aggregations now return `null` in the REST response, instead of `NaN`. This makes it consistent with the rest of the aggregations. Note: this only applies to the REST response, the java objects continue to return `NaN` (also - consistent with other aggregations) \ No newline at end of file + consistent with other aggregations) + +Suggesters:: +* Plugins that register suggesters can now define their own types of suggestions and must + explicitly indicate the type of suggestion that they produce. Existing plugins will + require changes to their plugin registration. See the `custom-suggester` example + plugin {pull}30284[#30284] \ No newline at end of file diff --git a/plugins/examples/custom-suggester/build.gradle b/plugins/examples/custom-suggester/build.gradle new file mode 100644 index 00000000000..b36d5cd218d --- /dev/null +++ b/plugins/examples/custom-suggester/build.gradle @@ -0,0 +1,33 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'custom-suggester' + description 'An example plugin showing how to write and register a custom suggester' + classname 'org.elasticsearch.example.customsuggester.CustomSuggesterPlugin' +} + +integTestCluster { + numNodes = 2 +} + +// this plugin has no unit tests, only rest tests +tasks.test.enabled = false \ No newline at end of file diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java new file mode 100644 index 00000000000..b6a5b5e8f84 --- /dev/null +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggester.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.example.customsuggester; + +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.CharsRefBuilder; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.Suggester; + +import java.util.Locale; + +public class CustomSuggester extends Suggester { + + // This is a pretty dumb implementation which returns the original text + fieldName + custom config option + 12 or 123 + @Override + public Suggest.Suggestion> innerExecute( + String name, + CustomSuggestionContext suggestion, + IndexSearcher searcher, + CharsRefBuilder spare) { + + // Get the suggestion context + String text = suggestion.getText().utf8ToString(); + + // create two suggestions with 12 and 123 appended + CustomSuggestion response = new CustomSuggestion(name, suggestion.getSize(), "suggestion-dummy-value"); + + CustomSuggestion.Entry entry = new CustomSuggestion.Entry(new Text(text), 0, text.length(), "entry-dummy-value"); + + String firstOption = + String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "12"); + CustomSuggestion.Entry.Option option12 = new CustomSuggestion.Entry.Option(new Text(firstOption), 0.9f, "option-dummy-value-1"); + entry.addOption(option12); + + String secondOption = + String.format(Locale.ROOT, "%s-%s-%s-%s", text, suggestion.getField(), suggestion.options.get("suffix"), "123"); + CustomSuggestion.Entry.Option option123 = new CustomSuggestion.Entry.Option(new Text(secondOption), 0.8f, "option-dummy-value-2"); + entry.addOption(option123); + + response.addTerm(entry); + + return response; + } +} diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java new file mode 100644 index 00000000000..91ffa672e53 --- /dev/null +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggesterPlugin.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.example.customsuggester; + +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.SearchPlugin; + +import java.util.Collections; +import java.util.List; + +public class CustomSuggesterPlugin extends Plugin implements SearchPlugin { + @Override + public List> getSuggesters() { + return Collections.singletonList( + new SearchPlugin.SuggesterSpec<>( + CustomSuggestionBuilder.SUGGESTION_NAME, + CustomSuggestionBuilder::new, + CustomSuggestionBuilder::fromXContent, + CustomSuggestion::new + ) + ); + } +} diff --git a/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java new file mode 100644 index 00000000000..f7ec27b7af0 --- /dev/null +++ b/plugins/examples/custom-suggester/src/main/java/org/elasticsearch/example/customsuggester/CustomSuggestion.java @@ -0,0 +1,227 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.example.customsuggester; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.suggest.Suggest; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class CustomSuggestion extends Suggest.Suggestion { + + public static final int TYPE = 999; + + public static final ParseField DUMMY = new ParseField("dummy"); + + private String dummy; + + public CustomSuggestion(String name, int size, String dummy) { + super(name, size); + this.dummy = dummy; + } + + public CustomSuggestion(StreamInput in) throws IOException { + super(in); + dummy = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(dummy); + } + + @Override + public String getWriteableName() { + return CustomSuggestionBuilder.SUGGESTION_NAME; + } + + @Override + public int getWriteableType() { + return TYPE; + } + + /** + * A meaningless value used to test that plugin suggesters can add fields to their Suggestion types + * + * This can't be serialized to xcontent because Suggestions appear in xcontent as an array of entries, so there is no place + * to add a custom field. But we can still use a custom field internally and use it to define a Suggestion's behavior + */ + public String getDummy() { + return dummy; + } + + @Override + protected Entry newEntry() { + return new Entry(); + } + + @Override + protected Entry newEntry(StreamInput in) throws IOException { + return new Entry(in); + } + + public static CustomSuggestion fromXContent(XContentParser parser, String name) throws IOException { + CustomSuggestion suggestion = new CustomSuggestion(name, -1, null); + parseEntries(parser, suggestion, Entry::fromXContent); + return suggestion; + } + + public static class Entry extends Suggest.Suggestion.Entry { + + private static final ObjectParser PARSER = new ObjectParser<>("CustomSuggestionEntryParser", true, Entry::new); + + static { + declareCommonFields(PARSER); + PARSER.declareString((entry, dummy) -> entry.dummy = dummy, DUMMY); + PARSER.declareObjectArray(Entry::addOptions, (p, c) -> Option.fromXContent(p), new ParseField(OPTIONS)); + } + + private String dummy; + + public Entry() {} + + public Entry(Text text, int offset, int length, String dummy) { + super(text, offset, length); + this.dummy = dummy; + } + + public Entry(StreamInput in) throws IOException { + super(in); + dummy = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(dummy); + } + + @Override + protected Option newOption() { + return new Option(); + } + + @Override + protected Option newOption(StreamInput in) throws IOException { + return new Option(in); + } + + /* + * the value of dummy will always be the same, so this just tests that we can merge entries with custom fields + */ + @Override + protected void merge(Suggest.Suggestion.Entry