honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Transform] data nanos/date histogram IT (#53654) 

add an integration test for date nanos in combination with date_histogram
This commit is contained in:
Hendrik Muhs 2020-03-17 20:57:54 +01:00
parent 9c0e846db3
commit a6dca577e5
3 changed files with 36 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java
View File

@ -56,6 +56,7 @@ public class TransformPivotRestIT extends TransformRestTestCase {
}
createReviewsIndex();
createReviewsIndexNano();
indicesCreated = true;
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
@ -496,9 +497,17 @@ public class TransformPivotRestIT extends TransformRestTestCase {
}
public void testDateHistogramPivot() throws Exception {
String transformId = "simple_date_histogram_pivot";
String transformIndex = "pivot_reviews_via_date_histogram";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
assertDateHistogramPivot(REVIEWS_INDEX_NAME);
}
public void testDateHistogramPivotNanos() throws Exception {
assertDateHistogramPivot(REVIEWS_DATE_NANO_INDEX_NAME);
}
private void assertDateHistogramPivot(String indexName) throws Exception {
String transformId = "simple_date_histogram_pivot_" + indexName;
String transformIndex = "pivot_reviews_via_date_histogram_" + indexName;
setupDataAccessRole(DATA_ACCESS_ROLE, indexName, transformIndex);
final Request createTransformRequest = createRequestWithAuth(
"PUT",
@ -506,13 +515,7 @@ public class TransformPivotRestIT extends TransformRestTestCase {
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
);
String config = "{"
+ " \"source\": {\"index\":\""
+ REVIEWS_INDEX_NAME
+ "\"},"
+ " \"dest\": {\"index\":\""
+ transformIndex
+ "\"},";
String config = "{" + " \"source\": {\"index\":\"" + indexName + "\"}," + " \"dest\": {\"index\":\"" + transformIndex + "\"},";
config += " \"pivot\": {"
+ " \"group_by\": {"

29
x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java
View File

@ -50,6 +50,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
private static final String BASIC_AUTH_VALUE_SUPER_USER = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);
protected static final String REVIEWS_INDEX_NAME = "reviews";
protected static final String REVIEWS_DATE_NANO_INDEX_NAME = "reviews_nano";
private static boolean useDeprecatedEndpoints;
@ -75,18 +76,20 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
return super.buildClient(settings, hosts);
}
protected void createReviewsIndex(String indexName, int numDocs) throws IOException {
protected void createReviewsIndex(String indexName, int numDocs, String dateType) throws IOException {
int[] distributionTable = { 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 3, 3, 2, 1, 1, 1 };
// create mapping
try (XContentBuilder builder = jsonBuilder()) {
builder.startObject();
{
builder.startObject("mappings")
.startObject("properties")
.startObject("timestamp")
.field("type", "date")
.endObject()
builder.startObject("mappings").startObject("properties").startObject("timestamp").field("type", dateType);
if (dateType.equals("date_nanos")) {
builder.field("format", "strict_date_optional_time_nanos");
}
builder.endObject()
.startObject("user_id")
.field("type", "keyword")
.endObject()
@ -128,7 +131,13 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
int sec = 10 + (i % 49);
String location = (user + 10) + "," + (user + 15);
String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec + "Z";
String date_string = "2017-01-" + day + "T" + hour + ":" + min + ":" + sec;
if (dateType.equals("date_nanos")) {
String randomNanos = "," + randomIntBetween(100000000, 999999999);
date_string += randomNanos;
}
date_string += "Z";
bulk.append("{\"user_id\":\"")
.append("user_")
.append(user)
@ -170,7 +179,7 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
}
protected void createReviewsIndex(String indexName) throws IOException {
createReviewsIndex(indexName, 1000);
createReviewsIndex(indexName, 1000, "date");
}
protected void createPivotReviewsTransform(String transformId, String transformIndex, String query) throws IOException {
@ -182,6 +191,10 @@ public abstract class TransformRestTestCase extends ESRestTestCase {
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, null);
}
protected void createReviewsIndexNano() throws IOException {
createReviewsIndex(REVIEWS_DATE_NANO_INDEX_NAME, 1000, "date_nanos");
}
protected void createContinuousPivotReviewsTransform(String transformId, String transformIndex, String authHeader) throws IOException {
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);

4
x-pack/plugin/transform/qa/single-node-tests/src/test/java/org/elasticsearch/xpack/transform/integration/TransformTaskFailedStateIT.java
View File

@ -64,7 +64,7 @@ public class TransformTaskFailedStateIT extends TransformRestTestCase {
public void testForceStopFailedTransform() throws Exception {
String transformId = "test-force-stop-failed-transform";
createReviewsIndex(REVIEWS_INDEX_NAME, 10);
createReviewsIndex(REVIEWS_INDEX_NAME, 10, "date");
String transformIndex = "failure_pivot_reviews";
createDestinationIndexWithBadMapping(transformIndex);
createContinuousPivotReviewsTransform(transformId, transformIndex, null);
@ -102,7 +102,7 @@ public class TransformTaskFailedStateIT extends TransformRestTestCase {
public void testStartFailedTransform() throws Exception {
String transformId = "test-force-start-failed-transform";
createReviewsIndex(REVIEWS_INDEX_NAME, 10);
createReviewsIndex(REVIEWS_INDEX_NAME, 10, "date");
String transformIndex = "failure_pivot_reviews";
createDestinationIndexWithBadMapping(transformIndex);
createContinuousPivotReviewsTransform(transformId, transformIndex, null);