honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 17:38:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change default backing index naming scheme

Browse Source
This commit is contained in:
Dan Hermann 2020-06-09 09:31:34 -05:00 committed by GitHub
parent e7cc2448d2
commit b501b282f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
34 changed files with 190 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/reference/data-streams.asciidoc
View File

@ -20,7 +20,7 @@ template>> containing:
* A name or wildcard pattern for the data stream in the `index_patterns` property.
* A `data_stream` definition that contains the `timestamp_field` property.
The `timestamp_field` must be the primary timestamp field
The `timestamp_field` must be the primary timestamp field
for the data source. This field must be included in every
document indexed to the data stream.
@ -78,7 +78,7 @@ Response:
"failed" : 0,
"successful" : 1
},
"_index" : "logs-foobar-000001",
"_index" : ".ds-logs-foobar-000001",
"_type" : "_doc",
"_id" : "W0tpsmIBdwcYyG50zbta",
"_version" : 1,

4
docs/reference/indices/get-data-stream.asciidoc
View File

@ -76,11 +76,11 @@ The API returns the following response:
"timestamp_field" : "@timestamp", <2>
"indices" : [ <3>
{
"index_name" : "my-data-stream-000001",
"index_name" : ".ds-my-data-stream-000001",
"index_uuid" : "DXAE-xcCQTKF93bMm9iawA"
},
{
"index_name" : "my-data-stream-000002",
"index_name" : ".ds-my-data-stream-000002",
"index_uuid" : "Wzxq0VhsQKyPxHhaK3WYAg"
}
],

6
docs/reference/indices/rollover-index.asciidoc
View File

@ -256,7 +256,7 @@ POST /my-data-stream/_rollover <2>
--------------------------------------------------
// TEST[continued]
// TEST[setup:huge_twitter]
// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"my-data-stream-000001"}}/]
// TEST[s/# Add > 1000 documents to my-data-stream/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":".ds-my-data-stream-000001"}}/]
<1> Creates a data stream called `my-data-stream` with one initial backing index
named `my-data-stream-000001`.
<2> This request creates a new backing index, `my-data-stream-000002`, and adds
@ -276,8 +276,8 @@ The API returns the following response:
{
"acknowledged": true,
"shards_acknowledged": true,
"old_index": "my-data-stream-000001", <1>
"new_index": "my-data-stream-000002", <2>
"old_index": ".ds-my-data-stream-000001", <1>
"new_index": ".ds-my-data-stream-000002", <2>
"rolled_over": true, <3>
"dry_run": false, <4>
"conditions": { <5>

26
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/10_basic.yml
View File

@ -23,8 +23,8 @@ setup:
---
"Create data stream":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
- do:
indices.create_data_stream:
@ -43,12 +43,12 @@ setup:
- match: { 0.timestamp_field: '@timestamp' }
- match: { 0.generation: 1 }
- length: { 0.indices: 1 }
- match: { 0.indices.0.index_name: 'simple-data-stream1-000001' }
- match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
- match: { 1.name: simple-data-stream2 }
- match: { 1.timestamp_field: '@timestamp2' }
- match: { 0.generation: 1 }
- length: { 1.indices: 1 }
- match: { 1.indices.0.index_name: 'simple-data-stream2-000001' }
- match: { 1.indices.0.index_name: '.ds-simple-data-stream2-000001' }
- do:
index:
@ -64,7 +64,7 @@ setup:
index: simple-data-stream1
body: { query: { match_all: {} } }
- length: { hits.hits: 1 }
- match: { hits.hits.0._index: simple-data-stream1-000001 }
- match: { hits.hits.0._index: .ds-simple-data-stream1-000001 }
- match: { hits.hits.0._source.foo: 'bar' }
- do:
@ -95,8 +95,8 @@ setup:
---
"Get data stream":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
- do:
indices.create_data_stream:
@ -160,8 +160,8 @@ setup:
---
"Delete data stream with backing indices":
- skip:
version: " - 7.8.99"
reason: data streams only supported in 7.9+
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
- do:
indices.create_data_stream:
@ -178,10 +178,10 @@ setup:
- do:
indices.get:
index: ['simple-data-stream1-000001', 'test_index']
index: ['.ds-simple-data-stream1-000001', 'test_index']
- is_true: test_index.settings
- is_true: simple-data-stream1-000001.settings
- is_true: \.ds-simple-data-stream1-000001.settings
- do:
indices.get_data_stream: {}
@ -189,7 +189,7 @@ setup:
- match: { 0.timestamp_field: '@timestamp' }
- match: { 0.generation: 1 }
- length: { 0.indices: 1 }
- match: { 0.indices.0.index_name: 'simple-data-stream1-000001' }
- match: { 0.indices.0.index_name: '.ds-simple-data-stream1-000001' }
- do:
indices.delete_data_stream:
@ -199,4 +199,4 @@ setup:
- do:
catch: missing
indices.get:
index: "simple-data-stream1-000001"
index: ".ds-simple-data-stream1-000001"

8
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/20_unsupported_apis.yml
View File

@ -1,8 +1,8 @@
---
"Test apis that do not supported data streams":
- skip:
version: " - 7.8.99"
reason: "data streams only supported in 7.9+"
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
features: allowed_warnings
- do:
@ -25,14 +25,14 @@
index: logs-foobar
refresh: true
body: { foo: bar }
- match: {_index: logs-foobar-000001}
- match: {_index: .ds-logs-foobar-000001}
- do:
search:
index: logs-foobar
body: { query: { match_all: {} } }
- length: { hits.hits: 1 }
- match: { hits.hits.0._index: logs-foobar-000001 }
- match: { hits.hits.0._index: .ds-logs-foobar-000001 }
- match: { hits.hits.0._source.foo: 'bar' }
- do:

8
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/30_auto_create_data_stream.yml
View File

@ -1,8 +1,8 @@
---
"Put index template":
- skip:
version: " - 7.8.99"
reason: "data streams supported from 7.9"
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
features: allowed_warnings
- do:
@ -34,7 +34,7 @@
index: logs-foobar
body: { query: { match_all: {} } }
- length: { hits.hits: 1 }
- match: { hits.hits.0._index: logs-foobar-000001 }
- match: { hits.hits.0._index: .ds-logs-foobar-000001 }
- match: { hits.hits.0._source.foo: 'bar' }
- do:
@ -43,7 +43,7 @@
- match: { 0.name: logs-foobar }
- match: { 0.timestamp_field: 'timestamp' }
- length: { 0.indices: 1 }
- match: { 0.indices.0.index_name: 'logs-foobar-000001' }
- match: { 0.indices.0.index_name: '.ds-logs-foobar-000001' }
- do:
indices.delete_data_stream:

8
rest-api-spec/src/main/resources/rest-api-spec/test/indices.data_stream/40_supported_apis.yml
View File

@ -31,9 +31,9 @@ teardown:
- do:
indices.get:
index: logs-foobar
- is_true: logs-foobar-000001
- is_true: \.ds-logs-foobar-000001
- is_false: logs-foobar
- match: { logs-foobar-000001.settings.index.number_of_shards: '1' }
- match: { \.ds-logs-foobar-000001.settings.index.number_of_shards: '1' }
---
"Verify get mapping api":
@ -44,5 +44,5 @@ teardown:
- do:
indices.get_mapping:
index: logs-foobar
- is_true: logs-foobar-000001.mappings
- is_false: logs-foobar.mappings
- is_true: \.ds-logs-foobar-000001.mappings
- is_false: \.ds-logs-foobar.mappings

24
rest-api-spec/src/main/resources/rest-api-spec/test/indices.delete/20_backing_indices.yml
View File

@ -15,7 +15,7 @@ setup:
"Delete backing index on data stream":
- skip:
version: " - 7.99.99"
reason: "mute bwc until backported"
reason: "change to 7.8.99 after backport"
- do:
indices.create_data_stream:
@ -27,25 +27,25 @@ setup:
indices.rollover:
alias: "simple-data-stream"
- match: { old_index: simple-data-stream-000001 }
- match: { new_index: simple-data-stream-000002 }
- match: { old_index: .ds-simple-data-stream-000001 }
- match: { new_index: .ds-simple-data-stream-000002 }
- match: { rolled_over: true }
- match: { dry_run: false }
# ensure new index is created
- do:
indices.exists:
index: simple-data-stream-000002
index: .ds-simple-data-stream-000002
- is_true: ''
- do:
indices.delete:
index: simple-data-stream-000001
index: .ds-simple-data-stream-000001
- do:
indices.exists:
index: simple-data-stream-000001
index: .ds-simple-data-stream-000001
- is_false: ''
@ -56,7 +56,7 @@ setup:
- match: { 0.timestamp_field: '@timestamp' }
- match: { 0.generation: 2 }
- length: { 0.indices: 1 }
- match: { 0.indices.0.index_name: 'simple-data-stream-000002' }
- match: { 0.indices.0.index_name: '.ds-simple-data-stream-000002' }
- do:
indices.delete_data_stream:
@ -79,26 +79,26 @@ setup:
indices.rollover:
alias: "simple-data-stream"
- match: { old_index: simple-data-stream-000001 }
- match: { new_index: simple-data-stream-000002 }
- match: { old_index: .ds-simple-data-stream-000001 }
- match: { new_index: .ds-simple-data-stream-000002 }
- match: { rolled_over: true }
- match: { dry_run: false }
# ensure new index is created
- do:
indices.exists:
index: simple-data-stream-000002
index: .ds-simple-data-stream-000002
- is_true: ''
- do:
catch: bad_request
indices.delete:
index: simple-data-stream-000002
index: .ds-simple-data-stream-000002
- do:
indices.exists:
index: simple-data-stream-000002
index: .ds-simple-data-stream-000002
- is_true: ''

12
rest-api-spec/src/main/resources/rest-api-spec/test/indices.get/20_backing_indices.yml
View File

@ -1,8 +1,8 @@
---
"Get backing indices for data stream":
- skip:
version: " - 7.9.99"
reason: "data streams only supported in 7.9+"
version: " - 7.99.99"
reason: "change to 7.8.99 after backport"
features: allowed_warnings
- do:
@ -30,11 +30,11 @@
- do:
indices.get:
index: ['data-stream1-000001', 'test_index']
index: ['.ds-data-stream1-000001', 'test_index']
- is_true: data-stream1-000001.settings
- is_true: data-stream1-000001.data_stream
- match: { data-stream1-000001.data_stream: data-stream1 }
- is_true: \.ds-data-stream1-000001.settings
- is_true: \.ds-data-stream1-000001.data_stream
- match: { \.ds-data-stream1-000001.data_stream: data-stream1 }
- is_true: test_index.settings
- is_false: test_index.data_stream

2
rest-api-spec/src/main/resources/rest-api-spec/test/indices.open/10_basic.yml
View File

@ -144,7 +144,7 @@
- do:
catch: bad_request
indices.close:
index: "simple-data-stream1-000001"
index: ".ds-simple-data-stream1-000001"
- do:
indices.delete_data_stream:

12
rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/50_data_streams.yml
View File

@ -2,7 +2,7 @@
"Roll over a data stream":
- skip:
version: " - 7.99.99"
reason: "mute bwc until backported"
reason: "change to 7.8.99 after backport"
features: allowed_warnings
- do:
@ -25,15 +25,15 @@
indices.rollover:
alias: "data-stream-for-rollover"
- match: { old_index: data-stream-for-rollover-000001 }
- match: { new_index: data-stream-for-rollover-000002 }
- match: { old_index: .ds-data-stream-for-rollover-000001 }
- match: { new_index: .ds-data-stream-for-rollover-000002 }
- match: { rolled_over: true }
- match: { dry_run: false }
# ensure new index is created
- do:
indices.exists:
index: data-stream-for-rollover-000002
index: .ds-data-stream-for-rollover-000002
- is_true: ''
@ -44,8 +44,8 @@
- match: { 0.timestamp_field: '@timestamp' }
- match: { 0.generation: 2 }
- length: { 0.indices: 2 }
- match: { 0.indices.0.index_name: 'data-stream-for-rollover-000001' }
- match: { 0.indices.1.index_name: 'data-stream-for-rollover-000002' }
- match: { 0.indices.0.index_name: '.ds-data-stream-for-rollover-000001' }
- match: { 0.indices.1.index_name: '.ds-data-stream-for-rollover-000002' }
- do:
indices.delete_data_stream:

86
server/src/internalClusterTest/java/org/elasticsearch/indices/DataStreamIT.java
View File

@ -107,20 +107,24 @@ public class DataStreamIT extends ESIntegTestCase {
assertThat(getDataStreamResponse.getDataStreams().get(0).getName(), equalTo("metrics-bar"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField(), equalTo("@timestamp2"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().size(), equalTo(1));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().get(0).getName(), equalTo("metrics-bar-000001"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().get(0).getName(),
equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 1)));
assertThat(getDataStreamResponse.getDataStreams().get(1).getName(), equalTo("metrics-foo"));
assertThat(getDataStreamResponse.getDataStreams().get(1).getTimeStampField(), equalTo("@timestamp1"));
assertThat(getDataStreamResponse.getDataStreams().get(1).getIndices().size(), equalTo(1));
assertThat(getDataStreamResponse.getDataStreams().get(1).getIndices().get(0).getName(), equalTo("metrics-foo-000001"));
assertThat(getDataStreamResponse.getDataStreams().get(1).getIndices().get(0).getName(),
equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 1)));
String backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 1);
GetIndexResponse getIndexResponse =
client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-bar-000001")).actionGet();
assertThat(getIndexResponse.getSettings().get("metrics-bar-000001"), notNullValue());
assertThat(getIndexResponse.getSettings().get("metrics-bar-000001").getAsBoolean("index.hidden", null), is(true));
client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet();
assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue());
assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true));
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-foo-000001")).actionGet();
assertThat(getIndexResponse.getSettings().get("metrics-foo-000001"), notNullValue());
assertThat(getIndexResponse.getSettings().get("metrics-foo-000001").getAsBoolean("index.hidden", null), is(true));
backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 1);
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet();
assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue());
assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true));
int numDocsBar = randomIntBetween(2, 16);
indexDocs("metrics-bar", numDocsBar);
@ -131,20 +135,22 @@ public class DataStreamIT extends ESIntegTestCase {
verifyDocs("metrics-foo", numDocsFoo, 1, 1);
RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-foo", null)).get();
assertThat(rolloverResponse.getNewIndex(), equalTo("metrics-foo-000002"));
assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-foo", 2)));
assertTrue(rolloverResponse.isRolledOver());
rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest("metrics-bar", null)).get();
assertThat(rolloverResponse.getNewIndex(), equalTo("metrics-bar-000002"));
assertThat(rolloverResponse.getNewIndex(), equalTo(DataStream.getDefaultBackingIndexName("metrics-bar", 2)));
assertTrue(rolloverResponse.isRolledOver());
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-foo-000002")).actionGet();
assertThat(getIndexResponse.getSettings().get("metrics-foo-000002"), notNullValue());
assertThat(getIndexResponse.getSettings().get("metrics-foo-000002").getAsBoolean("index.hidden", null), is(true));
backingIndex = DataStream.getDefaultBackingIndexName("metrics-foo", 2);
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet();
assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue());
assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true));
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-bar-000002")).actionGet();
assertThat(getIndexResponse.getSettings().get("metrics-bar-000002"), notNullValue());
assertThat(getIndexResponse.getSettings().get("metrics-bar-000002").getAsBoolean("index.hidden", null), is(true));
backingIndex = DataStream.getDefaultBackingIndexName("metrics-bar", 2);
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet();
assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue());
assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true));
int numDocsBar2 = randomIntBetween(2, 16);
indexDocs("metrics-bar", numDocsBar2);
@ -160,13 +166,17 @@ public class DataStreamIT extends ESIntegTestCase {
assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0));
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-bar-000001")).actionGet());
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(
DataStream.getDefaultBackingIndexName("metrics-bar", 1))).actionGet());
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-bar-000002")).actionGet());
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(
DataStream.getDefaultBackingIndexName("metrics-bar", 2))).actionGet());
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-foo-000001")).actionGet());
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(
DataStream.getDefaultBackingIndexName("metrics-foo", 1))).actionGet());
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices("metrics-foo-000002")).actionGet());
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(
DataStream.getDefaultBackingIndexName("metrics-foo", 2))).actionGet());
}
public void testOtherWriteOps() throws Exception {
@ -207,14 +217,14 @@ public class DataStreamIT extends ESIntegTestCase {
IndexRequest indexRequest = new IndexRequest(dataStreamName).source("{}", XContentType.JSON)
.opType(DocWriteRequest.OpType.CREATE);
IndexResponse indexResponse = client().index(indexRequest).actionGet();
assertThat(indexResponse.getIndex(), equalTo(DataStream.getBackingIndexName(dataStreamName, 1)));
assertThat(indexResponse.getIndex(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1)));
}
{
BulkRequest bulkRequest = new BulkRequest()
.add(new IndexRequest(dataStreamName).source("{}", XContentType.JSON)
.opType(DocWriteRequest.OpType.CREATE));
BulkResponse bulkItemResponses = client().bulk(bulkRequest).actionGet();
assertThat(bulkItemResponses.getItems()[0].getIndex(), equalTo(DataStream.getBackingIndexName(dataStreamName, 1)));
assertThat(bulkItemResponses.getItems()[0].getIndex(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1)));
}
}
@ -248,30 +258,32 @@ public class DataStreamIT extends ESIntegTestCase {
indexDocs(dataStreamName, numDocs);
verifyDocs(dataStreamName, numDocs, 1, 1);
String backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request("*");
GetDataStreamAction.Response getDataStreamResponse = client().admin().indices().getDataStreams(getDataStreamRequest).actionGet();
assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1));
assertThat(getDataStreamResponse.getDataStreams().get(0).getName(), equalTo(dataStreamName));
assertThat(getDataStreamResponse.getDataStreams().get(0).getTimeStampField(), equalTo("@timestamp"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().size(), equalTo(1));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().get(0).getName(), equalTo(dataStreamName + "-000001"));
assertThat(getDataStreamResponse.getDataStreams().get(0).getIndices().get(0).getName(), equalTo(backingIndex));
GetIndexResponse getIndexResponse =
client().admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName)).actionGet();
assertThat(getIndexResponse.getSettings().get(dataStreamName + "-000001"), notNullValue());
assertThat(getIndexResponse.getSettings().get(dataStreamName + "-000001").getAsBoolean("index.hidden", null), is(true));
assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue());
assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true));
assertThat(ObjectPath.eval("properties.baz_field.type",
getIndexResponse.mappings().get(dataStreamName + "-000001").get("_doc").getSourceAsMap()), equalTo("keyword"));
getIndexResponse.mappings().get(backingIndex).get("_doc").getSourceAsMap()), equalTo("keyword"));
backingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
RolloverResponse rolloverResponse = client().admin().indices().rolloverIndex(new RolloverRequest(dataStreamName, null)).get();
assertThat(rolloverResponse.getNewIndex(), equalTo(dataStreamName + "-000002"));
assertThat(rolloverResponse.getNewIndex(), equalTo(backingIndex));
assertTrue(rolloverResponse.isRolledOver());
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName + "-000002")).actionGet();
assertThat(getIndexResponse.getSettings().get(dataStreamName + "-000002"), notNullValue());
assertThat(getIndexResponse.getSettings().get(dataStreamName + "-000002").getAsBoolean("index.hidden", null), is(true));
getIndexResponse = client().admin().indices().getIndex(new GetIndexRequest().indices(backingIndex)).actionGet();
assertThat(getIndexResponse.getSettings().get(backingIndex), notNullValue());
assertThat(getIndexResponse.getSettings().get(backingIndex).getAsBoolean("index.hidden", null), is(true));
assertThat(ObjectPath.eval("properties.baz_field.type",
getIndexResponse.mappings().get(dataStreamName + "-000002").get("_doc").getSourceAsMap()), equalTo("keyword"));
getIndexResponse.mappings().get(backingIndex).get("_doc").getSourceAsMap()), equalTo("keyword"));
int numDocs2 = randomIntBetween(2, 16);
indexDocs(dataStreamName, numDocs2);
@ -283,9 +295,11 @@ public class DataStreamIT extends ESIntegTestCase {
assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(0));
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName + "-000001")).actionGet());
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(
DataStream.getDefaultBackingIndexName(dataStreamName, 1))).actionGet());
expectThrows(IndexNotFoundException.class,
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(dataStreamName + "-000002")).actionGet());
() -> client().admin().indices().getIndex(new GetIndexRequest().indices(
DataStream.getDefaultBackingIndexName(dataStreamName, 2))).actionGet());
}
public void testDataStreamsResolvability() throws Exception {
@ -392,10 +406,12 @@ public class DataStreamIT extends ESIntegTestCase {
}
BulkResponse bulkResponse = client().bulk(bulkRequest).actionGet();
assertThat(bulkResponse.getItems().length, equalTo(numDocs));
String backingIndex = DataStream.getDefaultBackingIndexName(dataStream, 1);
String backingIndexPrefix = backingIndex.substring(0, backingIndex.length() - 3);
for (BulkItemResponse itemResponse : bulkResponse) {
assertThat(itemResponse.getFailureMessage(), nullValue());
assertThat(itemResponse.status(), equalTo(RestStatus.CREATED));
assertThat(itemResponse.getIndex(), startsWith(dataStream + "-00000"));
assertThat(itemResponse.getIndex(), startsWith(backingIndexPrefix));
}
client().admin().indices().refresh(new RefreshRequest(dataStream)).actionGet();
}
@ -408,7 +424,7 @@ public class DataStreamIT extends ESIntegTestCase {
List<String> expectedIndices = new ArrayList<>();
for (long k = minGeneration; k <= maxGeneration; k++) {
expectedIndices.add(DataStream.getBackingIndexName(dataStream, k));
expectedIndices.add(DataStream.getDefaultBackingIndexName(dataStream, k));
}
Arrays.stream(searchResponse.getHits().getHits()).forEach(hit -> {
assertTrue(expectedIndices.contains(hit.getIndex()));

2
server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java
View File

@ -147,7 +147,7 @@ public class MetadataRolloverService {
final DataStream ds = dataStream.getDataStream();
final IndexMetadata originalWriteIndex = dataStream.getWriteIndex();
final String newWriteIndexName = DataStream.getBackingIndexName(ds.getName(), ds.getGeneration() + 1);
final String newWriteIndexName = DataStream.getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1);
CreateIndexClusterStateUpdateRequest createIndexClusterStateRequest =
prepareDataStreamCreateIndexRequest(dataStreamName, newWriteIndexName, createIndexRequest);

14
server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
View File

@ -37,6 +37,8 @@ import java.util.Objects;
public final class DataStream extends AbstractDiffable<DataStream> implements ToXContentObject {
public static final String BACKING_INDEX_PREFIX = ".ds-";
private final String name;
private final String timeStampField;
private final List<Index> indices;
@ -48,7 +50,7 @@ public final class DataStream extends AbstractDiffable<DataStream> implements To
this.indices = indices;
this.generation = generation;
assert indices.size() > 0;
assert indices.get(indices.size() - 1).getName().equals(getBackingIndexName(name, generation));
assert indices.get(indices.size() - 1).getName().equals(getDefaultBackingIndexName(name, generation));
}
public DataStream(String name, String timeStampField, List<Index> indices) {
@ -76,11 +78,11 @@ public final class DataStream extends AbstractDiffable<DataStream> implements To
* the updated list of backing indices and incremented generation.
*
* @param newWriteIndex the new write backing index. Must conform to the naming convention for
* backing indices on data streams. See {@link #getBackingIndexName}.
* backing indices on data streams. See {@link #getDefaultBackingIndexName}.
* @return new {@code DataStream} instance with the rollover operation applied
*/
public DataStream rollover(Index newWriteIndex) {
assert newWriteIndex.getName().equals(getBackingIndexName(name, generation + 1));
assert newWriteIndex.getName().equals(getDefaultBackingIndexName(name, generation + 1));
List<Index> backingIndices = new ArrayList<>(indices);
backingIndices.add(newWriteIndex);
return new DataStream(name, timeStampField, backingIndices, generation + 1);
@ -101,15 +103,15 @@ public final class DataStream extends AbstractDiffable<DataStream> implements To
}
/**
* Generates the name of the index that conforms to the naming convention for backing indices
* Generates the name of the index that conforms to the default naming convention for backing indices
* on data streams given the specified data stream name and generation.
*
* @param dataStreamName name of the data stream
* @param generation generation of the data stream
* @return backing index name
*/
public static String getBackingIndexName(String dataStreamName, long generation) {
return String.format(Locale.ROOT, "%s-%06d", dataStreamName, generation);
public static String getDefaultBackingIndexName(String dataStreamName, long generation) {
return String.format(Locale.ROOT, BACKING_INDEX_PREFIX + "%s-%06d", dataStreamName, generation);
}
public DataStream(StreamInput in) throws IOException {

4
server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java
View File

@ -31,7 +31,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
import static org.elasticsearch.cluster.metadata.DataStream.getBackingIndexName;
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_HIDDEN_SETTING;
import static org.elasticsearch.common.collect.List.copyOf;
@ -294,7 +294,7 @@ public interface IndexAbstraction {
this.dataStream = dataStream;
this.dataStreamIndices = copyOf(dataStreamIndices);
this.writeIndex = dataStreamIndices.get(dataStreamIndices.size() - 1);
assert writeIndex.getIndex().getName().equals(getBackingIndexName(dataStream.getName(), dataStream.getGeneration()));
assert writeIndex.getIndex().getName().equals(getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration()));
}
@Override

4
server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
View File

@ -1714,8 +1714,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
/**
* Parses the number from the rolled over index name. It also supports the date-math format (ie. index name is wrapped in &lt; and &gt;)
* Eg.
* - For "logs-000002" it'll return 2
* E.g.
* - For ".ds-logs-000002" it will return 2
* - For "&lt;logs-{now/d}-3&gt;" it'll return 3
* @throws IllegalArgumentException if the index doesn't contain a "-" separator or if the last token after the separator is not a
* number

11
server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
View File

@ -1531,18 +1531,19 @@ public class Metadata implements Iterable<IndexMetadata>, Diffable<Metadata>, To
/**
* Validates there isn't any index with a name that would clash with the future backing indices of the existing data streams.
*
* For eg. if data stream `foo` has backing indices [`foo-000001`, `foo-000002`] and the indices lookup contains indices
* `foo-000001`, `foo-000002` and `foo-000006` this will throw an IllegalStateException (as attempting to rollover the `foo` data
* stream from generation 5 to 6 will not be possible)
* E.g., if data stream `foo` has backing indices [`.ds-foo-000001`, `.ds-foo-000002`] and the indices lookup contains indices
* `.ds-foo-000001`, `.ds-foo-000002` and `.ds-foo-000006` this will throw an IllegalStateException (as attempting to rollover the
* `foo` data stream from generation 5 to 6 will not be possible)
*
* @param indicesLookup the indices in the system (this includes the data streams backing indices)
* @param indicesLookup the indices in the system (this includes the data stream backing indices)
* @param dsMetadata the data streams in the system
*/
static void validateDataStreams(SortedMap<String, IndexAbstraction> indicesLookup, @Nullable DataStreamMetadata dsMetadata) {
if (dsMetadata != null) {
for (DataStream ds : dsMetadata.dataStreams().values()) {
Map<String, IndexAbstraction> conflicts =
indicesLookup.subMap(ds.getName() + "-", ds.getName() + ".") // '.' is the char after '-'
indicesLookup.subMap(DataStream.BACKING_INDEX_PREFIX + ds.getName() + "-",
DataStream.BACKING_INDEX_PREFIX + ds.getName() + ".") // '.' is the char after '-'
.entrySet().stream()
.filter(entry -> {
if (entry.getValue().getType() != IndexAbstraction.Type.CONCRETE_INDEX) {

2
server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java
View File

@ -130,7 +130,7 @@ public class MetadataCreateDataStreamService {
ComposableIndexTemplate template = lookupTemplateForDataStream(request.name, currentState.metadata());
String firstBackingIndexName = DataStream.getBackingIndexName(request.name, 1);
String firstBackingIndexName = DataStream.getDefaultBackingIndexName(request.name, 1);
CreateIndexClusterStateUpdateRequest createIndexRequest =
new CreateIndexClusterStateUpdateRequest("initialize_data_stream", firstBackingIndexName, firstBackingIndexName)
.dataStreamName(request.name)

2
server/src/test/java/org/elasticsearch/action/admin/indices/datastream/DeleteDataStreamRequestTests.java
View File

@ -126,7 +126,7 @@ public class DeleteDataStreamRequestTests extends AbstractWireSerializingTestCas
for (Tuple<String, Integer> dsTuple : dataStreams) {
List<IndexMetadata> backingIndices = new ArrayList<>();
for (int backingIndexNumber = 1; backingIndexNumber <= dsTuple.v2(); backingIndexNumber++) {
backingIndices.add(createIndexMetadata(DataStream.getBackingIndexName(dsTuple.v1(), backingIndexNumber), true));
backingIndices.add(createIndexMetadata(DataStream.getDefaultBackingIndexName(dsTuple.v1(), backingIndexNumber), true));
}
allIndices.addAll(backingIndices);

6
server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
View File

@ -315,7 +315,7 @@ public class MetadataRolloverServiceTests extends ESTestCase {
public void testCreateIndexRequestForDataStream() {
DataStream dataStream = DataStreamTests.randomInstance();
final String newWriteIndexName = DataStream.getBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
final String newWriteIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
final RolloverRequest rolloverRequest = new RolloverRequest(dataStream.getName(), randomAlphaOfLength(10));
final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE;
rolloverRequest.getCreateIndexRequest().waitForActiveShards(activeShardCount);
@ -560,8 +560,8 @@ public class MetadataRolloverServiceTests extends ESTestCase {
randomBoolean());
long after = testThreadPool.absoluteTimeInMillis();
String sourceIndexName = DataStream.getBackingIndexName(dataStream.getName(), dataStream.getGeneration());
String newIndexName = DataStream.getBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
String sourceIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration());
String newIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1);
assertEquals(sourceIndexName, rolloverResult.sourceIndexName);
assertEquals(newIndexName, rolloverResult.rolloverIndexName);
Metadata rolloverMetadata = rolloverResult.clusterState.metadata();

18
server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java
View File

@ -29,7 +29,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import static org.elasticsearch.cluster.metadata.DataStream.getBackingIndexName;
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
import static org.hamcrest.Matchers.equalTo;
public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
@ -47,7 +47,7 @@ public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
List<Index> indices = randomIndexInstances();
long generation = indices.size() + randomLongBetween(1, 128);
String dataStreamName = randomAlphaOfLength(10).toLowerCase(Locale.ROOT);
indices.add(new Index(getBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random())));
indices.add(new Index(getDefaultBackingIndexName(dataStreamName, generation), UUIDs.randomBase64UUID(random())));
return new DataStream(dataStreamName, randomAlphaOfLength(10), indices, generation);
}
@ -68,7 +68,7 @@ public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
public void testRollover() {
DataStream ds = randomInstance();
Index newWriteIndex = new Index(getBackingIndexName(ds.getName(), ds.getGeneration() + 1), UUIDs.randomBase64UUID(random()));
Index newWriteIndex = new Index(getDefaultBackingIndexName(ds.getName(), ds.getGeneration() + 1), UUIDs.randomBase64UUID(random()));
DataStream rolledDs = ds.rollover(newWriteIndex);
assertThat(rolledDs.getName(), equalTo(ds.getName()));
@ -86,7 +86,7 @@ public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
List<Index> indices = new ArrayList<>(numBackingIndices);
for (int k = 1; k <= numBackingIndices; k++) {
indices.add(new Index(DataStream.getBackingIndexName(dataStreamName, k), UUIDs.randomBase64UUID(random())));
indices.add(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, k), UUIDs.randomBase64UUID(random())));
}
DataStream original = new DataStream(dataStreamName, "@timestamp", indices);
DataStream updated = original.removeBackingIndex(indices.get(indexToRemove - 1));
@ -98,4 +98,14 @@ public class DataStreamTests extends AbstractSerializingTestCase<DataStream> {
assertThat(updated.getIndices().get(k), equalTo(original.getIndices().get(k < (indexToRemove - 1) ? k : k + 1)));
}
}
public void testDefaultBackingIndexName() {
// this test does little more than flag that changing the default naming convention for backing indices
// will also require changing a lot of hard-coded values in REST tests and docs
long backingIndexNum = randomLongBetween(1, 1000001);
String dataStreamName = randomAlphaOfLength(6);
String defaultBackingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, backingIndexNum);
String expectedBackingIndexName = String.format(Locale.ROOT, ".ds-%s-%06d", dataStreamName, backingIndexNum);
assertThat(defaultBackingIndexName, equalTo(expectedBackingIndexName));
}
}

2
server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java
View File

@ -371,7 +371,7 @@ public class IndexMetadataTests extends ESTestCase {
}
public void testParseIndexNameReturnsCounter() {
assertThat(parseIndexNameCounter("logs-000003"), is(3));
assertThat(parseIndexNameCounter(".ds-logs-000003"), is(3));
assertThat(parseIndexNameCounter("shrink-logs-000003"), is(3));
}

22
server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java
View File

@ -1777,8 +1777,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "my-data-stream");
assertThat(result.length, equalTo(2));
assertThat(result[0].getName(), equalTo(dataStreamName + "-000001"));
assertThat(result[1].getName(), equalTo(dataStreamName + "-000002"));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 1)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2)));
}
{
// Ignore data streams
@ -1809,7 +1809,7 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Index result = indexNameExpressionResolver.concreteWriteIndex(state, indicesOptions, "my-data-stream", false, true);
assertThat(result.getName(), equalTo(dataStreamName + "-000002"));
assertThat(result.getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStreamName, 2)));
}
{
// Ignore data streams
@ -1861,18 +1861,18 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(4));
assertThat(result[0].getName(), equalTo(dataStream1 + "-000001"));
assertThat(result[1].getName(), equalTo(dataStream1 + "-000002"));
assertThat(result[2].getName(), equalTo(dataStream2 + "-000001"));
assertThat(result[3].getName(), equalTo(dataStream2 + "-000002"));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 1)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
assertThat(result[2].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 1)));
assertThat(result[3].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream2, 2)));;
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN;
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-m*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(2));
assertThat(result[0].getName(), equalTo(dataStream1 + "-000001"));
assertThat(result[1].getName(), equalTo(dataStream1 + "-000002"));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 1)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
}
{
IndicesOptions indicesOptions = IndicesOptions.STRICT_EXPAND_OPEN; // without include data streams
@ -1906,8 +1906,8 @@ public class IndexNameExpressionResolverTests extends ESTestCase {
Index[] result = indexNameExpressionResolver.concreteIndices(state, indicesOptions, true, "logs-*");
Arrays.sort(result, Comparator.comparing(Index::getName));
assertThat(result.length, equalTo(3));
assertThat(result[0].getName(), equalTo("logs-foobar-000001"));
assertThat(result[1].getName(), equalTo("logs-foobar-000002"));
assertThat(result[0].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 1)));
assertThat(result[1].getName(), equalTo(DataStream.getDefaultBackingIndexName(dataStream1, 2)));
assertThat(result[2].getName(), equalTo("logs-foobarbaz-0"));
}
}

4
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java
View File

@ -53,8 +53,8 @@ public class MetadataCreateDataStreamServiceTests extends ESTestCase {
ClusterState newState = MetadataCreateDataStreamService.createDataStream(metadataCreateIndexService, cs, req);
assertThat(newState.metadata().dataStreams().size(), equalTo(1));
assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName));
assertThat(newState.metadata().index(DataStream.getBackingIndexName(dataStreamName, 1)), notNullValue());
assertThat(newState.metadata().index(DataStream.getBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"),
assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)), notNullValue());
assertThat(newState.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, 1)).getSettings().get("index.hidden"),
equalTo("true"));
}

7
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java
View File

@ -120,12 +120,13 @@ public class MetadataDeleteIndexServiceTests extends ESTestCase {
int numIndexToDelete = randomIntBetween(1, numBackingIndices - 1);
Index indexToDelete = before.metadata().index(DataStream.getBackingIndexName(dataStreamName, numIndexToDelete)).getIndex();
Index indexToDelete = before.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, numIndexToDelete)).getIndex();
ClusterState after = service.deleteIndices(before, Set.of(indexToDelete));
assertThat(after.metadata().getIndices().get(indexToDelete.getName()), IsNull.nullValue());
assertThat(after.metadata().getIndices().size(), equalTo(numBackingIndices - 1));
assertThat(after.metadata().getIndices().get(DataStream.getBackingIndexName(dataStreamName, numIndexToDelete)), IsNull.nullValue());
assertThat(after.metadata().getIndices().get(
DataStream.getDefaultBackingIndexName(dataStreamName, numIndexToDelete)), IsNull.nullValue());
}
public void testDeleteCurrentWriteIndexForDataStream() {
@ -134,7 +135,7 @@ public class MetadataDeleteIndexServiceTests extends ESTestCase {
ClusterState before = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(
List.of(new Tuple<>(dataStreamName, numBackingIndices)), List.of());
Index indexToDelete = before.metadata().index(DataStream.getBackingIndexName(dataStreamName, numBackingIndices)).getIndex();
Index indexToDelete = before.metadata().index(DataStream.getDefaultBackingIndexName(dataStreamName, numBackingIndices)).getIndex();
Exception e = expectThrows(IllegalArgumentException.class, () -> service.deleteIndices(before, Set.of(indexToDelete)));
assertThat(e.getMessage(), containsString("index [" + indexToDelete.getName() + "] is the write index for data stream [" +

2
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexStateServiceTests.java
View File

@ -399,7 +399,7 @@ public class MetadataIndexStateServiceTests extends ESTestCase {
String dataStreamName = randomAlphaOfLength(6).toLowerCase(Locale.ROOT);
int numBackingIndices = randomIntBetween(1, 5);
dataStreamsToCreate.add(new Tuple<>(dataStreamName, numBackingIndices));
writeIndices.add(DataStream.getBackingIndexName(dataStreamName, numBackingIndices));
writeIndices.add(DataStream.getDefaultBackingIndexName(dataStreamName, numBackingIndices));
}
ClusterState cs = DeleteDataStreamRequestTests.getClusterStateWithDataStreams(dataStreamsToCreate,
org.elasticsearch.common.collect.List.of());

20
server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java
View File

@ -998,13 +998,14 @@ public class MetadataTests extends ESTestCase {
IllegalStateException e = expectThrows(IllegalStateException.class, b::build);
assertThat(e.getMessage(),
containsString("index, alias, and data stream names need to be unique, but the following duplicates were found [" +
dataStreamName + " (alias of [" + DataStream.getBackingIndexName(dataStreamName, 1) + "]) conflicts with data stream]"));
dataStreamName + " (alias of [" + DataStream.getDefaultBackingIndexName(dataStreamName, 1) +
"]) conflicts with data stream]"));
}
public void testBuilderRejectsDataStreamWithConflictingBackingIndices() {
final String dataStreamName = "my-data-stream";
IndexMetadata validIdx = createFirstBackingIndex(dataStreamName).build();
final String conflictingIndex = DataStream.getBackingIndexName(dataStreamName, 2);
final String conflictingIndex = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
IndexMetadata invalidIdx = createBackingIndex(dataStreamName, 2).build();
Metadata.Builder b = Metadata.builder()
.put(validIdx, false)
@ -1018,7 +1019,7 @@ public class MetadataTests extends ESTestCase {
public void testBuilderRejectsDataStreamWithConflictingBackingAlias() {
final String dataStreamName = "my-data-stream";
final String conflictingName = DataStream.getBackingIndexName(dataStreamName, 2);
final String conflictingName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
IndexMetadata idx = createFirstBackingIndex(dataStreamName)
.putAlias(new AliasMetadata.Builder(conflictingName))
.build();
@ -1039,7 +1040,7 @@ public class MetadataTests extends ESTestCase {
Metadata.Builder b = Metadata.builder();
for (int k = 1; k <= numBackingIndices; k++) {
lastBackingIndexNum = randomIntBetween(lastBackingIndexNum + 1, lastBackingIndexNum + 50);
IndexMetadata im = IndexMetadata.builder(DataStream.getBackingIndexName(dataStreamName, lastBackingIndexNum))
IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, lastBackingIndexNum))
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)
@ -1081,7 +1082,8 @@ public class MetadataTests extends ESTestCase {
assertThat(value.isHidden(), is(false));
assertThat(value.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM));
assertThat(value.getIndices().size(), equalTo(ds.getIndices().size()));
assertThat(value.getWriteIndex().getIndex().getName(), equalTo(name + "-00000" + ds.getIndices().size()));
assertThat(value.getWriteIndex().getIndex().getName(),
equalTo(DataStream.getDefaultBackingIndexName(name, ds.getGeneration())));
}
}
@ -1153,7 +1155,7 @@ public class MetadataTests extends ESTestCase {
expectThrows(IllegalStateException.class, () -> validateDataStreams(indicesLookup, dataStreamMetadata));
assertThat(illegalStateException.getMessage(),
is("data stream [foo-datastream] could create backing indices that conflict with 1 existing index(s) or alias(s) " +
"including 'foo-datastream-000020'"));
"including '" + index.getName() + "'"));
}
public void testValidateDataStreamsIgnoresIndicesWithoutCounter() {
@ -1192,7 +1194,7 @@ public class MetadataTests extends ESTestCase {
for (int i = 1; i <= generations; i++) {
IndexMetadata idx;
if (i % 2 == 0 && i < generations) {
idx = IndexMetadata.builder("shrink-" + DataStream.getBackingIndexName(dataStreamName, i))
idx = IndexMetadata.builder("shrink-" + DataStream.getDefaultBackingIndexName(dataStreamName, i))
.settings(ESTestCase.settings(Version.CURRENT).put("index.hidden", true))
.numberOfShards(1)
.numberOfReplicas(1)
@ -1283,7 +1285,7 @@ public class MetadataTests extends ESTestCase {
int lastIndexNum = randomIntBetween(9, 50);
Metadata.Builder b = Metadata.builder();
for (int k = 1; k <= numIndices; k++) {
IndexMetadata im = IndexMetadata.builder(DataStream.getBackingIndexName("index", lastIndexNum))
IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName("index", lastIndexNum))
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)
@ -1298,7 +1300,7 @@ public class MetadataTests extends ESTestCase {
int lastBackingIndexNum = 0;
for (int k = 1; k <= numBackingIndices; k++) {
lastBackingIndexNum = randomIntBetween(lastBackingIndexNum + 1, lastBackingIndexNum + 50);
IndexMetadata im = IndexMetadata.builder(DataStream.getBackingIndexName(dataStreamName, lastBackingIndexNum))
IndexMetadata im = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, lastBackingIndexNum))
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(1)

2
test/framework/src/main/java/org/elasticsearch/cluster/DataStreamTestHelper.java
View File

@ -39,7 +39,7 @@ public final class DataStreamTestHelper {
}
public static IndexMetadata.Builder createBackingIndex(String dataStreamName, int generation) {
return IndexMetadata.builder(DataStream.getBackingIndexName(dataStreamName, generation))
return IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, generation))
.settings(SETTINGS)
.numberOfShards(NUMBER_OF_SHARDS)
.numberOfReplicas(NUMBER_OF_REPLICAS);

2
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/RolloverStepTests.java
View File

@ -121,7 +121,7 @@ public class RolloverStepTests extends AbstractStepMasterTimeoutTestCase<Rollove
public void testPerformActionOnDataStream() {
String dataStreamName = "test-datastream";
IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamName + "-000001")
IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
.settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();

4
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/UpdateRolloverLifecycleDateStepTests.java
View File

@ -83,12 +83,12 @@ public class UpdateRolloverLifecycleDateStepTests extends AbstractStepTestCase<U
long creationDate = randomLongBetween(0, 1000000);
long rolloverTime = randomValueOtherThan(creationDate, () -> randomNonNegativeLong());
String dataStreamName = "test-datastream";
IndexMetadata originalIndexMeta = IndexMetadata.builder(dataStreamName + "-000001")
IndexMetadata originalIndexMeta = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
.putRolloverInfo(new RolloverInfo(dataStreamName, Collections.emptyList(), rolloverTime))
.settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
IndexMetadata rolledIndexMeta= IndexMetadata.builder(dataStreamName + "-000002")
IndexMetadata rolledIndexMeta= IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
.settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();

4
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsTests.java
View File

@ -149,11 +149,11 @@ public class WaitForActiveShardsTests extends AbstractStepTestCase<WaitForActive
public void testResultEvaluatedOnDataStream() throws IOException {
String dataStreamName = "test-datastream";
IndexMetadata originalIndexMeta = IndexMetadata.builder(dataStreamName + "-000001")
IndexMetadata originalIndexMeta = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
.settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
IndexMetadata rolledIndexMeta= IndexMetadata.builder(dataStreamName + "-000002")
IndexMetadata rolledIndexMeta= IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 2))
.settings(settings(Version.CURRENT).put("index.write.wait_for_active_shards", "3"))
.numberOfShards(1).numberOfReplicas(3).build();

2
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStepTests.java
View File

@ -139,7 +139,7 @@ public class WaitForRolloverReadyStepTests extends AbstractStepTestCase<WaitForR
public void testEvaluateConditionOnDataStreamTarget() {
String dataStreamName = "test-datastream";
IndexMetadata indexMetadata = IndexMetadata.builder(dataStreamName + "-000001")
IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, 1))
.settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();

9
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ilm;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.rest.ESRestTestCase;
@ -32,9 +33,11 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
String dataStream = "logs-foo";
indexDocument(client(), dataStream, true);
assertBusy(() -> assertTrue(indexExists("logs-foo-000002")));
assertBusy(() -> assertTrue(Boolean.parseBoolean((String) getIndexSettingsAsMap("logs-foo-000002").get("index.hidden"))));
assertBusy(() -> assertThat(getStepKeyForIndex(client(), "logs-foo-000001"), equalTo(PhaseCompleteStep.finalStep("hot").getKey())));
assertBusy(() -> assertTrue(indexExists(DataStream.getDefaultBackingIndexName(dataStream, 2))));
assertBusy(() -> assertTrue(Boolean.parseBoolean((String) getIndexSettingsAsMap(
DataStream.getDefaultBackingIndexName(dataStream, 2)).get("index.hidden"))));
assertBusy(() -> assertThat(getStepKeyForIndex(client(), DataStream.getDefaultBackingIndexName(dataStream, 1)),
equalTo(PhaseCompleteStep.finalStep("hot").getKey())));
}
}

2
x-pack/plugin/src/test/resources/rest-api-spec/test/data_stream/10_data_stream_resolvability.yml
View File

@ -103,7 +103,7 @@
ilm.explain_lifecycle:
index: logs-foobar
- is_false: indices.logs-foobar.managed
- is_true: indices.logs-foobar-000001.managed
- is_true: indices.\.ds-logs-foobar-000001.managed
- do:
indices.delete_data_stream: