honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 01:19:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove ingest.new_date_format (#25583)

Browse Source
This commit is contained in:
Tal Levy 2017-07-10 13:07:50 -07:00 committed by GitHub
parent b22bbf94da
commit e04be73ad5
15 changed files with 66 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java
View File

@ -162,18 +162,18 @@ public class SimulatePipelineRequest extends ActionRequest {
if (pipeline == null) {
throw new IllegalArgumentException("pipeline [" + pipelineId + "] does not exist");
}
List<IngestDocument> ingestDocumentList = parseDocs(config, pipelineStore.isNewIngestDateFormat());
List<IngestDocument> ingestDocumentList = parseDocs(config);
return new Parsed(pipeline, ingestDocumentList, verbose);
}
static Parsed parse(Map<String, Object> config, boolean verbose, PipelineStore pipelineStore) throws Exception {
Map<String, Object> pipelineConfig = ConfigurationUtils.readMap(null, null, config, Fields.PIPELINE);
Pipeline pipeline = PIPELINE_FACTORY.create(SIMULATED_PIPELINE_ID, pipelineConfig, pipelineStore.getProcessorFactories());
List<IngestDocument> ingestDocumentList = parseDocs(config, pipelineStore.isNewIngestDateFormat());
List<IngestDocument> ingestDocumentList = parseDocs(config);
return new Parsed(pipeline, ingestDocumentList, verbose);
}
private static List<IngestDocument> parseDocs(Map<String, Object> config, boolean newDateFormat) {
private static List<IngestDocument> parseDocs(Map<String, Object> config) {
List<Map<String, Object>> docs = ConfigurationUtils.readList(null, null, config, Fields.DOCS);
List<IngestDocument> ingestDocumentList = new ArrayList<>();
for (Map<String, Object> dataMap : docs) {
@ -183,7 +183,7 @@ public class SimulatePipelineRequest extends ActionRequest {
ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"),
ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.ROUTING.getFieldName()),
ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.PARENT.getFieldName()),
document, newDateFormat);
document);
ingestDocumentList.add(ingestDocument);
}
return ingestDocumentList;

9
core/src/main/java/org/elasticsearch/action/ingest/WriteableIngestDocument.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.action.ingest;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
@ -27,6 +28,8 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.ingest.IngestDocument;
import java.io.IOException;
import java.time.ZoneId;
import java.util.Date;
import java.util.Map;
import java.util.Objects;
@ -42,6 +45,12 @@ final class WriteableIngestDocument implements Writeable, ToXContent {
WriteableIngestDocument(StreamInput in) throws IOException {
Map<String, Object> sourceAndMetadata = in.readMap();
Map<String, Object> ingestMetadata = in.readMap();
if (in.getVersion().before(Version.V_6_0_0_beta1)) {
ingestMetadata.computeIfPresent("timestamp", (k, o) -> {
Date date = (Date) o;
return date.toInstant().atZone(ZoneId.systemDefault());
});
}
this.ingestDocument = new IngestDocument(sourceAndMetadata, ingestMetadata);
}

10
core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java
View File

@ -53,6 +53,9 @@ import java.nio.file.FileSystemException;
import java.nio.file.FileSystemLoopException;
import java.nio.file.NoSuchFileException;
import java.nio.file.NotDirectoryException;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -542,6 +545,8 @@ public abstract class StreamInput extends InputStream {
return readBytesRef();
case 22:
return readGeoPoint();
case 23:
return readZonedDateTime();
default:
throw new IOException("Can't read unknown type [" + type + "]");
}
@ -562,6 +567,11 @@ public abstract class StreamInput extends InputStream {
return new DateTime(readLong(), DateTimeZone.forID(timeZoneId));
}
private ZonedDateTime readZonedDateTime() throws IOException {
final String timeZoneId = readString();
return ZonedDateTime.ofInstant(Instant.ofEpochMilli(readLong()), ZoneId.of(timeZoneId));
}
private Object[] readArray() throws IOException {
int size8 = readArraySize();
Object[] list8 = new Object[size8];

8
core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java
View File

@ -50,6 +50,7 @@ import java.nio.file.FileSystemException;
import java.nio.file.FileSystemLoopException;
import java.nio.file.NoSuchFileException;
import java.nio.file.NotDirectoryException;
import java.time.ZonedDateTime;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@ -620,6 +621,13 @@ public abstract class StreamOutput extends OutputStream {
o.writeByte((byte) 22);
o.writeGeoPoint((GeoPoint) v);
});
writers.put(ZonedDateTime.class, (o, v) -> {
o.writeByte((byte) 23);
final ZonedDateTime zonedDateTime = (ZonedDateTime) v;
zonedDateTime.getZone().getId();
o.writeString(zonedDateTime.getZone().getId());
o.writeLong(zonedDateTime.toInstant().toEpochMilli());
});
WRITERS = Collections.unmodifiableMap(writers);
}

3
core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View File

@ -411,7 +411,6 @@ public final class ClusterSettings extends AbstractScopedSettings {
SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING,
ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING,
FastVectorHighlighter.SETTING_TV_HIGHLIGHT_MULTI_VALUE,
Node.BREAKER_TYPE_KEY,
IngestService.NEW_INGEST_DATE_FORMAT
Node.BREAKER_TYPE_KEY
)));
}

11
core/src/main/java/org/elasticsearch/ingest/IngestDocument.java
View File

@ -57,11 +57,6 @@ public final class IngestDocument {
private final Map<String, Object> ingestMetadata;
public IngestDocument(String index, String type, String id, String routing, String parent, Map<String, Object> source) {
this(index, type, id, routing, parent, source, false);
}
public IngestDocument(String index, String type, String id, String routing, String parent, Map<String, Object> source,
boolean newDateFormat) {
this.sourceAndMetadata = new HashMap<>();
this.sourceAndMetadata.putAll(source);
this.sourceAndMetadata.put(MetaData.INDEX.getFieldName(), index);
@ -75,11 +70,7 @@ public final class IngestDocument {
}
this.ingestMetadata = new HashMap<>();
if (newDateFormat) {
this.ingestMetadata.put(TIMESTAMP, ZonedDateTime.now(ZoneOffset.UTC));
} else {
this.ingestMetadata.put(TIMESTAMP, new Date());
}
this.ingestMetadata.put(TIMESTAMP, ZonedDateTime.now(ZoneOffset.UTC));
}
/**

7
core/src/main/java/org/elasticsearch/ingest/IngestService.java
View File

@ -40,13 +40,10 @@ import static org.elasticsearch.common.settings.Setting.Property;
* Holder class for several ingest related services.
*/
public class IngestService {
public static final Setting<Boolean> NEW_INGEST_DATE_FORMAT =
Setting.boolSetting("ingest.new_date_format", false, Property.NodeScope, Property.Dynamic, Property.Deprecated);
private final PipelineStore pipelineStore;
private final PipelineExecutionService pipelineExecutionService;
public IngestService(ClusterSettings clusterSettings, Settings settings, ThreadPool threadPool,
public IngestService(Settings settings, ThreadPool threadPool,
Environment env, ScriptService scriptService, AnalysisRegistry analysisRegistry,
List<IngestPlugin> ingestPlugins) {
Processor.Parameters parameters = new Processor.Parameters(env, scriptService,
@ -60,7 +57,7 @@ public class IngestService {
}
}
}
this.pipelineStore = new PipelineStore(clusterSettings, settings, Collections.unmodifiableMap(processorFactories));
this.pipelineStore = new PipelineStore(settings, Collections.unmodifiableMap(processorFactories));
this.pipelineExecutionService = new PipelineExecutionService(pipelineStore, threadPool);
}

3
core/src/main/java/org/elasticsearch/ingest/PipelineExecutionService.java
View File

@ -160,8 +160,7 @@ public class PipelineExecutionService implements ClusterStateApplier {
String routing = indexRequest.routing();
String parent = indexRequest.parent();
Map<String, Object> sourceAsMap = indexRequest.sourceAsMap();
IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, parent,
sourceAsMap, store.isNewIngestDateFormat());
IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, parent, sourceAsMap);
pipeline.execute(ingestDocument);
Map<IngestDocument.MetaData, String> metadataMap = ingestDocument.extractMetadata();

13
core/src/main/java/org/elasticsearch/ingest/PipelineStore.java
View File

@ -52,7 +52,6 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
private final Pipeline.Factory factory = new Pipeline.Factory();
private final Map<String, Processor.Factory> processorFactories;
private volatile boolean newIngestDateFormat;
// Ideally this should be in IngestMetadata class, but we don't have the processor factories around there.
// We know of all the processor factories when a node with all its plugin have been initialized. Also some
@ -60,15 +59,9 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
// are loaded, so in the cluster state we just save the pipeline config and here we keep the actual pipelines around.
volatile Map<String, Pipeline> pipelines = new HashMap<>();
public PipelineStore(ClusterSettings clusterSettings, Settings settings, Map<String, Processor.Factory> processorFactories) {
public PipelineStore(Settings settings, Map<String, Processor.Factory> processorFactories) {
super(settings);
this.processorFactories = processorFactories;
this.newIngestDateFormat = IngestService.NEW_INGEST_DATE_FORMAT.get(settings);
clusterSettings.addSettingsUpdateConsumer(IngestService.NEW_INGEST_DATE_FORMAT, this::setNewIngestDateFormat);
}
private void setNewIngestDateFormat(Boolean newIngestDateFormat) {
this.newIngestDateFormat = newIngestDateFormat;
}
@Override
@ -212,10 +205,6 @@ public class PipelineStore extends AbstractComponent implements ClusterStateAppl
return processorFactories;
}
public boolean isNewIngestDateFormat() {
return newIngestDateFormat;
}
/**
* @return pipeline configuration specified by id. If multiple ids or wildcards are specified multiple pipelines
* may be returned

2
core/src/main/java/org/elasticsearch/node/Node.java
View File

@ -334,7 +334,7 @@ public class Node implements Closeable {
final ClusterService clusterService = new ClusterService(settings, settingsModule.getClusterSettings(), threadPool);
clusterService.addListener(scriptModule.getScriptService());
resourcesToClose.add(clusterService);
final IngestService ingestService = new IngestService(clusterService.getClusterSettings(), settings, threadPool, this.environment,
final IngestService ingestService = new IngestService(settings, threadPool, this.environment,
scriptModule.getScriptService(), analysisModule.getAnalysisRegistry(), pluginsService.filterPlugins(IngestPlugin.class));
final DiskThresholdMonitor listener = new DiskThresholdMonitor(settings, clusterService::state,
clusterService.getClusterSettings(), client);

47
core/src/test/java/org/elasticsearch/ingest/IngestDocumentTests.java
View File

@ -46,19 +46,14 @@ import static org.hamcrest.Matchers.sameInstance;
public class IngestDocumentTests extends ESTestCase {
private static final Date BOGUS_TIMESTAMP = new Date(0L);
private static final ZonedDateTime BOGUS_TIMESTAMP_NEW_DATE_FORMAT = ZonedDateTime.of(2016, 10, 23, 0, 0, 0, 0, ZoneOffset.UTC);
private static final ZonedDateTime BOGUS_TIMESTAMP = ZonedDateTime.of(2016, 10, 23, 0, 0, 0, 0, ZoneOffset.UTC);
private IngestDocument ingestDocument;
private IngestDocument ingestDocumentWithNewDateFormat;
public IngestDocument getTestIngestDocument(boolean newDateFormat) {
@Before
public void setTestIngestDocument() {
Map<String, Object> document = new HashMap<>();
Map<String, Object> ingestMap = new HashMap<>();
if (newDateFormat) {
ingestMap.put("timestamp", BOGUS_TIMESTAMP_NEW_DATE_FORMAT);
} else {
ingestMap.put("timestamp", BOGUS_TIMESTAMP);
}
ingestMap.put("timestamp", BOGUS_TIMESTAMP);
document.put("_ingest", ingestMap);
document.put("foo", "bar");
document.put("int", 123);
@ -79,18 +74,7 @@ public class IngestDocumentTests extends ESTestCase {
list.add(null);
document.put("list", list);
return new IngestDocument("index", "type", "id", null, null, document, newDateFormat);
}
@Before
public void setIngestDocuments() {
ingestDocument = getTestIngestDocument(false);
ingestDocumentWithNewDateFormat = getTestIngestDocument(true);
}
public void testDefaultConstructorUsesDateClass() {
IngestDocument ingestDocument = new IngestDocument("foo", "bar", "baz", "fuzz", "buzz", Collections.emptyMap());
assertThat(ingestDocument.getFieldValue("_ingest.timestamp", Object.class).getClass(), equalTo(Date.class));
ingestDocument = new IngestDocument("index", "type", "id", null, null, document);
}
public void testSimpleGetFieldValue() {
@ -101,16 +85,10 @@ public class IngestDocumentTests extends ESTestCase {
assertThat(ingestDocument.getFieldValue("_index", String.class), equalTo("index"));
assertThat(ingestDocument.getFieldValue("_type", String.class), equalTo("type"));
assertThat(ingestDocument.getFieldValue("_id", String.class), equalTo("id"));
assertThat(ingestDocument.getFieldValue("_ingest.timestamp", Date.class),
both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP))));
assertThat(ingestDocument.getFieldValue("_source._ingest.timestamp", Date.class), equalTo(BOGUS_TIMESTAMP));
}
public void testNewDateFormat() {
assertThat(ingestDocumentWithNewDateFormat.getFieldValue("_ingest.timestamp", ZonedDateTime.class),
both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP_NEW_DATE_FORMAT))));
assertThat(ingestDocumentWithNewDateFormat.getFieldValue("_source._ingest.timestamp", ZonedDateTime.class),
equalTo(BOGUS_TIMESTAMP_NEW_DATE_FORMAT));
assertThat(ingestDocument.getFieldValue("_ingest.timestamp", ZonedDateTime.class),
both(notNullValue()).and(not(equalTo(BOGUS_TIMESTAMP))));
assertThat(ingestDocument.getFieldValue("_source._ingest.timestamp", ZonedDateTime.class),
equalTo(BOGUS_TIMESTAMP));
}
public void testGetSourceObject() {
@ -994,10 +972,11 @@ public class IngestDocumentTests extends ESTestCase {
long before = System.currentTimeMillis();
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
long after = System.currentTimeMillis();
Date timestamp = (Date) ingestDocument.getIngestMetadata().get(IngestDocument.TIMESTAMP);
ZonedDateTime timestamp = (ZonedDateTime) ingestDocument.getIngestMetadata().get(IngestDocument.TIMESTAMP);
long actualMillis = timestamp.toInstant().toEpochMilli();
assertThat(timestamp, notNullValue());
assertThat(timestamp.getTime(), greaterThanOrEqualTo(before));
assertThat(timestamp.getTime(), lessThanOrEqualTo(after));
assertThat(actualMillis, greaterThanOrEqualTo(before));
assertThat(actualMillis, lessThanOrEqualTo(after));
}
public void testCopyConstructor() {

9
core/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java
View File

@ -40,8 +40,8 @@ public class IngestServiceTests extends ESTestCase {
public void testIngestPlugin() {
ThreadPool tp = Mockito.mock(ThreadPool.class);
IngestService ingestService = new IngestService(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
Settings.EMPTY, tp, null, null, null, Collections.singletonList(DUMMY_PLUGIN));
IngestService ingestService = new IngestService(Settings.EMPTY, tp, null, null,
null, Collections.singletonList(DUMMY_PLUGIN));
Map<String, Processor.Factory> factories = ingestService.getPipelineStore().getProcessorFactories();
assertTrue(factories.containsKey("foo"));
assertEquals(1, factories.size());
@ -50,9 +50,8 @@ public class IngestServiceTests extends ESTestCase {
public void testIngestPluginDuplicate() {
ThreadPool tp = Mockito.mock(ThreadPool.class);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
new IngestService(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
Settings.EMPTY, tp, null, null, null, Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN))
);
new IngestService(Settings.EMPTY, tp, null, null,
null, Arrays.asList(DUMMY_PLUGIN, DUMMY_PLUGIN)));
assertTrue(e.getMessage(), e.getMessage().contains("already registered"));
}
}

12
core/src/test/java/org/elasticsearch/ingest/PipelineStoreTests.java
View File

@ -50,7 +50,6 @@ import static org.hamcrest.Matchers.nullValue;
public class PipelineStoreTests extends ESTestCase {
private ClusterSettings clusterSettings;
private PipelineStore store;
@Before
@ -95,8 +94,7 @@ public class PipelineStoreTests extends ESTestCase {
}
};
});
clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
store = new PipelineStore(clusterSettings, Settings.EMPTY, processorFactories);
store = new PipelineStore(Settings.EMPTY, processorFactories);
}
public void testUpdatePipelines() {
@ -371,12 +369,4 @@ public class PipelineStoreTests extends ESTestCase {
IngestInfo ingestInfo = new IngestInfo(Collections.singletonList(new ProcessorInfo("set")));
store.validatePipeline(Collections.singletonMap(discoveryNode, ingestInfo), putRequest);
}
public void testUpdateIngestNewDateFormatSetting() throws Exception {
assertFalse(store.isNewIngestDateFormat());
clusterSettings.applySettings(Settings.builder().put(IngestService.NEW_INGEST_DATE_FORMAT.getKey(), true).build());
assertTrue(store.isNewIngestDateFormat());
assertWarnings("[ingest.new_date_format] setting was deprecated in Elasticsearch and will be " +
"removed in a future release! See the breaking changes documentation for the next major version.");
}
}

10
docs/reference/migration/migrate_6_0/ingest.asciidoc
View File

@ -3,4 +3,12 @@
==== Timestamp meta-data field type has changed
The type of the "timestamp" meta-data field has changed from `java.lang.String` to `java.util.Date`.
The type of the "timestamp" meta-data field has changed from `java.lang.String` to `java.util.Date`.
==== The format of the string-formatted ingest.timestamp field has changed
Previously, since Elasticsearch 5.4.0, you needed to use `ingest.new_date_format` to have the
`ingest.timestamp` metadata field be formatted in such a way that ES can coerce it to a field
of type `date` without further transformation. This is not necessary anymore and this setting was
removed. You can now simply set a field to `{{ingest.timestamp}}` in a pipeline, and have that
field be of type `date` without any mapping errors.

83
qa/smoke-test-ingest-with-all-dependencies/src/test/resources/rest-api-spec/test/ingest/60_pipeline_timestamp_date_mapping.yml
View File

@ -1,5 +1,5 @@
---
"Test timestamp templating does not match date-mapping defaults":
"Test timestamp templating matches date-mapping defaults":
- do:
cluster.health:
wait_for_status: green
@ -23,25 +23,6 @@
"field": "my_time",
"value": "{{ _ingest.timestamp }}"
}
},
{
"grok" : {
"field" : "my_time",
"patterns": ["%{DAY:day} %{MONTH:month} %{MONTHDAY:monthday} %{TIME:time} %{WORD} %{YEAR:year}"]
}
},
{
"set": {
"field": "my_time",
"value": "{{day}} {{month}} {{monthday}} {{time}} {{year}}"
}
},
{
"date" : {
"field" : "my_time",
"target_field": "my_time",
"formats": ["EEE MMM dd HH:mm:ss yyyy"]
}
}
]
}
@ -55,65 +36,3 @@
pipeline: "my_timely_pipeline"
body: {}
---
"Test timestamp templating matches date-mapping defaults with ingest.new_date_format":
- skip:
version: " - 5.3.99"
reason: deprecated in 5.4.0
features: "warnings"
- do:
cluster.health:
wait_for_status: green
- do:
indices.create:
index: timetest_newdateformat
body:
mappings:
test: { "properties": { "my_time": {"type": "date"}}}
- do:
cluster.put_settings:
body:
transient:
ingest.new_date_format: true
warnings:
- "[ingest.new_date_format] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
- match: {transient: {ingest: {new_date_format: "true"}}}
- do:
ingest.put_pipeline:
id: "my_timely_pipeline_with_new_date_format"
body: >
{
"description": "_description",
"processors": [
{
"set" : {
"field": "my_time",
"value": "{{ _ingest.timestamp }}"
}
}
]
}
- match: { acknowledged: true }
- do:
index:
index: timetest
type: test
id: 1
pipeline: "my_timely_pipeline_with_new_date_format"
body: {}
- do:
cluster.put_settings:
body:
transient:
ingest.new_date_format: false
warnings:
- "[ingest.new_date_format] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version."
- match: {transient: {ingest: {new_date_format: "false"}}}