honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove DeDotProcessor from Ingest

This commit is contained in:
Tal Levy 2016-02-02 14:16:01 -08:00
parent 3191fc7347
commit 9e7e2ab10b
6 changed files with 0 additions and 324 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
core/src/main/java/org/elasticsearch/ingest/processor/DeDotProcessor.java
View File

@ -1,106 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest.processor;
import org.elasticsearch.ingest.core.AbstractProcessor;
import org.elasticsearch.ingest.core.AbstractProcessorFactory;
import org.elasticsearch.ingest.core.ConfigurationUtils;
import org.elasticsearch.ingest.core.IngestDocument;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
/**
* Processor that replaces dots in document field names with a
* specified separator.
*/
public class DeDotProcessor extends AbstractProcessor {
public static final String TYPE = "dedot";
static final String DEFAULT_SEPARATOR = "_";
private final String separator;
DeDotProcessor(String tag, String separator) {
super(tag);
this.separator = separator;
}
public String getSeparator() {
return separator;
}
@Override
public void execute(IngestDocument document) {
deDot(document.getSourceAndMetadata());
}
@Override
public String getType() {
return TYPE;
}
/**
* Recursively iterates through Maps and Lists in search of map entries with
* keys containing dots. The dots in these fields are replaced with {@link #separator}.
*
* @param obj The current object in context to be checked for dots in its fields.
*/
private void deDot(Object obj) {
if (obj instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> doc = (Map) obj;
Iterator<Map.Entry<String, Object>> it = doc.entrySet().iterator();
Map<String, Object> deDottedFields = new HashMap<>();
while (it.hasNext()) {
Map.Entry<String, Object> entry = it.next();
deDot(entry.getValue());
String fieldName = entry.getKey();
if (fieldName.contains(".")) {
String deDottedFieldName = fieldName.replaceAll("\\.", separator);
deDottedFields.put(deDottedFieldName, entry.getValue());
it.remove();
}
}
doc.putAll(deDottedFields);
} else if (obj instanceof List) {
@SuppressWarnings("unchecked")
List<Object> list = (List) obj;
for (Object value : list) {
deDot(value);
}
}
}
public static class Factory extends AbstractProcessorFactory<DeDotProcessor> {
@Override
public DeDotProcessor doCreate(String processorTag, Map<String, Object> config) throws Exception {
String separator = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "separator");
if (separator == null) {
separator = DEFAULT_SEPARATOR;
}
return new DeDotProcessor(processorTag, separator);
}
}
}

2
core/src/main/java/org/elasticsearch/node/NodeModule.java
View File

@ -28,7 +28,6 @@ import org.elasticsearch.ingest.core.TemplateService;
import org.elasticsearch.ingest.processor.AppendProcessor;
import org.elasticsearch.ingest.processor.ConvertProcessor;
import org.elasticsearch.ingest.processor.DateProcessor;
import org.elasticsearch.ingest.processor.DeDotProcessor;
import org.elasticsearch.ingest.processor.FailProcessor;
import org.elasticsearch.ingest.processor.GsubProcessor;
import org.elasticsearch.ingest.processor.JoinProcessor;
@ -75,7 +74,6 @@ public class NodeModule extends AbstractModule {
registerProcessor(ConvertProcessor.TYPE, (templateService) -> new ConvertProcessor.Factory());
registerProcessor(GsubProcessor.TYPE, (templateService) -> new GsubProcessor.Factory());
registerProcessor(FailProcessor.TYPE, FailProcessor.Factory::new);
registerProcessor(DeDotProcessor.TYPE, (templateService) -> new DeDotProcessor.Factory());
}
@Override

56
core/src/test/java/org/elasticsearch/ingest/processor/DeDotProcessorFactoryTests.java
View File

@ -1,56 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest.processor;
import org.elasticsearch.ingest.core.AbstractProcessorFactory;
import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.CoreMatchers.equalTo;
public class DeDotProcessorFactoryTests extends ESTestCase {
private DeDotProcessor.Factory factory;
@Before
public void init() {
factory = new DeDotProcessor.Factory();
}
public void testCreate() throws Exception {
Map<String, Object> config = new HashMap<>();
config.put("separator", "_");
String processorTag = randomAsciiOfLength(10);
config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
DeDotProcessor deDotProcessor = factory.create(config);
assertThat(deDotProcessor.getSeparator(), equalTo("_"));
assertThat(deDotProcessor.getTag(), equalTo(processorTag));
}
public void testCreateMissingSeparatorField() throws Exception {
Map<String, Object> config = new HashMap<>();
DeDotProcessor deDotProcessor = factory.create(config);
assertThat(deDotProcessor.getSeparator(), equalTo(DeDotProcessor.DEFAULT_SEPARATOR));
}
}

75
core/src/test/java/org/elasticsearch/ingest/processor/DeDotProcessorTests.java
View File

@ -1,75 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.ingest.processor;
import org.elasticsearch.ingest.core.IngestDocument;
import org.elasticsearch.ingest.core.Processor;
import org.elasticsearch.test.ESTestCase;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
public class DeDotProcessorTests extends ESTestCase {
public void testSimple() throws Exception {
Map<String, Object> source = new HashMap<>();
source.put("a.b", "hello world!");
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
String separator = randomUnicodeOfCodepointLengthBetween(1, 10);
Processor processor = new DeDotProcessor(randomAsciiOfLength(10), separator);
processor.execute(ingestDocument);
assertThat(ingestDocument.getSourceAndMetadata().get("a" + separator + "b" ), equalTo("hello world!"));
}
public void testSimpleMap() throws Exception {
Map<String, Object> source = new HashMap<>();
Map<String, Object> subField = new HashMap<>();
subField.put("b.c", "hello world!");
source.put("a", subField);
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new DeDotProcessor(randomAsciiOfLength(10), "_");
processor.execute(ingestDocument);
IngestDocument expectedDocument = new IngestDocument(
Collections.singletonMap("a", Collections.singletonMap("b_c", "hello world!")),
Collections.emptyMap());
assertThat(ingestDocument, equalTo(expectedDocument));
}
public void testSimpleList() throws Exception {
Map<String, Object> source = new HashMap<>();
Map<String, Object> subField = new HashMap<>();
subField.put("b.c", "hello world!");
source.put("a", Arrays.asList(subField));
IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
Processor processor = new DeDotProcessor(randomAsciiOfLength(10), "_");
processor.execute(ingestDocument);
IngestDocument expectedDocument = new IngestDocument(
Collections.singletonMap("a",
Collections.singletonList(Collections.singletonMap("b_c", "hello world!"))),
Collections.emptyMap());
assertThat(ingestDocument, equalTo(expectedDocument));
}
}

21
docs/reference/ingest/ingest.asciidoc
View File

@ -534,27 +534,6 @@ to the requester.
}
--------------------------------------------------
==== DeDot Processor
The DeDot Processor is used to remove dots (".") from field names and
replace them with a specific `separator` string.
[[dedot-options]]
.DeDot Options
[options="header"]
|======
| Name | Required | Default | Description
| `separator` | yes | "_" | The string to replace dots with in all field names
|======
[source,js]
--------------------------------------------------
{
"dedot": {
"separator": "_"
}
}
--------------------------------------------------
=== Accessing data in pipelines

64
rest-api-spec/src/main/resources/rest-api-spec/test/ingest/80_dedot_processor.yaml
View File

@ -1,64 +0,0 @@
---
"Test De-Dot Processor With Provided Separator":
- do:
ingest.put_pipeline:
id: "my_pipeline"
body: >
{
"description": "_description",
"processors": [
{
"dedot" : {
"separator" : "3"
}
}
]
}
- match: { acknowledged: true }
- do:
index:
index: test
type: test
id: 1
pipeline: "my_pipeline"
body: {"a.b.c": "hello world"}
- do:
get:
index: test
type: test
id: 1
- match: { _source.a3b3c: "hello world" }
---
"Test De-Dot Processor With Default Separator":
- do:
ingest.put_pipeline:
id: "my_pipeline"
body: >
{
"description": "_description",
"processors": [
{
"dedot" : {
}
}
]
}
- match: { acknowledged: true }
- do:
index:
index: test
type: test
id: 1
pipeline: "my_pipeline"
body: {"a.b.c": "hello world"}
- do:
get:
index: test
type: test
id: 1
- match: { _source.a_b_c: "hello world" }