honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Validate properties values according to database type (#17940) 

Fixes #17683.
This commit is contained in:
Tal Levy 2016-04-29 07:58:27 -07:00
parent f349c4f135
commit 07c2fbf83a
3 changed files with 86 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java
View File

@ -58,6 +58,8 @@ import static org.elasticsearch.ingest.core.ConfigurationUtils.readStringPropert
public final class GeoIpProcessor extends AbstractProcessor {
public static final String TYPE = "geoip";
private static final String CITY_DB_TYPE = "GeoLite2-City";
private static final String COUNTRY_DB_TYPE = "GeoLite2-Country";
private final String field;
private final String targetField;
@ -79,14 +81,14 @@ public final class GeoIpProcessor extends AbstractProcessor {
Map<String, Object> geoData;
switch (dbReader.getMetadata().getDatabaseType()) {
case "GeoLite2-City":
case CITY_DB_TYPE:
try {
geoData = retrieveCityGeoData(ipAddress);
} catch (AddressNotFoundRuntimeException e) {
geoData = Collections.emptyMap();
}
break;
case "GeoLite2-Country":
case COUNTRY_DB_TYPE:
try {
geoData = retrieveCountryGeoData(ipAddress);
} catch (AddressNotFoundRuntimeException e) {
@ -215,10 +217,11 @@ public final class GeoIpProcessor extends AbstractProcessor {
}
public static final class Factory extends AbstractProcessorFactory<GeoIpProcessor> implements Closeable {
static final Set<Property> DEFAULT_PROPERTIES = EnumSet.of(
Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME, Property.CITY_NAME, Property.LOCATION
static final Set<Property> DEFAULT_CITY_PROPERTIES = EnumSet.of(
Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE, Property.REGION_NAME,
Property.CITY_NAME, Property.LOCATION
);
static final Set<Property> DEFAULT_COUNTRY_PROPERTIES = EnumSet.of(Property.CONTINENT_NAME, Property.COUNTRY_ISO_CODE);
private final Map<String, DatabaseReader> databaseReaders;
@ -233,24 +236,33 @@ public final class GeoIpProcessor extends AbstractProcessor {
String databaseFile = readStringProperty(TYPE, processorTag, config, "database_file", "GeoLite2-City.mmdb");
List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
DatabaseReader databaseReader = databaseReaders.get(databaseFile);
if (databaseReader == null) {
throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist");
}
String databaseType = databaseReader.getMetadata().getDatabaseType();
final Set<Property> properties;
if (propertyNames != null) {
properties = EnumSet.noneOf(Property.class);
for (String fieldName : propertyNames) {
try {
properties.add(Property.parse(fieldName));
} catch (Exception e) {
throw newConfigurationException(TYPE, processorTag, "properties", "illegal field option [" + fieldName + "]. valid values are [" + Arrays.toString(Property.values()) + "]");
properties.add(Property.parseProperty(databaseType, fieldName));
} catch (IllegalArgumentException e) {
throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage());
}
}
} else {
properties = DEFAULT_PROPERTIES;
if (CITY_DB_TYPE.equals(databaseType)) {
properties = DEFAULT_CITY_PROPERTIES;
} else if (COUNTRY_DB_TYPE.equals(databaseType)) {
properties = DEFAULT_COUNTRY_PROPERTIES;
} else {
throw newConfigurationException(TYPE, processorTag, "database_file", "Unsupported database type [" + databaseType + "]");
}
}
DatabaseReader databaseReader = databaseReaders.get(databaseFile);
if (databaseReader == null) {
throw newConfigurationException(TYPE, processorTag, "database_file", "database file [" + databaseFile + "] doesn't exist");
}
return new GeoIpProcessor(processorTag, ipField, databaseReader, targetField, properties);
}
@ -279,13 +291,29 @@ public final class GeoIpProcessor extends AbstractProcessor {
REGION_NAME,
CITY_NAME,
TIMEZONE,
LATITUDE,
LONGITUDE,
LOCATION;
public static Property parse(String value) {
return valueOf(value.toUpperCase(Locale.ROOT));
}
static final EnumSet<Property> ALL_CITY_PROPERTIES = EnumSet.allOf(Property.class);
static final EnumSet<Property> ALL_COUNTRY_PROPERTIES = EnumSet.of(Property.IP, Property.CONTINENT_NAME,
Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE);
public static Property parseProperty(String databaseType, String value) {
Set<Property> validProperties = EnumSet.noneOf(Property.class);
if (CITY_DB_TYPE.equals(databaseType)) {
validProperties = ALL_CITY_PROPERTIES;
} else if (COUNTRY_DB_TYPE.equals(databaseType)) {
validProperties = ALL_COUNTRY_PROPERTIES;
}
try {
Property property = valueOf(value.toUpperCase(Locale.ROOT));
if (validProperties.contains(property) == false) {
throw new IllegalArgumentException("invalid");
}
return property;
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("illegal property value [" + value + "]. valid values are " + Arrays.toString(validProperties.toArray()));
}
}
}
}

41
plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
View File

@ -19,8 +19,10 @@
package org.elasticsearch.ingest.geoip;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.maxmind.geoip2.DatabaseReader;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.ingest.core.AbstractProcessorFactory;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.StreamsUtils;
@ -79,7 +81,25 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip"));
assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-City"));
assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_PROPERTIES));
assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_CITY_PROPERTIES));
}
public void testCountryBuildDefaults() throws Exception {
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb");
String processorTag = randomAsciiOfLength(10);
config.put(AbstractProcessorFactory.TAG_KEY, processorTag);
GeoIpProcessor processor = factory.create(config);
assertThat(processor.getTag(), equalTo(processorTag));
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip"));
assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
}
public void testBuildTargetField() throws Exception {
@ -101,6 +121,23 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
assertThat(processor.getField(), equalTo("_field"));
assertThat(processor.getTargetField(), equalTo("geoip"));
assertThat(processor.getDbReader().getMetadata().getDatabaseType(), equalTo("GeoLite2-Country"));
assertThat(processor.getProperties(), sameInstance(GeoIpProcessor.Factory.DEFAULT_COUNTRY_PROPERTIES));
}
public void testBuildWithCountryDbAndCityFields() throws Exception {
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders);
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
config.put("database_file", "GeoLite2-Country.mmdb");
EnumSet<GeoIpProcessor.Property> cityOnlyProperties = EnumSet.complementOf(GeoIpProcessor.Property.ALL_COUNTRY_PROPERTIES);
String cityProperty = RandomPicks.randomFrom(Randomness.get(), cityOnlyProperties).toString();
config.put("properties", Collections.singletonList(cityProperty));
try {
factory.create(config);
fail("Exception expected");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + cityProperty + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]"));
}
}
public void testBuildNonExistingDbFile() throws Exception {
@ -146,7 +183,7 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
factory.create(config);
fail("exception expected");
} catch (ElasticsearchParseException e) {
assertThat(e.getMessage(), equalTo("[properties] illegal field option [invalid]. valid values are [[IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LATITUDE, LONGITUDE, LOCATION]]"));
assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME, REGION_NAME, CITY_NAME, TIMEZONE, LOCATION]"));
}
config = new HashMap<>();

2
plugins/ingest-geoip/src/test/resources/rest-api-spec/test/ingest_geoip/20_geoip_processor.yaml
View File

@ -54,7 +54,7 @@
{
"geoip" : {
"field" : "field1",
"properties" : ["city_name", "country_iso_code", "ip", "latitude", "longitude", "location", "timezone", "country_name", "region_name", "continent_name"]
"properties" : ["city_name", "country_iso_code", "ip", "location", "timezone", "country_name", "region_name", "continent_name"]
}
}
]