honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix GeoIpProcessorFactoryTests on windows (#45668) 

Switches windows build to use geoip database loaded on heap instead
of memory mapping it.

Closes #44552
This commit is contained in:
Igor Motov 2019-08-21 08:45:51 -04:00
parent 47bbd9d9a9
commit 28006fe19f
2 changed files with 10 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/ingest-geoip/build.gradle
View File

@ -17,6 +17,8 @@
* under the License. * under the License.
*/ */
import org.apache.tools.ant.taskdefs.condition.Os
esplugin { esplugin {
description 'Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database' description 'Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database'
classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin' classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
@ -64,3 +66,11 @@ thirdPartyAudit.ignoreMissingClasses (
'org.apache.http.impl.client.HttpClientBuilder', 'org.apache.http.impl.client.HttpClientBuilder',
'org.apache.http.util.EntityUtils' 'org.apache.http.util.EntityUtils'
) )
test {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
// Windows cannot cleanup database files properly unless it loads everything on heap.
// See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows for more information
systemProperty 'es.geoip.load_db_on_heap', 'true'
}
}

54
modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.ingest.geoip; package org.elasticsearch.ingest.geoip;
import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.util.Constants;
import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Randomness;
import org.elasticsearch.index.VersionType; import org.elasticsearch.index.VersionType;
@ -55,16 +54,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
@BeforeClass @BeforeClass
public static void loadDatabaseReaders() throws IOException { public static void loadDatabaseReaders() throws IOException {
// there are still problems on windows
assumeFalse("https://github.com/elastic/elasticsearch/issues/44552", Constants.WINDOWS);
// Skip setup because Windows cannot cleanup these files properly. The reason is that they are using
// a MappedByteBuffer which will keep the file mappings active until it is garbage-collected. As a consequence,
// the corresponding file appears to be still in use and Windows cannot delete it.
if (Constants.WINDOWS) {
return;
}
final Path geoIpDir = createTempDir(); final Path geoIpDir = createTempDir();
final Path configDir = createTempDir(); final Path configDir = createTempDir();
final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
@ -76,13 +65,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
@AfterClass @AfterClass
public static void closeDatabaseReaders() throws IOException { public static void closeDatabaseReaders() throws IOException {
// Skip setup because Windows cannot cleanup these files properly. The reason is that they are using
// a MappedByteBuffer which will keep the file mappings active until it is garbage-collected. As a consequence,
// the corresponding file appears to be still in use and Windows cannot delete it.
if (Constants.WINDOWS) {
return;
}
for (DatabaseReaderLazyLoader reader : databaseReaders.values()) { for (DatabaseReaderLazyLoader reader : databaseReaders.values()) {
reader.close(); reader.close();
} }
@ -90,9 +72,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildDefaults() throws Exception { public void testBuildDefaults() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
@ -109,9 +88,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testSetIgnoreMissing() throws Exception { public void testSetIgnoreMissing() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
@ -129,9 +105,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testCountryBuildDefaults() throws Exception { public void testCountryBuildDefaults() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
@ -150,9 +123,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testAsnBuildDefaults() throws Exception { public void testAsnBuildDefaults() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
@ -171,9 +141,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildTargetField() throws Exception { public void testBuildTargetField() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
@ -185,9 +152,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildDbFile() throws Exception { public void testBuildDbFile() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
@ -201,9 +165,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildWithCountryDbAndAsnFields() throws Exception { public void testBuildWithCountryDbAndAsnFields() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
@ -218,9 +179,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildWithAsnDbAndCityFields() throws Exception { public void testBuildWithAsnDbAndCityFields() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
config.put("field", "_field"); config.put("field", "_field");
@ -235,9 +193,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildNonExistingDbFile() throws Exception { public void testBuildNonExistingDbFile() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>(); Map<String, Object> config = new HashMap<>();
@ -248,9 +203,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildFields() throws Exception { public void testBuildFields() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Set<GeoIpProcessor.Property> properties = EnumSet.noneOf(GeoIpProcessor.Property.class); Set<GeoIpProcessor.Property> properties = EnumSet.noneOf(GeoIpProcessor.Property.class);
@ -275,9 +227,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testBuildIllegalFieldOption() throws Exception { public void testBuildIllegalFieldOption() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000)); GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config1 = new HashMap<>(); Map<String, Object> config1 = new HashMap<>();
@ -295,9 +244,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
} }
public void testLazyLoading() throws Exception { public void testLazyLoading() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
final Path geoIpDir = createTempDir(); final Path geoIpDir = createTempDir();
final Path configDir = createTempDir(); final Path configDir = createTempDir();
final Path geoIpConfigDir = configDir.resolve("ingest-geoip"); final Path geoIpConfigDir = configDir.resolve("ingest-geoip");