Fix GeoIpProcessorFactoryTests on windows ()

Switches windows build to use geoip database loaded on heap instead
of memory mapping it.

Closes 
This commit is contained in:
Igor Motov 2019-08-21 08:45:51 -04:00
parent 47bbd9d9a9
commit 28006fe19f
2 changed files with 10 additions and 54 deletions
modules/ingest-geoip
build.gradle
src/test/java/org/elasticsearch/ingest/geoip

View File

@ -17,6 +17,8 @@
* under the License.
*/
import org.apache.tools.ant.taskdefs.condition.Os
esplugin {
description 'Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database'
classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
@ -64,3 +66,11 @@ thirdPartyAudit.ignoreMissingClasses (
'org.apache.http.impl.client.HttpClientBuilder',
'org.apache.http.util.EntityUtils'
)
test {
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
// Windows cannot cleanup database files properly unless it loads everything on heap.
// See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows for more information
systemProperty 'es.geoip.load_db_on_heap', 'true'
}
}

View File

@ -20,7 +20,6 @@
package org.elasticsearch.ingest.geoip;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.util.Constants;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Randomness;
import org.elasticsearch.index.VersionType;
@ -55,16 +54,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
@BeforeClass
public static void loadDatabaseReaders() throws IOException {
// there are still problems on windows
assumeFalse("https://github.com/elastic/elasticsearch/issues/44552", Constants.WINDOWS);
// Skip setup because Windows cannot cleanup these files properly. The reason is that they are using
// a MappedByteBuffer which will keep the file mappings active until it is garbage-collected. As a consequence,
// the corresponding file appears to be still in use and Windows cannot delete it.
if (Constants.WINDOWS) {
return;
}
final Path geoIpDir = createTempDir();
final Path configDir = createTempDir();
final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
@ -76,13 +65,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
@AfterClass
public static void closeDatabaseReaders() throws IOException {
// Skip setup because Windows cannot cleanup these files properly. The reason is that they are using
// a MappedByteBuffer which will keep the file mappings active until it is garbage-collected. As a consequence,
// the corresponding file appears to be still in use and Windows cannot delete it.
if (Constants.WINDOWS) {
return;
}
for (DatabaseReaderLazyLoader reader : databaseReaders.values()) {
reader.close();
}
@ -90,9 +72,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildDefaults() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
@ -109,9 +88,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testSetIgnoreMissing() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
@ -129,9 +105,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testCountryBuildDefaults() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
@ -150,9 +123,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testAsnBuildDefaults() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
@ -171,9 +141,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildTargetField() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -185,9 +152,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildDbFile() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -201,9 +165,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildWithCountryDbAndAsnFields() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -218,9 +179,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildWithAsnDbAndCityFields() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
config.put("field", "_field");
@ -235,9 +193,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildNonExistingDbFile() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config = new HashMap<>();
@ -248,9 +203,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildFields() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Set<GeoIpProcessor.Property> properties = EnumSet.noneOf(GeoIpProcessor.Property.class);
@ -275,9 +227,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testBuildIllegalFieldOption() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
Map<String, Object> config1 = new HashMap<>();
@ -295,9 +244,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
}
public void testLazyLoading() throws Exception {
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
final Path geoIpDir = createTempDir();
final Path configDir = createTempDir();
final Path geoIpConfigDir = configDir.resolve("ingest-geoip");