Fix GeoIpProcessorFactoryTests on windows (#45668)
Switches windows build to use geoip database loaded on heap instead of memory mapping it. Closes #44552
This commit is contained in:
parent
47bbd9d9a9
commit
28006fe19f
modules/ingest-geoip
|
@ -17,6 +17,8 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
import org.apache.tools.ant.taskdefs.condition.Os
|
||||
|
||||
esplugin {
|
||||
description 'Ingest processor that uses looksup geo data based on ip adresses using the Maxmind geo database'
|
||||
classname 'org.elasticsearch.ingest.geoip.IngestGeoIpPlugin'
|
||||
|
@ -64,3 +66,11 @@ thirdPartyAudit.ignoreMissingClasses (
|
|||
'org.apache.http.impl.client.HttpClientBuilder',
|
||||
'org.apache.http.util.EntityUtils'
|
||||
)
|
||||
|
||||
test {
|
||||
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
|
||||
// Windows cannot cleanup database files properly unless it loads everything on heap.
|
||||
// See https://github.com/maxmind/MaxMind-DB-Reader-java#file-lock-on-windows for more information
|
||||
systemProperty 'es.geoip.load_db_on_heap', 'true'
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,7 +20,6 @@
|
|||
package org.elasticsearch.ingest.geoip;
|
||||
|
||||
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
|
||||
import org.apache.lucene.util.Constants;
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.common.Randomness;
|
||||
import org.elasticsearch.index.VersionType;
|
||||
|
@ -55,16 +54,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
|
||||
@BeforeClass
|
||||
public static void loadDatabaseReaders() throws IOException {
|
||||
// there are still problems on windows
|
||||
assumeFalse("https://github.com/elastic/elasticsearch/issues/44552", Constants.WINDOWS);
|
||||
|
||||
// Skip setup because Windows cannot cleanup these files properly. The reason is that they are using
|
||||
// a MappedByteBuffer which will keep the file mappings active until it is garbage-collected. As a consequence,
|
||||
// the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
if (Constants.WINDOWS) {
|
||||
return;
|
||||
}
|
||||
|
||||
final Path geoIpDir = createTempDir();
|
||||
final Path configDir = createTempDir();
|
||||
final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
|
||||
|
@ -76,13 +65,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
|
||||
@AfterClass
|
||||
public static void closeDatabaseReaders() throws IOException {
|
||||
// Skip setup because Windows cannot cleanup these files properly. The reason is that they are using
|
||||
// a MappedByteBuffer which will keep the file mappings active until it is garbage-collected. As a consequence,
|
||||
// the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
if (Constants.WINDOWS) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (DatabaseReaderLazyLoader reader : databaseReaders.values()) {
|
||||
reader.close();
|
||||
}
|
||||
|
@ -90,9 +72,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildDefaults() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
|
@ -109,9 +88,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testSetIgnoreMissing() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
|
@ -129,9 +105,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testCountryBuildDefaults() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
|
@ -150,9 +123,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testAsnBuildDefaults() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
|
@ -171,9 +141,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildTargetField() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
config.put("field", "_field");
|
||||
|
@ -185,9 +152,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildDbFile() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
config.put("field", "_field");
|
||||
|
@ -201,9 +165,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildWithCountryDbAndAsnFields() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
config.put("field", "_field");
|
||||
|
@ -218,9 +179,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildWithAsnDbAndCityFields() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
config.put("field", "_field");
|
||||
|
@ -235,9 +193,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildNonExistingDbFile() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Map<String, Object> config = new HashMap<>();
|
||||
|
@ -248,9 +203,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildFields() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Set<GeoIpProcessor.Property> properties = EnumSet.noneOf(GeoIpProcessor.Property.class);
|
||||
|
@ -275,9 +227,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testBuildIllegalFieldOption() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
|
||||
|
||||
Map<String, Object> config1 = new HashMap<>();
|
||||
|
@ -295,9 +244,6 @@ public class GeoIpProcessorFactoryTests extends ESTestCase {
|
|||
}
|
||||
|
||||
public void testLazyLoading() throws Exception {
|
||||
// This test uses a MappedByteBuffer which will keep the file mappings active until it is garbage-collected.
|
||||
// As a consequence, the corresponding file appears to be still in use and Windows cannot delete it.
|
||||
assumeFalse("windows deletion behavior is asinine", Constants.WINDOWS);
|
||||
final Path geoIpDir = createTempDir();
|
||||
final Path configDir = createTempDir();
|
||||
final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
|
||||
|
|
Loading…
Reference in New Issue