From 4fc08aa01933de77dbc5735ffab1b91ac295ece6 Mon Sep 17 00:00:00 2001 From: gmarz Date: Mon, 11 May 2015 10:17:28 -0400 Subject: [PATCH] Revert "Revert "VirtualLock implementation for Windows (mlockall equivalent)"" This reverts commit 5dc8b99365723154670f120e67c7479efdd542da. --- .../elasticsearch/bootstrap/Bootstrap.java | 7 +- .../common/jna/Kernel32Library.java | 90 ++++++++++++++++++- .../org/elasticsearch/common/jna/Natives.java | 39 ++++++++ .../org/elasticsearch/common/jna/SizeT.java | 35 ++++++++ 4 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/elasticsearch/common/jna/SizeT.java diff --git a/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index ea59cd4c657..e4a86baf9ab 100644 --- a/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -20,6 +20,7 @@ package org.elasticsearch.bootstrap; import org.apache.lucene.util.StringHelper; +import org.apache.lucene.util.Constants; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.Version; import org.elasticsearch.common.PidFile; @@ -87,7 +88,11 @@ public class Bootstrap { public static void initializeNatives(boolean mlockAll, boolean ctrlHandler) { // mlockall if requested if (mlockAll) { - Natives.tryMlockall(); + if (Constants.WINDOWS) { + Natives.tryVirtualLock(); + } else { + Natives.tryMlockall(); + } } // check if the user is running as root, and bail diff --git a/src/main/java/org/elasticsearch/common/jna/Kernel32Library.java b/src/main/java/org/elasticsearch/common/jna/Kernel32Library.java index 04549d78f1d..d2b634ae768 100644 --- a/src/main/java/org/elasticsearch/common/jna/Kernel32Library.java +++ b/src/main/java/org/elasticsearch/common/jna/Kernel32Library.java @@ -20,7 +20,7 @@ package org.elasticsearch.common.jna; import com.google.common.collect.ImmutableList; -import com.sun.jna.Native; +import com.sun.jna.*; import com.sun.jna.win32.StdCallLibrary; import org.apache.lucene.util.Constants; @@ -28,6 +28,7 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @@ -134,4 +135,91 @@ public class Kernel32Library { */ boolean handle(int code); } + + + /** + * Memory protection constraints + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366786%28v=vs.85%29.aspx + */ + public static final int PAGE_NOACCESS = 0x0001; + public static final int PAGE_GUARD = 0x0100; + public static final int MEM_COMMIT = 0x1000; + + /** + * Contains information about a range of pages in the virtual address space of a process. + * The VirtualQuery and VirtualQueryEx functions use this structure. + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366775%28v=vs.85%29.aspx + */ + public static class MemoryBasicInformation extends Structure { + public Pointer BaseAddress; + public Pointer AllocationBase; + public NativeLong AllocationProtect; + public SizeT RegionSize; + public NativeLong State; + public NativeLong Protect; + public NativeLong Type; + + @Override + protected List getFieldOrder() { + return Arrays.asList(new String[]{"BaseAddress", "AllocationBase", "AllocationProtect", "RegionSize", "State", "Protect", "Type"}); + } + } + + /** + * Locks the specified region of the process's virtual address space into physical + * memory, ensuring that subsequent access to the region will not incur a page fault. + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366895%28v=vs.85%29.aspx + * + * @param address A pointer to the base address of the region of pages to be locked. + * @param size The size of the region to be locked, in bytes. + * @return true if the function succeeds + */ + public native boolean VirtualLock(Pointer address, SizeT size); + + /** + * Retrieves information about a range of pages within the virtual address space of a specified process. + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/aa366907%28v=vs.85%29.aspx + * + * @param handle A handle to the process whose memory information is queried. + * @param address A pointer to the base address of the region of pages to be queried. + * @param memoryInfo A pointer to a structure in which information about the specified page range is returned. + * @param length The size of the buffer pointed to by the memoryInfo parameter, in bytes. + * @return the actual number of bytes returned in the information buffer. + */ + public native int VirtualQueryEx(Pointer handle, Pointer address, MemoryBasicInformation memoryInfo, int length); + + /** + * Sets the minimum and maximum working set sizes for the specified process. + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms686234%28v=vs.85%29.aspx + * + * @param handle A handle to the process whose working set sizes is to be set. + * @param minSize The minimum working set size for the process, in bytes. + * @param maxSize The maximum working set size for the process, in bytes. + * @return true if the function succeeds. + */ + public native boolean SetProcessWorkingSetSize(Pointer handle, SizeT minSize, SizeT maxSize); + + /** + * Retrieves a pseudo handle for the current process. + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms683179%28v=vs.85%29.aspx + * + * @return a pseudo handle to the current process. + */ + public native Pointer GetCurrentProcess(); + + /** + * Closes an open object handle. + * + * https://msdn.microsoft.com/en-us/library/windows/desktop/ms724211%28v=vs.85%29.aspx + * + * @param handle A valid handle to an open object. + * @return true if the function succeeds. + */ + public native boolean CloseHandle(Pointer handle); } diff --git a/src/main/java/org/elasticsearch/common/jna/Natives.java b/src/main/java/org/elasticsearch/common/jna/Natives.java index b1cea21a954..69f412bda6c 100644 --- a/src/main/java/org/elasticsearch/common/jna/Natives.java +++ b/src/main/java/org/elasticsearch/common/jna/Natives.java @@ -20,10 +20,13 @@ package org.elasticsearch.common.jna; import com.sun.jna.Native; +import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; import org.apache.lucene.util.Constants; import org.elasticsearch.common.jna.Kernel32Library.ConsoleCtrlHandler; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.monitor.jvm.JvmInfo; import java.util.Locale; @@ -75,6 +78,42 @@ public class Natives { } } + public static void tryVirtualLock() + { + Kernel32Library kernel = Kernel32Library.getInstance(); + Pointer process = null; + try { + process = kernel.GetCurrentProcess(); + // By default, Windows limits the number of pages that can be locked. + // Thus, we need to first increase the working set size of the JVM by + // the amount of memory we wish to lock, plus a small overhead (1MB). + SizeT size = new SizeT(JvmInfo.jvmInfo().getMem().getHeapInit().getBytes() + (1024 * 1024)); + if (!kernel.SetProcessWorkingSetSize(process, size, size)) { + logger.warn("Unable to lock JVM memory. Failed to set working set size. Error code " + Native.getLastError()); + } else { + Kernel32Library.MemoryBasicInformation memInfo = new Kernel32Library.MemoryBasicInformation(); + long address = 0; + while (kernel.VirtualQueryEx(process, new Pointer(address), memInfo, memInfo.size()) != 0) { + boolean lockable = memInfo.State.longValue() == Kernel32Library.MEM_COMMIT + && (memInfo.Protect.longValue() & Kernel32Library.PAGE_NOACCESS) != Kernel32Library.PAGE_NOACCESS + && (memInfo.Protect.longValue() & Kernel32Library.PAGE_GUARD) != Kernel32Library.PAGE_GUARD; + if (lockable) { + kernel.VirtualLock(memInfo.BaseAddress, new SizeT(memInfo.RegionSize.longValue())); + } + // Move to the next region + address += memInfo.RegionSize.longValue(); + } + LOCAL_MLOCKALL = true; + } + } catch (UnsatisfiedLinkError e) { + // this will have already been logged by Kernel32Library, no need to repeat it + } finally { + if (process != null) { + kernel.CloseHandle(process); + } + } + } + public static void addConsoleCtrlHandler(ConsoleCtrlHandler handler) { // The console Ctrl handler is necessary on Windows platforms only. if (Constants.WINDOWS) { diff --git a/src/main/java/org/elasticsearch/common/jna/SizeT.java b/src/main/java/org/elasticsearch/common/jna/SizeT.java new file mode 100644 index 00000000000..ab2fcd70552 --- /dev/null +++ b/src/main/java/org/elasticsearch/common/jna/SizeT.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.jna; + +import com.sun.jna.IntegerType; +import com.sun.jna.Native; + +public class SizeT extends IntegerType { + + public SizeT() { + this(0); + } + + public SizeT(long value) { + super(Native.SIZE_T_SIZE, value); + } + +}