Set direct memory if unable to detect JVM config (#7606)

* Set direct memory if unable to detect JVM config

Java 9 and above prevents us from detecting the maximum available direct
memory.

This change adds a fallback method to use at most 25% of maximum heap
size, which should be a reasonable default.

Unless -XX:MaxDirectMemorySize is set, recent JVMs will default maximum
direct memory to match the maximum heap size, so this should work out of
the box in most cases. For completeness we print instructions in the log
to explain how to adjust settings if necessary.

* skip test rather than succeeding

* reword log message

Co-Authored-By: Himanshu <g.himanshu@gmail.com>

processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java

@@ -52,7 +52,22 @@ public abstract class DruidProcessingConfig extends ExecutorServiceConfig implem
       return computedBufferSizeBytes.get();
     }
 
-    long directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+    long directSizeBytes;
+    try {
+      directSizeBytes = JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+      log.info(
+          "Detected max direct memory size of [%,d] bytes",
+          directSizeBytes
+      );
+    }
+    catch (UnsupportedOperationException e) {
+      // max direct memory defaults to max heap size on recent JDK version, unless set explicitly
+      directSizeBytes = computeMaxMemoryFromMaxHeapSize();
+      log.info(
+          "Defaulting to at most [%,d] bytes (25%% of max heap size) of direct memory for computation buffers",
+          directSizeBytes
+      );
+    }
 
     int numProcessingThreads = getNumThreads();
     int numMergeBuffers = getNumMergeBuffers();
@@ -62,17 +77,20 @@ public abstract class DruidProcessingConfig extends ExecutorServiceConfig implem
     final int computedSizePerBuffer = Math.min(sizePerBuffer, MAX_DEFAULT_PROCESSING_BUFFER_SIZE_BYTES);
     if (computedBufferSizeBytes.compareAndSet(null, computedSizePerBuffer)) {
       log.info(
-          "Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers " +
-          "out of [%,d] max direct memory",
+          "Auto sizing buffers to [%,d] bytes each for [%,d] processing and [%,d] merge buffers",
           computedSizePerBuffer,
           numProcessingThreads,
-          numMergeBuffers,
-          directSizeBytes
+          numMergeBuffers
       );
     }
     return computedSizePerBuffer;
   }
 
+  public static long computeMaxMemoryFromMaxHeapSize()
+  {
+    return Runtime.getRuntime().maxMemory() / 4;
+  }
+
   @Config({"druid.computation.buffer.poolCacheMaxCount", "${base_path}.buffer.poolCacheMaxCount"})
   public int poolCacheMaxCount()
   {

server/src/main/java/org/apache/druid/guice/DruidProcessingModule.java

@@ -157,9 +157,14 @@ public class DruidProcessingModule implements Module
       }
     }
     catch (UnsupportedOperationException e) {
+      log.debug("Checking for direct memory size is not support on this platform: %s", e);
       log.info(
-          "Could not verify that you have enough direct memory, so I hope you do! Error message was: %s",
-          e.getMessage()
+          "Unable to determine max direct memory size. If druid.processing.buffer.sizeBytes is explicitly configured, "
+          + "then make sure to set -XX:MaxDirectMemorySize to at least \"druid.processing.buffer.sizeBytes * "
+          + "(druid.processing.numMergeBuffers[%,d] + druid.processing.numThreads[%,d] + 1)\", "
+          + "or else set -XX:MaxDirectMemorySize to at least 25%% of maximum jvm heap size.",
+          config.getNumMergeBuffers(),
+          config.getNumThreads()
       );
     }
   }

server/src/main/java/org/apache/druid/server/StatusResource.java

@@ -225,7 +225,15 @@ public class StatusResource
       totalMemory = runtime.getTotalHeapSizeBytes();
       freeMemory = runtime.getFreeHeapSizeBytes();
       usedMemory = totalMemory - freeMemory;
-      directMemory = runtime.getDirectMemorySizeBytes();
+
+      long directMemory = -1;
+      try {
+        directMemory = runtime.getDirectMemorySizeBytes();
+      }
+      catch (UnsupportedOperationException ignore) {
+        // querying direct memory is not supported
+      }
+      this.directMemory = directMemory;
     }
 
     @JsonProperty

server/src/test/java/org/apache/druid/guice/DruidProcessingModuleTest.java

@@ -21,6 +21,8 @@ package org.apache.druid.guice;
 
 import com.google.inject.ProvisionException;
 import org.apache.druid.query.DruidProcessingConfig;
+import org.apache.druid.utils.JvmUtils;
+import org.junit.Assume;
 import org.junit.Test;
 
 public class DruidProcessingModuleTest
@@ -29,6 +31,15 @@ public class DruidProcessingModuleTest
   @Test(expected = ProvisionException.class)
   public void testMemoryCheckThrowsException()
   {
+    // JDK 9 and above do not support checking for direct memory size
+    // so this test only validates functionality for Java 8.
+    try {
+      JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes();
+    }
+    catch (UnsupportedOperationException e) {
+      Assume.assumeNoException(e);
+    }
+
     DruidProcessingModule module = new DruidProcessingModule();
     module.getIntermediateResultsPool(new DruidProcessingConfig()
     {