src/java.base/share/native/libjli/java.c

rev 54472 : 8222334: java -Xss0 triggers StackOverflowError
Summary: Launcher to use the ThreadStackSize decided by hotpot or system if input is 0
Reviewed-by: dholmes

@@ -202,14 +202,17 @@
  * Running Java code in primordial thread caused many problems. We will
  * create a new thread to invoke JVM. See 6316197 for more information.
  */
 static jlong threadStackSize    = 0;  /* stack size of the new thread */
 static jlong maxHeapSize        = 0;  /* max heap size */
-static jlong initialHeapSize    = 0;  /* inital heap size */
+static jlong initialHeapSize    = 0;  /* initial heap size */
 
 /*
- * A minimum -Xss stack size suitable for all platforms.
+ * A minimum initial-thread stack size suitable for most platforms.
+ * This is the minimum amount of stack needed to load the JVM such
+ * that it can reject a too small -Xss value. If this is too small
+ * JVM initialization would cause a StackOverflowError.
  */
 #ifndef STACK_SIZE_MINIMUM
 #define STACK_SIZE_MINIMUM (64 * KB)
 #endif
 

@@ -932,20 +935,22 @@
         }
     }
     options[numOptions].optionString = str;
     options[numOptions++].extraInfo = info;
 
+    /*
+     * -Xss is used both by the JVM and here to establish the stack size of the thread
+     * created to launch the JVM. In the latter case we need to ensure we don't go
+     * below the minimum stack size allowed. If -Xss is zero that tells the JVM to use
+     * 'default' sizes (either from JVM or system configuration, e.g. 'ulimit -s' on linux),
+     * and is not itself a small stack size that will be rejected. So we ignore -Xss0 here.
+     */
     if (JLI_StrCCmp(str, "-Xss") == 0) {
         jlong tmp;
         if (parse_size(str + 4, &tmp)) {
             threadStackSize = tmp;
-            /*
-             * Make sure the thread stack size is big enough that we won't get a stack
-             * overflow before the JVM startup code can check to make sure the stack
-             * is big enough.
-             */
-            if (threadStackSize < (jlong)STACK_SIZE_MINIMUM) {
+            if (threadStackSize > 0 && threadStackSize < (jlong)STACK_SIZE_MINIMUM) {
                 threadStackSize = STACK_SIZE_MINIMUM;
             }
         }
     }
 

@@ -2320,27 +2325,27 @@
 int
 ContinueInNewThread(InvocationFunctions* ifn, jlong threadStackSize,
                     int argc, char **argv,
                     int mode, char *what, int ret)
 {
-
+    if (threadStackSize == 0) {
     /*
-     * If user doesn't specify stack size, check if VM has a preference.
+        * If the user hasn't specified a non-zero stack size ask the JVM for its default.
+        * A returned 0 means 'use the system default' for a platform, e.g., Windows.
      * Note that HotSpot no longer supports JNI_VERSION_1_1 but it will
      * return its default stack size through the init args structure.
      */
-    if (threadStackSize == 0) {
       struct JDK1_1InitArgs args1_1;
       memset((void*)&args1_1, 0, sizeof(args1_1));
       args1_1.version = JNI_VERSION_1_1;
       ifn->GetDefaultJavaVMInitArgs(&args1_1);  /* ignore return value */
       if (args1_1.javaStackSize > 0) {
          threadStackSize = args1_1.javaStackSize;
       }
     }
 
-    { /* Create a new thread to create JVM and invoke main method */
+    /* Create a new thread to create JVM and invoke main method */
       JavaMainArgs args;
       int rslt;
 
       args.argc = argc;
       args.argv = argv;

@@ -2352,11 +2357,10 @@
       /* If the caller has deemed there is an error we
        * simply return that, otherwise we return the value of
        * the callee
        */
       return (ret != 0) ? ret : rslt;
-    }
 }
 
 static void
 DumpState()
 {