src/java.base/share/native/libjli/java.c

rev 54472 : 8222334: java -Xss0 triggers StackOverflowError
Summary: Launcher to use the ThreadStackSize decided by hotpot or system if input is 0
Reviewed-by: dholmes

@@ -940,12 +940,17 @@
             threadStackSize = tmp;
             /*
              * Make sure the thread stack size is big enough that we won't get a stack
              * overflow before the JVM startup code can check to make sure the stack
              * is big enough.
+             * If the input thread stack size is 0, try to ask HotSpot to determine
+             * a proper value later when creating thread for JavaMain, or use the
+             * system default stack size. VM would create inner threads using its
+             * calculated sizes, forced STACK_SIZE_MINIMUM here may trigger
+             * StackOverflowError in HotSpot. See JDK-8222334 for details.
              */
-            if (threadStackSize < (jlong)STACK_SIZE_MINIMUM) {
+            if (threadStackSize != 0 && threadStackSize < (jlong)STACK_SIZE_MINIMUM) {
                 threadStackSize = STACK_SIZE_MINIMUM;
             }
         }
     }
 

@@ -2333,10 +2338,16 @@
       memset((void*)&args1_1, 0, sizeof(args1_1));
       args1_1.version = JNI_VERSION_1_1;
       ifn->GetDefaultJavaVMInitArgs(&args1_1);  /* ignore return value */
       if (args1_1.javaStackSize > 0) {
          threadStackSize = args1_1.javaStackSize;
+      } else {
+        /*
+         * pthread_create would use system default stack size, e.g.,
+         * ulimit -s shows:
+         * stack size (kbytes, -s) 8192
+         */
       }
     }
 
     { /* Create a new thread to create JVM and invoke main method */
       JavaMainArgs args;