src/os_cpu/linux_x86/vm/os_linux_x86.cpp

rev 8669 : 8197429: Increased stack guard causes segfaults on x86-32
Reviewed-by: dholmes

@@ -890,10 +890,31 @@
  * updates (JDK-8023956).
  */
 void os::workaround_expand_exec_shield_cs_limit() {
 #if defined(IA32)
   size_t page_size = os::vm_page_size();
+
+  /*
+   * JDK-8197429
+   *
+   * Expand the stack mapping to the end of the initial stack before
+   * attempting to install the codebuf.  This is needed because newer
+   * Linux kernels impose a distance of a megabyte between stack
+   * memory and other memory regions.  If we try to install the
+   * codebuf before expanding the stack the installation will appear
+   * to succeed but we'll get a segfault later if we expand the stack
+   * in Java code.
+   *
+   */
+  if (os::is_primordial_thread()) {
+    address limit = Linux::initial_thread_stack_bottom();
+    if (! DisablePrimordialThreadGuardPages) {
+      limit += (StackYellowPages + StackRedPages) * page_size;
+    }
+    os::Linux::expand_stack_to(limit);
+  }
+
   /*
    * Take the highest VA the OS will give us and exec
    *
    * Although using -(pagesz) as mmap hint works on newer kernel as you would
    * think, older variants affected by this work-around don't (search forward only).

@@ -908,10 +929,20 @@
    * we don't have much control or understanding of the address space, just let it slide.
    */
   char* hint = (char*) (Linux::initial_thread_stack_bottom() -
                         ((StackYellowPages + StackRedPages + 1) * page_size));
   char* codebuf = os::attempt_reserve_memory_at(page_size, hint);
+
+  if (codebuf == NULL) {
+    // JDK-8197429: There may be a stack gap of one megabyte between
+    // the limit of the stack and the nearest memory region: this is a
+    // Linux kernel workaround for CVE-2017-1000364.  If we failed to
+    // map our codebuf, try again at an address one megabyte lower.
+    hint -= 1 * M;
+    codebuf = os::attempt_reserve_memory_at(page_size, hint);
+  }
+
   if ( (codebuf == NULL) || (!os::commit_memory(codebuf, page_size, true)) ) {
     return; // No matter, we tried, best effort.
   }
   if (PrintMiscellaneous && (Verbose || WizardMode)) {
      tty->print_cr("[CS limit NX emulation work-around, exec code at: %p]", codebuf);