src/os_cpu/linux_x86/vm/os_linux_x86.cpp

*** 172,181 ****
--- 172,225 ----
    intptr_t* fp;
    ExtendedPC epc = fetch_frame_from_context(ucVoid, &sp, &fp);
    return frame(sp, fp, epc.pc());
  }
  
+ frame os::fetch_frame_from_ucontext(Thread* thread, void* ucVoid) {
+   intptr_t* sp;
+   intptr_t* fp;
+   ExtendedPC epc = os::Linux::fetch_frame_from_ucontext(thread, (ucontext_t*)ucVoid, &sp, &fp);
+   return frame(sp, fp, epc.pc());
+ }
+ 
+ bool os::Linux::get_frame_at_stack_banging_point(JavaThread* thread, ucontext_t* uc, frame* fr) {
+   address pc = (address) os::Linux::ucontext_get_pc(uc);
+   if (Interpreter::contains(pc)) {
+     // interpreter performs stack banging after the fixed frame header has
+     // been generated while the compilers perform it before. To maintain
+     // semantic consistency between interpreted and compiled frames, the
+     // method returns the Java sender of the current frame.
+     *fr = os::fetch_frame_from_ucontext(thread, uc);
+     assert(fr->safe_for_sender(thread), "Safety check");
+     if (!fr->is_first_java_frame()) {
+       *fr = fr->java_sender();
+     }
+   } else {
+     // more complex code with compiled code
+     assert(!Interpreter::contains(pc), "Interpreted methods should have been handled above");
+     CodeBlob* cb = CodeCache::find_blob(pc);
+     if (cb == NULL || !cb->is_nmethod() || cb->is_frame_complete_at(pc)) {
+       // Not sure where the pc points to, fallback to default 
+       // stack overflow handling
+       return false;
+     } else {
+       // in compiled code, the stack banging is performed just after the return pc
+       // has been pushed on the stack
+       intptr_t* fp = os::Linux::ucontext_get_fp(uc);
+       intptr_t* sp = os::Linux::ucontext_get_sp(uc);
+       *fr = frame(sp + 1, fp, (address)*sp);
+       if (!fr->is_java_frame()) {
+         assert(fr->safe_for_sender(thread), "Safety check");
+         assert(!fr->is_first_frame(), "Safety check");
+         *fr = fr->java_sender();
+       }
+     }
+   }
+   assert(fr->is_java_frame(), "Safety check");
+   return true;
+ }
+ 
  // By default, gcc always save frame pointer (%ebp/%rbp) on stack. It may get
  // turned off by -fomit-frame-pointer,
  frame os::get_sender_for_C_frame(frame* fr) {
    return frame(fr->sender_sp(), fr->link(), fr->sender_pc());
  }
*** 302,318 ****
        // check if fault address is within thread stack
        if (addr < thread->stack_base() &&
            addr >= thread->stack_base() - thread->stack_size()) {
          // stack overflow
          if (thread->in_stack_yellow_zone(addr)) {
-           thread->disable_stack_yellow_zone();
            if (thread->thread_state() == _thread_in_Java) {
              // Throw a stack overflow exception.  Guard pages will be reenabled
              // while unwinding the stack.
              stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::STACK_OVERFLOW);
            } else {
              // Thread was in the vm or native code.  Return and try to finish.
              return 1;
            }
          } else if (thread->in_stack_red_zone(addr)) {
            // Fatal red zone violation.  Disable the guard pages and fall through
            // to handle_unexpected_exception way down below.
--- 346,381 ----
        // check if fault address is within thread stack
        if (addr < thread->stack_base() &&
            addr >= thread->stack_base() - thread->stack_size()) {
          // stack overflow
          if (thread->in_stack_yellow_zone(addr)) {
            if (thread->thread_state() == _thread_in_Java) {
+             if (thread->in_stack_reserved_zone(addr)) {
+               frame fr;
+               if (os::Linux::get_frame_at_stack_banging_point(thread, uc, &fr)) {
+                 assert(fr.is_java_frame(), "Must be a Java frame");
+                 frame activation = 
+                   SharedRuntime::look_for_reserved_stack_annotated_method(thread, fr);
+                 if (activation.sp() != NULL) {
+                   thread->disable_stack_reserved_zone();
+                   if (activation.is_interpreted_frame()) {
+                     thread->set_reserved_stack_activation(
+                       activation.fp() + frame::interpreter_frame_initial_sp_offset);
+                   } else {
+                     thread->set_reserved_stack_activation(activation.unextended_sp());
+                   }
+                   return 1;
+                 }
+               }
+             }
              // Throw a stack overflow exception.  Guard pages will be reenabled
              // while unwinding the stack.
+             thread->disable_stack_yellow_zone();
              stub = SharedRuntime::continuation_for_implicit_exception(thread, pc, SharedRuntime::STACK_OVERFLOW);
            } else {
              // Thread was in the vm or native code.  Return and try to finish.
+             thread->disable_stack_yellow_zone();
              return 1;
            }
          } else if (thread->in_stack_red_zone(addr)) {
            // Fatal red zone violation.  Disable the guard pages and fall through
            // to handle_unexpected_exception way down below.
*** 865,875 ****
     *
     * If we are embedded in an app other than launcher (initial != main stack),
     * we don't have much control or understanding of the address space, just let it slide.
     */
    char* hint = (char*) (Linux::initial_thread_stack_bottom() -
!                         ((StackYellowPages + StackRedPages + 1) * page_size));
    char* codebuf = os::attempt_reserve_memory_at(page_size, hint);
    if ( (codebuf == NULL) || (!os::commit_memory(codebuf, page_size, true)) ) {
      return; // No matter, we tried, best effort.
    }
  
--- 928,938 ----
     *
     * If we are embedded in an app other than launcher (initial != main stack),
     * we don't have much control or understanding of the address space, just let it slide.
     */
    char* hint = (char*) (Linux::initial_thread_stack_bottom() -
!                         ((StackReservedPages + StackYellowPages + StackRedPages + 1) * page_size));
    char* codebuf = os::attempt_reserve_memory_at(page_size, hint);
    if ( (codebuf == NULL) || (!os::commit_memory(codebuf, page_size, true)) ) {
      return; // No matter, we tried, best effort.
    }