1 /*
   2  * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include <fcntl.h>
  26 #include "precompiled.hpp"
  27 #include "code/codeCache.hpp"
  28 #include "compiler/compileBroker.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "logging/logConfiguration.hpp"
  32 #include "prims/whitebox.hpp"
  33 #include "runtime/arguments.hpp"
  34 #include "runtime/atomic.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/init.hpp"
  37 #include "runtime/os.hpp"
  38 #include "runtime/thread.inline.hpp"
  39 #include "runtime/vmThread.hpp"
  40 #include "runtime/vm_operations.hpp"
  41 #include "services/memTracker.hpp"
  42 #include "trace/traceMacros.hpp"
  43 #include "utilities/debug.hpp"
  44 #include "utilities/decoder.hpp"
  45 #include "utilities/defaultStream.hpp"
  46 #include "utilities/errorReporter.hpp"
  47 #include "utilities/events.hpp"
  48 #include "utilities/vmError.hpp"
  49 
  50 // List of environment variables that should be reported in error log file.
  51 const char *env_list[] = {
  52   // All platforms
  53   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  54   "JAVA_COMPILER", "PATH", "USERNAME",
  55 
  56   // Env variables that are defined on Solaris/Linux/BSD
  57   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  58   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  59 
  60   // defined on Linux
  61   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  62 
  63   // defined on Darwin
  64   "DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH",
  65   "DYLD_FRAMEWORK_PATH", "DYLD_FALLBACK_FRAMEWORK_PATH",
  66   "DYLD_INSERT_LIBRARIES",
  67 
  68   // defined on Windows
  69   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  70 
  71   (const char *)0
  72 };
  73 
  74 // A simple parser for -XX:OnError, usage:
  75 //  ptr = OnError;
  76 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
  77 //     ... ...
  78 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
  79   if (ptr == NULL || *ptr == NULL) return NULL;
  80 
  81   const char* cmd = *ptr;
  82 
  83   // skip leading blanks or ';'
  84   while (*cmd == ' ' || *cmd == ';') cmd++;
  85 
  86   if (*cmd == '\0') return NULL;
  87 
  88   const char * cmdend = cmd;
  89   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
  90 
  91   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
  92 
  93   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
  94   return buf;
  95 }
  96 
  97 static void print_bug_submit_message(outputStream *out, Thread *thread) {
  98   if (out == NULL) return;
  99   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 100   out->print_raw   ("#   ");
 101   out->print_raw_cr(Arguments::java_vendor_url_bug());
 102   // If the crash is in native code, encourage user to submit a bug to the
 103   // provider of that code.
 104   if (thread && thread->is_Java_thread() &&
 105       !thread->is_hidden_from_external_view()) {
 106     JavaThread* jt = (JavaThread*)thread;
 107     if (jt->thread_state() == _thread_in_native) {
 108       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 109     }
 110   }
 111   out->print_raw_cr("#");
 112 }
 113 
 114 bool VMError::coredump_status;
 115 char VMError::coredump_message[O_BUFLEN];
 116 
 117 void VMError::record_coredump_status(const char* message, bool status) {
 118   coredump_status = status;
 119   strncpy(coredump_message, message, sizeof(coredump_message));
 120   coredump_message[sizeof(coredump_message)-1] = 0;
 121 }
 122 
 123 // Return a string to describe the error
 124 char* VMError::error_string(char* buf, int buflen) {
 125   char signame_buf[64];
 126   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 127 
 128   if (signame) {
 129     jio_snprintf(buf, buflen,
 130                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 131                  signame, _id, _pc,
 132                  os::current_process_id(), os::current_thread_id());
 133   } else if (_filename != NULL && _lineno > 0) {
 134     // skip directory names
 135     char separator = os::file_separator()[0];
 136     const char *p = strrchr(_filename, separator);
 137     int n = jio_snprintf(buf, buflen,
 138                          "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT,
 139                          p ? p + 1 : _filename, _lineno,
 140                          os::current_process_id(), os::current_thread_id());
 141     if (n >= 0 && n < buflen && _message) {
 142       if (strlen(_detail_msg) > 0) {
 143         jio_snprintf(buf + n, buflen - n, "%s%s: %s",
 144         os::line_separator(), _message, _detail_msg);
 145       } else {
 146         jio_snprintf(buf + n, buflen - n, "%sError: %s",
 147                      os::line_separator(), _message);
 148       }
 149     }
 150   } else {
 151     jio_snprintf(buf, buflen,
 152                  "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 153                  _id, os::current_process_id(), os::current_thread_id());
 154   }
 155 
 156   return buf;
 157 }
 158 
 159 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 160                                 char* buf, int buflen, bool verbose) {
 161 #ifdef ZERO
 162   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 163     // StackFrameStream uses the frame anchor, which may not have
 164     // been set up.  This can be done at any time in Zero, however,
 165     // so if it hasn't been set up then we just set it up now and
 166     // clear it again when we're done.
 167     bool has_last_Java_frame = jt->has_last_Java_frame();
 168     if (!has_last_Java_frame)
 169       jt->set_last_Java_frame();
 170     st->print("Java frames:");
 171 
 172     // If the top frame is a Shark frame and the frame anchor isn't
 173     // set up then it's possible that the information in the frame
 174     // is garbage: it could be from a previous decache, or it could
 175     // simply have never been written.  So we print a warning...
 176     StackFrameStream sfs(jt);
 177     if (!has_last_Java_frame && !sfs.is_done()) {
 178       if (sfs.current()->zeroframe()->is_shark_frame()) {
 179         st->print(" (TOP FRAME MAY BE JUNK)");
 180       }
 181     }
 182     st->cr();
 183 
 184     // Print the frames
 185     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 186       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 187       st->cr();
 188     }
 189 
 190     // Reset the frame anchor if necessary
 191     if (!has_last_Java_frame)
 192       jt->reset_last_Java_frame();
 193   }
 194 #else
 195   if (jt->has_last_Java_frame()) {
 196     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 197     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 198       sfs.current()->print_on_error(st, buf, buflen, verbose);
 199       st->cr();
 200     }
 201   }
 202 #endif // ZERO
 203 }
 204 
 205 static void print_oom_reasons(outputStream* st) {
 206   st->print_cr("# Possible reasons:");
 207   st->print_cr("#   The system is out of physical RAM or swap space");
 208   if (UseCompressedOops) {
 209     st->print_cr("#   The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap");
 210   }
 211   if (LogBytesPerWord == 2) {
 212     st->print_cr("#   In 32 bit mode, the process size limit was hit");
 213   }
 214   st->print_cr("# Possible solutions:");
 215   st->print_cr("#   Reduce memory load on the system");
 216   st->print_cr("#   Increase physical memory or swap space");
 217   st->print_cr("#   Check if swap backing store is full");
 218   if (LogBytesPerWord == 2) {
 219     st->print_cr("#   Use 64 bit Java on a 64 bit OS");
 220   }
 221   st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
 222   st->print_cr("#   Decrease number of Java threads");
 223   st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
 224   st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
 225   if (UseCompressedOops) {
 226     switch (Universe::narrow_oop_mode()) {
 227       case Universe::UnscaledNarrowOop:
 228         st->print_cr("#   JVM is running with Unscaled Compressed Oops mode in which the Java heap is");
 229         st->print_cr("#     placed in the first 4GB address space. The Java Heap base address is the");
 230         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 231         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 4GB virtual address.");
 232         break;
 233       case Universe::ZeroBasedNarrowOop:
 234         st->print_cr("#   JVM is running with Zero Based Compressed Oops mode in which the Java heap is");
 235         st->print_cr("#     placed in the first 32GB address space. The Java Heap base address is the");
 236         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 237         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 32GB virtual address.");
 238         break;
 239     }
 240   }
 241   st->print_cr("# This output file may be truncated or incomplete.");
 242 }
 243 
 244 static const char* gc_mode() {
 245   if (UseG1GC)            return "g1 gc";
 246   if (UseParallelGC)      return "parallel gc";
 247   if (UseConcMarkSweepGC) return "concurrent mark sweep gc";
 248   if (UseSerialGC)        return "serial gc";
 249   if (UseEpsilonGC)       return "epsilon gc";
 250   return "ERROR in GC mode";
 251 }
 252 
 253 static void report_vm_version(outputStream* st, char* buf, int buflen) {
 254    // VM version
 255    st->print_cr("#");
 256    JDK_Version::current().to_string(buf, buflen);
 257    const char* runtime_name = JDK_Version::runtime_name() != NULL ?
 258                                 JDK_Version::runtime_name() : "";
 259    const char* runtime_version = JDK_Version::runtime_version() != NULL ?
 260                                    JDK_Version::runtime_version() : "";
 261    const char* jdk_debug_level = Abstract_VM_Version::printable_jdk_debug_level() != NULL ?
 262                                    Abstract_VM_Version::printable_jdk_debug_level() : "";
 263 
 264    st->print_cr("# JRE version: %s (%s) (%sbuild %s)", runtime_name, buf,
 265                  jdk_debug_level, runtime_version);
 266 
 267    // This is the long version with some default settings added
 268    st->print_cr("# Java VM: %s (%s%s, %s%s%s%s%s, %s, %s)",
 269                  Abstract_VM_Version::vm_name(),
 270                  jdk_debug_level,
 271                  Abstract_VM_Version::vm_release(),
 272                  Abstract_VM_Version::vm_info_string(),
 273                  TieredCompilation ? ", tiered" : "",
 274 #if INCLUDE_JVMCI
 275                  EnableJVMCI ? ", jvmci" : "",
 276                  UseJVMCICompiler ? ", jvmci compiler" : "",
 277 #else
 278                  "", "",
 279 #endif
 280                  UseCompressedOops ? ", compressed oops" : "",
 281                  gc_mode(),
 282                  Abstract_VM_Version::vm_platform_string()
 283                );
 284 }
 285 
 286 // This is the main function to report a fatal error. Only one thread can
 287 // call this function, so we don't need to worry about MT-safety. But it's
 288 // possible that the error handler itself may crash or die on an internal
 289 // error, for example, when the stack/heap is badly damaged. We must be
 290 // able to handle recursive errors that happen inside error handler.
 291 //
 292 // Error reporting is done in several steps. If a crash or internal error
 293 // occurred when reporting an error, the nested signal/exception handler
 294 // can skip steps that are already (or partially) done. Error reporting will
 295 // continue from the next step. This allows us to retrieve and print
 296 // information that may be unsafe to get after a fatal error. If it happens,
 297 // you may find nested report_and_die() frames when you look at the stack
 298 // in a debugger.
 299 //
 300 // In general, a hang in error handler is much worse than a crash or internal
 301 // error, as it's harder to recover from a hang. Deadlock can happen if we
 302 // try to grab a lock that is already owned by current thread, or if the
 303 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 304 // error handler and all the functions it called should avoid grabbing any
 305 // lock. An important thing to notice is that memory allocation needs a lock.
 306 //
 307 // We should avoid using large stack allocated buffers. Many errors happen
 308 // when stack space is already low. Making things even worse is that there
 309 // could be nested report_and_die() calls on stack (see above). Only one
 310 // thread can report error, so large buffers are statically allocated in data
 311 // segment.
 312 
 313 int          VMError::_current_step;
 314 const char*  VMError::_current_step_info;
 315 
 316 void VMError::report(outputStream* st, bool _verbose) {
 317 
 318 # define BEGIN if (_current_step == 0) { _current_step = __LINE__;
 319 # define STEP(s) } if (_current_step < __LINE__) { _current_step = __LINE__; _current_step_info = s;
 320 # define END }
 321 
 322   // don't allocate large buffer on stack
 323   static char buf[O_BUFLEN];
 324 
 325   BEGIN
 326 
 327   STEP("printing fatal error message")
 328 
 329     st->print_cr("#");
 330     if (should_report_bug(_id)) {
 331       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 332     } else {
 333       st->print_cr("# There is insufficient memory for the Java "
 334                    "Runtime Environment to continue.");
 335     }
 336 
 337 #ifndef PRODUCT
 338   // Error handler self tests
 339 
 340   // test secondary error handling. Test it twice, to test that resetting
 341   // error handler after a secondary crash works.
 342   STEP("test secondary crash 1")
 343     if (_verbose && TestCrashInErrorHandler != 0) {
 344       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 345         TestCrashInErrorHandler);
 346       controlled_crash(TestCrashInErrorHandler);
 347     }
 348 
 349   STEP("test secondary crash 2")
 350     if (_verbose && TestCrashInErrorHandler != 0) {
 351       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 352         TestCrashInErrorHandler);
 353       controlled_crash(TestCrashInErrorHandler);
 354     }
 355 
 356   STEP("test safefetch in error handler")
 357     // test whether it is safe to use SafeFetch32 in Crash Handler. Test twice
 358     // to test that resetting the signal handler works correctly.
 359     if (_verbose && TestSafeFetchInErrorHandler) {
 360       st->print_cr("Will test SafeFetch...");
 361       if (CanUseSafeFetch32()) {
 362         int* const invalid_pointer = (int*) get_segfault_address();
 363         const int x = 0x76543210;
 364         int i1 = SafeFetch32(invalid_pointer, x);
 365         int i2 = SafeFetch32(invalid_pointer, x);
 366         if (i1 == x && i2 == x) {
 367           st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern
 368         } else {
 369           st->print_cr("??");
 370         }
 371       } else {
 372         st->print_cr("not possible; skipped.");
 373       }
 374     }
 375 #endif // PRODUCT
 376 
 377   STEP("printing type of error")
 378 
 379      switch(_id) {
 380        case OOM_MALLOC_ERROR:
 381        case OOM_MMAP_ERROR:
 382          if (_size) {
 383            st->print("# Native memory allocation ");
 384            st->print((_id == (int)OOM_MALLOC_ERROR) ? "(malloc) failed to allocate " :
 385                                                  "(mmap) failed to map ");
 386            jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
 387            st->print("%s", buf);
 388            st->print(" bytes");
 389            if (strlen(_detail_msg) > 0) {
 390              st->print(" for ");
 391              st->print("%s", _detail_msg);
 392            }
 393            st->cr();
 394          } else {
 395            if (strlen(_detail_msg) > 0) {
 396              st->print("# ");
 397              st->print_cr("%s", _detail_msg);
 398            }
 399          }
 400          // In error file give some solutions
 401          if (_verbose) {
 402            print_oom_reasons(st);
 403          } else {
 404            return;  // that's enough for the screen
 405          }
 406          break;
 407        case INTERNAL_ERROR:
 408        default:
 409          break;
 410      }
 411 
 412   STEP("printing exception/signal name")
 413 
 414      st->print_cr("#");
 415      st->print("#  ");
 416      // Is it an OS exception/signal?
 417      if (os::exception_name(_id, buf, sizeof(buf))) {
 418        st->print("%s", buf);
 419        st->print(" (0x%x)", _id);                // signal number
 420        st->print(" at pc=" PTR_FORMAT, p2i(_pc));
 421      } else {
 422        if (should_report_bug(_id)) {
 423          st->print("Internal Error");
 424        } else {
 425          st->print("Out of Memory Error");
 426        }
 427        if (_filename != NULL && _lineno > 0) {
 428 #ifdef PRODUCT
 429          // In product mode chop off pathname?
 430          char separator = os::file_separator()[0];
 431          const char *p = strrchr(_filename, separator);
 432          const char *file = p ? p+1 : _filename;
 433 #else
 434          const char *file = _filename;
 435 #endif
 436          st->print(" (%s:%d)", file, _lineno);
 437        } else {
 438          st->print(" (0x%x)", _id);
 439        }
 440      }
 441 
 442   STEP("printing current thread and pid")
 443 
 444      // process id, thread id
 445      st->print(", pid=%d", os::current_process_id());
 446      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 447      st->cr();
 448 
 449   STEP("printing error message")
 450 
 451      if (should_report_bug(_id)) {  // already printed the message.
 452        // error message
 453        if (strlen(_detail_msg) > 0) {
 454          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
 455        } else if (_message) {
 456          st->print_cr("#  Error: %s", _message);
 457        }
 458      }
 459 
 460   STEP("printing Java version string")
 461 
 462      report_vm_version(st, buf, sizeof(buf));
 463 
 464   STEP("printing problematic frame")
 465 
 466      // Print current frame if we have a context (i.e. it's a crash)
 467      if (_context) {
 468        st->print_cr("# Problematic frame:");
 469        st->print("# ");
 470        frame fr = os::fetch_frame_from_context(_context);
 471        fr.print_on_error(st, buf, sizeof(buf));
 472        st->cr();
 473        st->print_cr("#");
 474      }
 475 
 476   STEP("printing core file information")
 477     st->print("# ");
 478     if (CreateCoredumpOnCrash) {
 479       if (coredump_status) {
 480         st->print("Core dump will be written. Default location: %s", coredump_message);
 481       } else {
 482         st->print("No core dump will be written. %s", coredump_message);
 483       }
 484     } else {
 485       st->print("CreateCoredumpOnCrash turned off, no core file dumped");
 486     }
 487     st->cr();
 488     st->print_cr("#");
 489 
 490   STEP("printing bug submit message")
 491 
 492      if (should_report_bug(_id) && _verbose) {
 493        print_bug_submit_message(st, _thread);
 494      }
 495 
 496   STEP("printing summary")
 497 
 498      if (_verbose) {
 499        st->cr();
 500        st->print_cr("---------------  S U M M A R Y ------------");
 501        st->cr();
 502      }
 503 
 504   STEP("printing VM option summary")
 505 
 506      if (_verbose) {
 507        // VM options
 508        Arguments::print_summary_on(st);
 509        st->cr();
 510      }
 511 
 512   STEP("printing summary machine and OS info")
 513 
 514      if (_verbose) {
 515        os::print_summary_info(st, buf, sizeof(buf));
 516      }
 517 
 518 
 519   STEP("printing date and time")
 520 
 521      if (_verbose) {
 522        os::print_date_and_time(st, buf, sizeof(buf));
 523      }
 524 
 525   STEP("printing thread")
 526 
 527      if (_verbose) {
 528        st->cr();
 529        st->print_cr("---------------  T H R E A D  ---------------");
 530        st->cr();
 531      }
 532 
 533   STEP("printing current thread")
 534 
 535      // current thread
 536      if (_verbose) {
 537        if (_thread) {
 538          st->print("Current thread (" PTR_FORMAT "):  ", p2i(_thread));
 539          _thread->print_on_error(st, buf, sizeof(buf));
 540          st->cr();
 541        } else {
 542          st->print_cr("Current thread is native thread");
 543        }
 544        st->cr();
 545      }
 546 
 547   STEP("printing current compile task")
 548 
 549      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 550         CompilerThread* t = (CompilerThread*)_thread;
 551         if (t->task()) {
 552            st->cr();
 553            st->print_cr("Current CompileTask:");
 554            t->task()->print_line_on_error(st, buf, sizeof(buf));
 555            st->cr();
 556         }
 557      }
 558 
 559 
 560   STEP("printing stack bounds")
 561 
 562      if (_verbose) {
 563        st->print("Stack: ");
 564 
 565        address stack_top;
 566        size_t stack_size;
 567 
 568        if (_thread) {
 569           stack_top = _thread->stack_base();
 570           stack_size = _thread->stack_size();
 571        } else {
 572           stack_top = os::current_stack_base();
 573           stack_size = os::current_stack_size();
 574        }
 575 
 576        address stack_bottom = stack_top - stack_size;
 577        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", p2i(stack_bottom), p2i(stack_top));
 578 
 579        frame fr = _context ? os::fetch_frame_from_context(_context)
 580                            : os::current_frame();
 581 
 582        if (fr.sp()) {
 583          st->print(",  sp=" PTR_FORMAT, p2i(fr.sp()));
 584          size_t free_stack_size = pointer_delta(fr.sp(), stack_bottom, 1024);
 585          st->print(",  free space=" SIZE_FORMAT "k", free_stack_size);
 586        }
 587 
 588        st->cr();
 589      }
 590 
 591   STEP("printing native stack")
 592 
 593    if (_verbose) {
 594      if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) {
 595        // We have printed the native stack in platform-specific code
 596        // Windows/x64 needs special handling.
 597      } else {
 598        frame fr = _context ? os::fetch_frame_from_context(_context)
 599                            : os::current_frame();
 600 
 601        print_native_stack(st, fr, _thread, buf, sizeof(buf));
 602      }
 603    }
 604 
 605   STEP("printing Java stack")
 606 
 607      if (_verbose && _thread && _thread->is_Java_thread()) {
 608        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 609      }
 610 
 611   STEP("printing target Java thread stack")
 612 
 613      // printing Java thread stack trace if it is involved in GC crash
 614      if (_verbose && _thread && (_thread->is_Named_thread())) {
 615        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 616        if (jt != NULL) {
 617          st->print_cr("JavaThread " PTR_FORMAT " (nid = %d) was being processed", p2i(jt), jt->osthread()->thread_id());
 618          print_stack_trace(st, jt, buf, sizeof(buf), true);
 619        }
 620      }
 621 
 622   STEP("printing siginfo")
 623 
 624      // signal no, signal code, address that caused the fault
 625      if (_verbose && _siginfo) {
 626        st->cr();
 627        os::print_siginfo(st, _siginfo);
 628        st->cr();
 629      }
 630 
 631   STEP("CDS archive access warning")
 632 
 633      // Print an explicit hint if we crashed on access to the CDS archive.
 634      if (_verbose && _siginfo) {
 635        check_failing_cds_access(st, _siginfo);
 636        st->cr();
 637      }
 638 
 639   STEP("printing register info")
 640 
 641      // decode register contents if possible
 642      if (_verbose && _context && Universe::is_fully_initialized()) {
 643        os::print_register_info(st, _context);
 644        st->cr();
 645      }
 646 
 647   STEP("printing registers, top of stack, instructions near pc")
 648 
 649      // registers, top of stack, instructions near pc
 650      if (_verbose && _context) {
 651        os::print_context(st, _context);
 652        st->cr();
 653      }
 654 
 655   STEP("printing code blob if possible")
 656 
 657      if (_verbose && _context) {
 658        CodeBlob* cb = CodeCache::find_blob(_pc);
 659        if (cb != NULL) {
 660          if (Interpreter::contains(_pc)) {
 661            // The interpreter CodeBlob is very large so try to print the codelet instead.
 662            InterpreterCodelet* codelet = Interpreter::codelet_containing(_pc);
 663            if (codelet != NULL) {
 664              codelet->print_on(st);
 665              Disassembler::decode(codelet->code_begin(), codelet->code_end(), st);
 666            }
 667          } else {
 668            StubCodeDesc* desc = StubCodeDesc::desc_for(_pc);
 669            if (desc != NULL) {
 670              desc->print_on(st);
 671              Disassembler::decode(desc->begin(), desc->end(), st);
 672            } else {
 673              Disassembler::decode(cb, st);
 674              st->cr();
 675            }
 676          }
 677        }
 678      }
 679 
 680   STEP("printing VM operation")
 681 
 682      if (_verbose && _thread && _thread->is_VM_thread()) {
 683         VMThread* t = (VMThread*)_thread;
 684         VM_Operation* op = t->vm_operation();
 685         if (op) {
 686           op->print_on_error(st);
 687           st->cr();
 688           st->cr();
 689         }
 690      }
 691 
 692   STEP("printing process")
 693 
 694      if (_verbose) {
 695        st->cr();
 696        st->print_cr("---------------  P R O C E S S  ---------------");
 697        st->cr();
 698      }
 699 
 700   STEP("printing all threads")
 701 
 702      // all threads
 703      if (_verbose && _thread) {
 704        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 705        st->cr();
 706      }
 707 
 708   STEP("printing VM state")
 709 
 710      if (_verbose) {
 711        // Safepoint state
 712        st->print("VM state:");
 713 
 714        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 715        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 716        else st->print("not at safepoint");
 717 
 718        // Also see if error occurred during initialization or shutdown
 719        if (!Universe::is_fully_initialized()) {
 720          st->print(" (not fully initialized)");
 721        } else if (VM_Exit::vm_exited()) {
 722          st->print(" (shutting down)");
 723        } else {
 724          st->print(" (normal execution)");
 725        }
 726        st->cr();
 727        st->cr();
 728      }
 729 
 730   STEP("printing owned locks on error")
 731 
 732      // mutexes/monitors that currently have an owner
 733      if (_verbose) {
 734        print_owned_locks_on_error(st);
 735        st->cr();
 736      }
 737 
 738   STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
 739 
 740      if (_verbose && Exceptions::has_exception_counts()) {
 741        st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 742        Exceptions::print_exception_counts_on_error(st);
 743        st->cr();
 744      }
 745 
 746   STEP("printing compressed oops mode")
 747 
 748      if (_verbose && UseCompressedOops) {
 749        Universe::print_compressed_oops_mode(st);
 750        if (UseCompressedClassPointers) {
 751          Metaspace::print_compressed_class_space(st);
 752        }
 753        st->cr();
 754      }
 755 
 756   STEP("printing heap information")
 757 
 758      if (_verbose && Universe::is_fully_initialized()) {
 759        Universe::heap()->print_on_error(st);
 760        st->cr();
 761        st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
 762        st->cr();
 763      }
 764 
 765   STEP("printing code cache information")
 766 
 767      if (_verbose && Universe::is_fully_initialized()) {
 768        // print code cache information before vm abort
 769        CodeCache::print_summary(st);
 770        st->cr();
 771      }
 772 
 773   STEP("printing ring buffers")
 774 
 775      if (_verbose) {
 776        Events::print_all(st);
 777        st->cr();
 778      }
 779 
 780   STEP("printing dynamic libraries")
 781 
 782      if (_verbose) {
 783        // dynamic libraries, or memory map
 784        os::print_dll_info(st);
 785        st->cr();
 786      }
 787 
 788   STEP("printing VM options")
 789 
 790      if (_verbose) {
 791        // VM options
 792        Arguments::print_on(st);
 793        st->cr();
 794      }
 795 
 796   STEP("printing warning if internal testing API used")
 797 
 798      if (WhiteBox::used()) {
 799        st->print_cr("Unsupported internal testing APIs have been used.");
 800        st->cr();
 801      }
 802 
 803   STEP("printing log configuration")
 804     if (_verbose){
 805       st->print_cr("Logging:");
 806       LogConfiguration::describe_current_configuration(st);
 807       st->cr();
 808     }
 809 
 810   STEP("printing all environment variables")
 811 
 812      if (_verbose) {
 813        os::print_environment_variables(st, env_list);
 814        st->cr();
 815      }
 816 
 817   STEP("printing signal handlers")
 818 
 819      if (_verbose) {
 820        os::print_signal_handlers(st, buf, sizeof(buf));
 821        st->cr();
 822      }
 823 
 824   STEP("Native Memory Tracking")
 825      if (_verbose) {
 826        MemTracker::error_report(st);
 827      }
 828 
 829   STEP("printing system")
 830 
 831      if (_verbose) {
 832        st->cr();
 833        st->print_cr("---------------  S Y S T E M  ---------------");
 834        st->cr();
 835      }
 836 
 837   STEP("printing OS information")
 838 
 839      if (_verbose) {
 840        os::print_os_info(st);
 841        st->cr();
 842      }
 843 
 844   STEP("printing CPU info")
 845      if (_verbose) {
 846        os::print_cpu_info(st, buf, sizeof(buf));
 847        st->cr();
 848      }
 849 
 850   STEP("printing memory info")
 851 
 852      if (_verbose) {
 853        os::print_memory_info(st);
 854        st->cr();
 855      }
 856 
 857   STEP("printing internal vm info")
 858 
 859      if (_verbose) {
 860        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 861        st->cr();
 862      }
 863 
 864   // print a defined marker to show that error handling finished correctly.
 865   STEP("printing end marker")
 866 
 867      if (_verbose) {
 868        st->print_cr("END.");
 869      }
 870 
 871   END
 872 
 873 # undef BEGIN
 874 # undef STEP
 875 # undef END
 876 }
 877 
 878 // Report for the vm_info_cmd. This prints out the information above omitting
 879 // crash and thread specific information.  If output is added above, it should be added
 880 // here also, if it is safe to call during a running process.
 881 void VMError::print_vm_info(outputStream* st) {
 882 
 883   char buf[O_BUFLEN];
 884   report_vm_version(st, buf, sizeof(buf));
 885 
 886   // STEP("printing summary")
 887 
 888   st->cr();
 889   st->print_cr("---------------  S U M M A R Y ------------");
 890   st->cr();
 891 
 892   // STEP("printing VM option summary")
 893 
 894   // VM options
 895   Arguments::print_summary_on(st);
 896   st->cr();
 897 
 898   // STEP("printing summary machine and OS info")
 899 
 900   os::print_summary_info(st, buf, sizeof(buf));
 901 
 902   // STEP("printing date and time")
 903 
 904   os::print_date_and_time(st, buf, sizeof(buf));
 905 
 906   // Skip: STEP("printing thread")
 907 
 908   // STEP("printing process")
 909 
 910   st->cr();
 911   st->print_cr("---------------  P R O C E S S  ---------------");
 912   st->cr();
 913 
 914   // STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
 915 
 916   if (Exceptions::has_exception_counts()) {
 917     st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 918     Exceptions::print_exception_counts_on_error(st);
 919     st->cr();
 920   }
 921 
 922   // STEP("printing compressed oops mode")
 923 
 924   if (UseCompressedOops) {
 925     Universe::print_compressed_oops_mode(st);
 926     if (UseCompressedClassPointers) {
 927       Metaspace::print_compressed_class_space(st);
 928     }
 929     st->cr();
 930   }
 931 
 932   // STEP("printing heap information")
 933 
 934   if (Universe::is_fully_initialized()) {
 935     MutexLocker hl(Heap_lock);
 936     Universe::heap()->print_on_error(st);
 937     st->cr();
 938     st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
 939     st->cr();
 940   }
 941 
 942   // STEP("printing code cache information")
 943 
 944   if (Universe::is_fully_initialized()) {
 945     // print code cache information before vm abort
 946     CodeCache::print_summary(st);
 947     st->cr();
 948   }
 949 
 950   // STEP("printing ring buffers")
 951 
 952   Events::print_all(st);
 953   st->cr();
 954 
 955   // STEP("printing dynamic libraries")
 956 
 957   // dynamic libraries, or memory map
 958   os::print_dll_info(st);
 959   st->cr();
 960 
 961   // STEP("printing VM options")
 962 
 963   // VM options
 964   Arguments::print_on(st);
 965   st->cr();
 966 
 967   // STEP("printing warning if internal testing API used")
 968 
 969   if (WhiteBox::used()) {
 970     st->print_cr("Unsupported internal testing APIs have been used.");
 971     st->cr();
 972   }
 973 
 974   // STEP("printing log configuration")
 975   st->print_cr("Logging:");
 976   LogConfiguration::describe(st);
 977   st->cr();
 978 
 979   // STEP("printing all environment variables")
 980 
 981   os::print_environment_variables(st, env_list);
 982   st->cr();
 983 
 984   // STEP("printing signal handlers")
 985 
 986   os::print_signal_handlers(st, buf, sizeof(buf));
 987   st->cr();
 988 
 989   // STEP("Native Memory Tracking")
 990 
 991   MemTracker::error_report(st);
 992 
 993   // STEP("printing system")
 994 
 995   st->cr();
 996   st->print_cr("---------------  S Y S T E M  ---------------");
 997   st->cr();
 998 
 999   // STEP("printing OS information")
1000 
1001   os::print_os_info(st);
1002   st->cr();
1003 
1004   // STEP("printing CPU info")
1005 
1006   os::print_cpu_info(st, buf, sizeof(buf));
1007   st->cr();
1008 
1009   // STEP("printing memory info")
1010 
1011   os::print_memory_info(st);
1012   st->cr();
1013 
1014   // STEP("printing internal vm info")
1015 
1016   st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
1017   st->cr();
1018 
1019   // print a defined marker to show that error handling finished correctly.
1020   // STEP("printing end marker")
1021 
1022   st->print_cr("END.");
1023 }
1024 
1025 volatile intptr_t VMError::first_error_tid = -1;
1026 
1027 // An error could happen before tty is initialized or after it has been
1028 // destroyed.
1029 // Please note: to prevent large stack allocations, the log- and
1030 // output-stream use a global scratch buffer for format printing.
1031 // (see VmError::report_and_die(). Access to those streams is synchronized
1032 // in  VmError::report_and_die() - there is only one reporting thread at
1033 // any given time.
1034 fdStream VMError::out(defaultStream::output_fd());
1035 fdStream VMError::log; // error log used by VMError::report_and_die()
1036 
1037 /** Expand a pattern into a buffer starting at pos and open a file using constructed path */
1038 static int expand_and_open(const char* pattern, char* buf, size_t buflen, size_t pos) {
1039   int fd = -1;
1040   if (Arguments::copy_expand_pid(pattern, strlen(pattern), &buf[pos], buflen - pos)) {
1041     // the O_EXCL flag will cause the open to fail if the file exists
1042     fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0666);
1043   }
1044   return fd;
1045 }
1046 
1047 /**
1048  * Construct file name for a log file and return it's file descriptor.
1049  * Name and location depends on pattern, default_pattern params and access
1050  * permissions.
1051  */
1052 static int prepare_log_file(const char* pattern, const char* default_pattern, char* buf, size_t buflen) {
1053   int fd = -1;
1054 
1055   // If possible, use specified pattern to construct log file name
1056   if (pattern != NULL) {
1057     fd = expand_and_open(pattern, buf, buflen, 0);
1058   }
1059 
1060   // Either user didn't specify, or the user's location failed,
1061   // so use the default name in the current directory
1062   if (fd == -1) {
1063     const char* cwd = os::get_current_directory(buf, buflen);
1064     if (cwd != NULL) {
1065       size_t pos = strlen(cwd);
1066       int fsep_len = jio_snprintf(&buf[pos], buflen-pos, "%s", os::file_separator());
1067       pos += fsep_len;
1068       if (fsep_len > 0) {
1069         fd = expand_and_open(default_pattern, buf, buflen, pos);
1070       }
1071     }
1072   }
1073 
1074    // try temp directory if it exists.
1075    if (fd == -1) {
1076      const char* tmpdir = os::get_temp_directory();
1077      if (tmpdir != NULL && strlen(tmpdir) > 0) {
1078        int pos = jio_snprintf(buf, buflen, "%s%s", tmpdir, os::file_separator());
1079        if (pos > 0) {
1080          fd = expand_and_open(default_pattern, buf, buflen, pos);
1081        }
1082      }
1083    }
1084 
1085   return fd;
1086 }
1087 
1088 int         VMError::_id;
1089 const char* VMError::_message;
1090 char        VMError::_detail_msg[1024];
1091 Thread*     VMError::_thread;
1092 address     VMError::_pc;
1093 void*       VMError::_siginfo;
1094 void*       VMError::_context;
1095 const char* VMError::_filename;
1096 int         VMError::_lineno;
1097 size_t      VMError::_size;
1098 
1099 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo,
1100                              void* context, const char* detail_fmt, ...)
1101 {
1102   va_list detail_args;
1103   va_start(detail_args, detail_fmt);
1104   report_and_die(sig, NULL, detail_fmt, detail_args, thread, pc, siginfo, context, NULL, 0, 0);
1105   va_end(detail_args);
1106 }
1107 
1108 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context)
1109 {
1110   report_and_die(thread, sig, pc, siginfo, context, "%s", "");
1111 }
1112 
1113 void VMError::report_and_die(const char* message, const char* detail_fmt, ...)
1114 {
1115   va_list detail_args;
1116   va_start(detail_args, detail_fmt);
1117   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, NULL, NULL, NULL, NULL, NULL, 0, 0);
1118   va_end(detail_args);
1119 }
1120 
1121 void VMError::report_and_die(const char* message)
1122 {
1123   report_and_die(message, "%s", "");
1124 }
1125 
1126 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, const char* message,
1127                              const char* detail_fmt, va_list detail_args)
1128 {
1129   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, 0);
1130 }
1131 
1132 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, size_t size,
1133                              VMErrorType vm_err_type, const char* detail_fmt, va_list detail_args) {
1134   report_and_die(vm_err_type, NULL, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, size);
1135 }
1136 
1137 void VMError::report_and_die(int id, const char* message, const char* detail_fmt, va_list detail_args,
1138                              Thread* thread, address pc, void* siginfo, void* context, const char* filename,
1139                              int lineno, size_t size)
1140 {
1141   // Don't allocate large buffer on stack
1142   static char buffer[O_BUFLEN];
1143   out.set_scratch_buffer(buffer, sizeof(buffer));
1144   log.set_scratch_buffer(buffer, sizeof(buffer));
1145 
1146   // How many errors occurred in error handler when reporting first_error.
1147   static int recursive_error_count;
1148 
1149   // We will first print a brief message to standard out (verbose = false),
1150   // then save detailed information in log file (verbose = true).
1151   static bool out_done = false;         // done printing to standard out
1152   static bool log_done = false;         // done saving error log
1153   static bool transmit_report_done = false; // done error reporting
1154 
1155   if (SuppressFatalErrorMessage) {
1156       os::abort(CreateCoredumpOnCrash);
1157   }
1158   intptr_t mytid = os::current_thread_id();
1159   if (first_error_tid == -1 &&
1160       Atomic::cmpxchg_ptr(mytid, &first_error_tid, -1) == -1) {
1161 
1162     // Initialize time stamps to use the same base.
1163     out.time_stamp().update_to(1);
1164     log.time_stamp().update_to(1);
1165 
1166     _id = id;
1167     _message = message;
1168     _thread = thread;
1169     _pc = pc;
1170     _siginfo = siginfo;
1171     _context = context;
1172     _filename = filename;
1173     _lineno = lineno;
1174     _size = size;
1175     jio_vsnprintf(_detail_msg, sizeof(_detail_msg), detail_fmt, detail_args);
1176 
1177     // first time
1178     set_error_reported();
1179 
1180     if (ShowMessageBoxOnError || PauseAtExit) {
1181       show_message_box(buffer, sizeof(buffer));
1182 
1183       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
1184       // WatcherThread can kill JVM if the error handler hangs.
1185       ShowMessageBoxOnError = false;
1186     }
1187 
1188     os::check_dump_limit(buffer, sizeof(buffer));
1189 
1190     // reset signal handlers or exception filter; make sure recursive crashes
1191     // are handled properly.
1192     reset_signal_handlers();
1193 
1194     TRACE_VM_ERROR();
1195 
1196   } else {
1197     // If UseOsErrorReporting we call this for each level of the call stack
1198     // while searching for the exception handler.  Only the first level needs
1199     // to be reported.
1200     if (UseOSErrorReporting && log_done) return;
1201 
1202     // This is not the first error, see if it happened in a different thread
1203     // or in the same thread during error reporting.
1204     if (first_error_tid != mytid) {
1205       char msgbuf[64];
1206       jio_snprintf(msgbuf, sizeof(msgbuf),
1207                    "[thread " INTX_FORMAT " also had an error]",
1208                    mytid);
1209       out.print_raw_cr(msgbuf);
1210 
1211       // error reporting is not MT-safe, block current thread
1212       os::infinite_sleep();
1213 
1214     } else {
1215       if (recursive_error_count++ > 30) {
1216         out.print_raw_cr("[Too many errors, abort]");
1217         os::die();
1218       }
1219 
1220       jio_snprintf(buffer, sizeof(buffer),
1221                    "[error occurred during error reporting (%s), id 0x%x]",
1222                    _current_step_info, _id);
1223       if (log.is_open()) {
1224         log.cr();
1225         log.print_raw_cr(buffer);
1226         log.cr();
1227       } else {
1228         out.cr();
1229         out.print_raw_cr(buffer);
1230         out.cr();
1231       }
1232     }
1233   }
1234 
1235   // print to screen
1236   if (!out_done) {
1237     report(&out, false);
1238 
1239     out_done = true;
1240 
1241     _current_step = 0;
1242     _current_step_info = "";
1243   }
1244 
1245   // print to error log file
1246   if (!log_done) {
1247     // see if log file is already open
1248     if (!log.is_open()) {
1249       // open log file
1250       int fd = prepare_log_file(ErrorFile, "hs_err_pid%p.log", buffer, sizeof(buffer));
1251       if (fd != -1) {
1252         out.print_raw("# An error report file with more information is saved as:\n# ");
1253         out.print_raw_cr(buffer);
1254 
1255         log.set_fd(fd);
1256       } else {
1257         out.print_raw_cr("# Can not save log file, dump to screen..");
1258         log.set_fd(defaultStream::output_fd());
1259         /* Error reporting currently needs dumpfile.
1260          * Maybe implement direct streaming in the future.*/
1261         transmit_report_done = true;
1262       }
1263     }
1264 
1265     report(&log, true);
1266     _current_step = 0;
1267     _current_step_info = "";
1268 
1269     // Run error reporting to determine whether or not to report the crash.
1270     if (!transmit_report_done && should_report_bug(_id)) {
1271       transmit_report_done = true;
1272       const int fd2 = ::dup(log.fd());
1273       FILE* const hs_err = ::fdopen(fd2, "r");
1274       if (NULL != hs_err) {
1275         ErrorReporter er;
1276         er.call(hs_err, buffer, O_BUFLEN);
1277       }
1278       ::fclose(hs_err);
1279     }
1280 
1281     if (log.fd() != defaultStream::output_fd()) {
1282       close(log.fd());
1283     }
1284 
1285     log.set_fd(-1);
1286     log_done = true;
1287   }
1288 
1289   static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
1290   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
1291     skip_replay = true;
1292     ciEnv* env = ciEnv::current();
1293     if (env != NULL) {
1294       int fd = prepare_log_file(ReplayDataFile, "replay_pid%p.log", buffer, sizeof(buffer));
1295       if (fd != -1) {
1296         FILE* replay_data_file = os::open(fd, "w");
1297         if (replay_data_file != NULL) {
1298           fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
1299           env->dump_replay_data_unsafe(&replay_data_stream);
1300           out.print_raw("#\n# Compiler replay data is saved as:\n# ");
1301           out.print_raw_cr(buffer);
1302         } else {
1303           int e = errno;
1304           out.print_raw("#\n# Can't open file to dump replay data. Error: ");
1305           out.print_raw_cr(os::strerror(e));
1306         }
1307       }
1308     }
1309   }
1310 
1311   static bool skip_bug_url = !should_report_bug(_id);
1312   if (!skip_bug_url) {
1313     skip_bug_url = true;
1314 
1315     out.print_raw_cr("#");
1316     print_bug_submit_message(&out, _thread);
1317   }
1318 
1319   static bool skip_OnError = false;
1320   if (!skip_OnError && OnError && OnError[0]) {
1321     skip_OnError = true;
1322 
1323     // Flush output and finish logs before running OnError commands.
1324     ostream_abort();
1325 
1326     out.print_raw_cr("#");
1327     out.print_raw   ("# -XX:OnError=\"");
1328     out.print_raw   (OnError);
1329     out.print_raw_cr("\"");
1330 
1331     char* cmd;
1332     const char* ptr = OnError;
1333     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1334       out.print_raw   ("#   Executing ");
1335 #if defined(LINUX) || defined(_ALLBSD_SOURCE)
1336       out.print_raw   ("/bin/sh -c ");
1337 #elif defined(SOLARIS)
1338       out.print_raw   ("/usr/bin/sh -c ");
1339 #endif
1340       out.print_raw   ("\"");
1341       out.print_raw   (cmd);
1342       out.print_raw_cr("\" ...");
1343 
1344       if (os::fork_and_exec(cmd) < 0) {
1345         out.print_cr("os::fork_and_exec failed: %s (%s=%d)",
1346                      os::strerror(errno), os::errno_name(errno), errno);
1347       }
1348     }
1349 
1350     // done with OnError
1351     OnError = NULL;
1352   }
1353 
1354   if (!UseOSErrorReporting) {
1355     // os::abort() will call abort hooks, try it first.
1356     static bool skip_os_abort = false;
1357     if (!skip_os_abort) {
1358       skip_os_abort = true;
1359       bool dump_core = should_report_bug(_id);
1360       os::abort(dump_core && CreateCoredumpOnCrash, _siginfo, _context);
1361     }
1362 
1363     // if os::abort() doesn't abort, try os::die();
1364     os::die();
1365   }
1366 }
1367 
1368 /*
1369  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
1370  * ensures utilities such as jmap can observe the process is a consistent state.
1371  */
1372 class VM_ReportJavaOutOfMemory : public VM_Operation {
1373  private:
1374   const char* _message;
1375  public:
1376   VM_ReportJavaOutOfMemory(const char* message) { _message = message; }
1377   VMOp_Type type() const                        { return VMOp_ReportJavaOutOfMemory; }
1378   void doit();
1379 };
1380 
1381 void VM_ReportJavaOutOfMemory::doit() {
1382   // Don't allocate large buffer on stack
1383   static char buffer[O_BUFLEN];
1384 
1385   tty->print_cr("#");
1386   tty->print_cr("# java.lang.OutOfMemoryError: %s", _message);
1387   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
1388 
1389   // make heap parsability
1390   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1391 
1392   char* cmd;
1393   const char* ptr = OnOutOfMemoryError;
1394   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1395     tty->print("#   Executing ");
1396 #if defined(LINUX)
1397     tty->print  ("/bin/sh -c ");
1398 #elif defined(SOLARIS)
1399     tty->print  ("/usr/bin/sh -c ");
1400 #endif
1401     tty->print_cr("\"%s\"...", cmd);
1402 
1403     if (os::fork_and_exec(cmd) < 0) {
1404       tty->print_cr("os::fork_and_exec failed: %s (%s=%d)",
1405                      os::strerror(errno), os::errno_name(errno), errno);
1406     }
1407   }
1408 }
1409 
1410 void VMError::report_java_out_of_memory(const char* message) {
1411   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
1412     MutexLocker ml(Heap_lock);
1413     VM_ReportJavaOutOfMemory op(message);
1414     VMThread::execute(&op);
1415   }
1416 }
1417 
1418 void VMError::show_message_box(char *buf, int buflen) {
1419   bool yes;
1420   do {
1421     error_string(buf, buflen);
1422     yes = os::start_debugging(buf,buflen);
1423   } while (yes);
1424 }