1 /*
   2  * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "code/codeCache.hpp"
  27 #include "compiler/compileBroker.hpp"
  28 #include "compiler/disassembler.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "logging/logConfiguration.hpp"
  31 #include "prims/jvm.h"
  32 #include "prims/whitebox.hpp"
  33 #include "runtime/arguments.hpp"
  34 #include "runtime/atomic.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/init.hpp"
  37 #include "runtime/os.hpp"
  38 #include "runtime/thread.inline.hpp"
  39 #include "runtime/vmThread.hpp"
  40 #include "runtime/vm_operations.hpp"
  41 #include "runtime/vm_version.hpp"
  42 #include "services/memTracker.hpp"
  43 #include "trace/traceMacros.hpp"
  44 #include "utilities/debug.hpp"
  45 #include "utilities/decoder.hpp"
  46 #include "utilities/defaultStream.hpp"
  47 #include "utilities/errorReporter.hpp"
  48 #include "utilities/events.hpp"
  49 #include "utilities/vmError.hpp"
  50 
  51 #ifndef PRODUCT
  52 #include <signal.h>
  53 #endif // PRODUCT
  54 
  55 bool VMError::_error_reported = false;
  56 
  57 // call this when the VM is dying--it might loosen some asserts
  58 bool VMError::is_error_reported() { return _error_reported; }
  59 
  60 // returns an address which is guaranteed to generate a SIGSEGV on read,
  61 // for test purposes, which is not NULL and contains bits in every word
  62 void* VMError::get_segfault_address() {
  63   return (void*)
  64 #ifdef _LP64
  65     0xABC0000000000ABCULL;
  66 #else
  67     0x00000ABC;
  68 #endif
  69 }
  70 
  71 // List of environment variables that should be reported in error log file.
  72 const char *env_list[] = {
  73   // All platforms
  74   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  75   "JAVA_COMPILER", "PATH", "USERNAME",
  76 
  77   // Env variables that are defined on Solaris/Linux/BSD
  78   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  79   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  80 
  81   // defined on Linux
  82   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  83 
  84   // defined on Darwin
  85   "DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH",
  86   "DYLD_FRAMEWORK_PATH", "DYLD_FALLBACK_FRAMEWORK_PATH",
  87   "DYLD_INSERT_LIBRARIES",
  88 
  89   // defined on Windows
  90   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  91 
  92   (const char *)0
  93 };
  94 
  95 // A simple parser for -XX:OnError, usage:
  96 //  ptr = OnError;
  97 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
  98 //     ... ...
  99 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
 100   if (ptr == NULL || *ptr == NULL) return NULL;
 101 
 102   const char* cmd = *ptr;
 103 
 104   // skip leading blanks or ';'
 105   while (*cmd == ' ' || *cmd == ';') cmd++;
 106 
 107   if (*cmd == '\0') return NULL;
 108 
 109   const char * cmdend = cmd;
 110   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
 111 
 112   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
 113 
 114   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
 115   return buf;
 116 }
 117 
 118 static void print_bug_submit_message(outputStream *out, Thread *thread) {
 119   if (out == NULL) return;
 120   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 121   out->print_raw   ("#   ");
 122   out->print_raw_cr(Arguments::java_vendor_url_bug());
 123   // If the crash is in native code, encourage user to submit a bug to the
 124   // provider of that code.
 125   if (thread && thread->is_Java_thread() &&
 126       !thread->is_hidden_from_external_view()) {
 127     JavaThread* jt = (JavaThread*)thread;
 128     if (jt->thread_state() == _thread_in_native) {
 129       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 130     }
 131   }
 132   out->print_raw_cr("#");
 133 }
 134 
 135 bool VMError::coredump_status;
 136 char VMError::coredump_message[O_BUFLEN];
 137 
 138 void VMError::record_coredump_status(const char* message, bool status) {
 139   coredump_status = status;
 140   strncpy(coredump_message, message, sizeof(coredump_message));
 141   coredump_message[sizeof(coredump_message)-1] = 0;
 142 }
 143 
 144 // Return a string to describe the error
 145 char* VMError::error_string(char* buf, int buflen) {
 146   char signame_buf[64];
 147   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 148 
 149   if (signame) {
 150     jio_snprintf(buf, buflen,
 151                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 152                  signame, _id, _pc,
 153                  os::current_process_id(), os::current_thread_id());
 154   } else if (_filename != NULL && _lineno > 0) {
 155     // skip directory names
 156     char separator = os::file_separator()[0];
 157     const char *p = strrchr(_filename, separator);
 158     int n = jio_snprintf(buf, buflen,
 159                          "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT,
 160                          p ? p + 1 : _filename, _lineno,
 161                          os::current_process_id(), os::current_thread_id());
 162     if (n >= 0 && n < buflen && _message) {
 163       if (strlen(_detail_msg) > 0) {
 164         jio_snprintf(buf + n, buflen - n, "%s%s: %s",
 165         os::line_separator(), _message, _detail_msg);
 166       } else {
 167         jio_snprintf(buf + n, buflen - n, "%sError: %s",
 168                      os::line_separator(), _message);
 169       }
 170     }
 171   } else {
 172     jio_snprintf(buf, buflen,
 173                  "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 174                  _id, os::current_process_id(), os::current_thread_id());
 175   }
 176 
 177   return buf;
 178 }
 179 
 180 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 181                                 char* buf, int buflen, bool verbose) {
 182 #ifdef ZERO
 183   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 184     // StackFrameStream uses the frame anchor, which may not have
 185     // been set up.  This can be done at any time in Zero, however,
 186     // so if it hasn't been set up then we just set it up now and
 187     // clear it again when we're done.
 188     bool has_last_Java_frame = jt->has_last_Java_frame();
 189     if (!has_last_Java_frame)
 190       jt->set_last_Java_frame();
 191     st->print("Java frames:");
 192 
 193     // If the top frame is a Shark frame and the frame anchor isn't
 194     // set up then it's possible that the information in the frame
 195     // is garbage: it could be from a previous decache, or it could
 196     // simply have never been written.  So we print a warning...
 197     StackFrameStream sfs(jt);
 198     if (!has_last_Java_frame && !sfs.is_done()) {
 199       if (sfs.current()->zeroframe()->is_shark_frame()) {
 200         st->print(" (TOP FRAME MAY BE JUNK)");
 201       }
 202     }
 203     st->cr();
 204 
 205     // Print the frames
 206     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 207       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 208       st->cr();
 209     }
 210 
 211     // Reset the frame anchor if necessary
 212     if (!has_last_Java_frame)
 213       jt->reset_last_Java_frame();
 214   }
 215 #else
 216   if (jt->has_last_Java_frame()) {
 217     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 218     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 219       sfs.current()->print_on_error(st, buf, buflen, verbose);
 220       st->cr();
 221     }
 222   }
 223 #endif // ZERO
 224 }
 225 
 226 void VMError::print_native_stack(outputStream* st, frame fr, Thread* t, char* buf, int buf_size) {
 227 
 228   // see if it's a valid frame
 229   if (fr.pc()) {
 230     st->print_cr("Native frames: (J=compiled Java code, A=aot compiled Java code, j=interpreted, Vv=VM code, C=native code)");
 231 
 232     int count = 0;
 233     while (count++ < StackPrintLimit) {
 234       fr.print_on_error(st, buf, buf_size);
 235       if (fr.pc()) { // print source file and line, if available
 236         char buf[128];
 237         int line_no;
 238         if (Decoder::get_source_info(fr.pc(), buf, sizeof(buf), &line_no)) {
 239           st->print("  (%s:%d)", buf, line_no);
 240         }
 241       }
 242       st->cr();
 243       // Compiled code may use EBP register on x86 so it looks like
 244       // non-walkable C frame. Use frame.sender() for java frames.
 245       if (t && t->is_Java_thread()) {
 246         // Catch very first native frame by using stack address.
 247         // For JavaThread stack_base and stack_size should be set.
 248         if (!t->on_local_stack((address)(fr.real_fp() + 1))) {
 249           break;
 250         }
 251         if (fr.is_java_frame() || fr.is_native_frame() || fr.is_runtime_frame()) {
 252           RegisterMap map((JavaThread*)t, false); // No update
 253           fr = fr.sender(&map);
 254         } else {
 255           fr = os::get_sender_for_C_frame(&fr);
 256         }
 257       } else {
 258         // is_first_C_frame() does only simple checks for frame pointer,
 259         // it will pass if java compiled code has a pointer in EBP.
 260         if (os::is_first_C_frame(&fr)) break;
 261         fr = os::get_sender_for_C_frame(&fr);
 262       }
 263     }
 264 
 265     if (count > StackPrintLimit) {
 266       st->print_cr("...<more frames>...");
 267     }
 268 
 269     st->cr();
 270   }
 271 }
 272 
 273 static void print_oom_reasons(outputStream* st) {
 274   st->print_cr("# Possible reasons:");
 275   st->print_cr("#   The system is out of physical RAM or swap space");
 276   if (UseCompressedOops) {
 277     st->print_cr("#   The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap");
 278   }
 279   if (LogBytesPerWord == 2) {
 280     st->print_cr("#   In 32 bit mode, the process size limit was hit");
 281   }
 282   st->print_cr("# Possible solutions:");
 283   st->print_cr("#   Reduce memory load on the system");
 284   st->print_cr("#   Increase physical memory or swap space");
 285   st->print_cr("#   Check if swap backing store is full");
 286   if (LogBytesPerWord == 2) {
 287     st->print_cr("#   Use 64 bit Java on a 64 bit OS");
 288   }
 289   st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
 290   st->print_cr("#   Decrease number of Java threads");
 291   st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
 292   st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
 293   if (UseCompressedOops) {
 294     switch (Universe::narrow_oop_mode()) {
 295       case Universe::UnscaledNarrowOop:
 296         st->print_cr("#   JVM is running with Unscaled Compressed Oops mode in which the Java heap is");
 297         st->print_cr("#     placed in the first 4GB address space. The Java Heap base address is the");
 298         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 299         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 4GB virtual address.");
 300         break;
 301       case Universe::ZeroBasedNarrowOop:
 302         st->print_cr("#   JVM is running with Zero Based Compressed Oops mode in which the Java heap is");
 303         st->print_cr("#     placed in the first 32GB address space. The Java Heap base address is the");
 304         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 305         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 32GB virtual address.");
 306         break;
 307       default:
 308         break;
 309     }
 310   }
 311   st->print_cr("# This output file may be truncated or incomplete.");
 312 }
 313 
 314 static const char* gc_mode() {
 315   if (UseG1GC)            return "g1 gc";
 316   if (UseParallelGC)      return "parallel gc";
 317   if (UseConcMarkSweepGC) return "concurrent mark sweep gc";
 318   if (UseSerialGC)        return "serial gc";
 319   return "ERROR in GC mode";
 320 }
 321 
 322 static void report_vm_version(outputStream* st, char* buf, int buflen) {
 323    // VM version
 324    st->print_cr("#");
 325    JDK_Version::current().to_string(buf, buflen);
 326    const char* runtime_name = JDK_Version::runtime_name() != NULL ?
 327                                 JDK_Version::runtime_name() : "";
 328    const char* runtime_version = JDK_Version::runtime_version() != NULL ?
 329                                    JDK_Version::runtime_version() : "";
 330    const char* jdk_debug_level = Abstract_VM_Version::printable_jdk_debug_level() != NULL ?
 331                                    Abstract_VM_Version::printable_jdk_debug_level() : "";
 332 
 333    st->print_cr("# JRE version: %s (%s) (%sbuild %s)", runtime_name, buf,
 334                  jdk_debug_level, runtime_version);
 335 
 336    // This is the long version with some default settings added
 337    st->print_cr("# Java VM: %s (%s%s, %s%s%s%s%s, %s, %s)",
 338                  Abstract_VM_Version::vm_name(),
 339                  jdk_debug_level,
 340                  Abstract_VM_Version::vm_release(),
 341                  Abstract_VM_Version::vm_info_string(),
 342                  TieredCompilation ? ", tiered" : "",
 343 #if INCLUDE_JVMCI
 344                  EnableJVMCI ? ", jvmci" : "",
 345                  UseJVMCICompiler ? ", jvmci compiler" : "",
 346 #else
 347                  "", "",
 348 #endif
 349                  UseCompressedOops ? ", compressed oops" : "",
 350                  gc_mode(),
 351                  Abstract_VM_Version::vm_platform_string()
 352                );
 353 }
 354 
 355 // This is the main function to report a fatal error. Only one thread can
 356 // call this function, so we don't need to worry about MT-safety. But it's
 357 // possible that the error handler itself may crash or die on an internal
 358 // error, for example, when the stack/heap is badly damaged. We must be
 359 // able to handle recursive errors that happen inside error handler.
 360 //
 361 // Error reporting is done in several steps. If a crash or internal error
 362 // occurred when reporting an error, the nested signal/exception handler
 363 // can skip steps that are already (or partially) done. Error reporting will
 364 // continue from the next step. This allows us to retrieve and print
 365 // information that may be unsafe to get after a fatal error. If it happens,
 366 // you may find nested report_and_die() frames when you look at the stack
 367 // in a debugger.
 368 //
 369 // In general, a hang in error handler is much worse than a crash or internal
 370 // error, as it's harder to recover from a hang. Deadlock can happen if we
 371 // try to grab a lock that is already owned by current thread, or if the
 372 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 373 // error handler and all the functions it called should avoid grabbing any
 374 // lock. An important thing to notice is that memory allocation needs a lock.
 375 //
 376 // We should avoid using large stack allocated buffers. Many errors happen
 377 // when stack space is already low. Making things even worse is that there
 378 // could be nested report_and_die() calls on stack (see above). Only one
 379 // thread can report error, so large buffers are statically allocated in data
 380 // segment.
 381 
 382 int          VMError::_current_step;
 383 const char*  VMError::_current_step_info;
 384 
 385 volatile jlong VMError::_reporting_start_time = -1;
 386 volatile bool VMError::_reporting_did_timeout = false;
 387 volatile jlong VMError::_step_start_time = -1;
 388 volatile bool VMError::_step_did_timeout = false;
 389 
 390 // Helper, return current timestamp for timeout handling.
 391 jlong VMError::get_current_timestamp() {
 392   return os::javaTimeNanos();
 393 }
 394 // Factor to translate the timestamp to seconds.
 395 #define TIMESTAMP_TO_SECONDS_FACTOR (1000 * 1000 * 1000)
 396 
 397 void VMError::record_reporting_start_time() {
 398   const jlong now = get_current_timestamp();
 399   Atomic::store(now, &_reporting_start_time);
 400 }
 401 
 402 jlong VMError::get_reporting_start_time() {
 403   return Atomic::load(&_reporting_start_time);
 404 }
 405 
 406 void VMError::record_step_start_time() {
 407   const jlong now = get_current_timestamp();
 408   Atomic::store(now, &_step_start_time);
 409 }
 410 
 411 jlong VMError::get_step_start_time() {
 412   return Atomic::load(&_step_start_time);
 413 }
 414 
 415 void VMError::report(outputStream* st, bool _verbose) {
 416 
 417 # define BEGIN if (_current_step == 0) { _current_step = __LINE__;
 418 # define STEP(s) } if (_current_step < __LINE__) { _current_step = __LINE__; _current_step_info = s; \
 419   record_step_start_time(); _step_did_timeout = false;
 420 # define END }
 421 
 422   // don't allocate large buffer on stack
 423   static char buf[O_BUFLEN];
 424 
 425   BEGIN
 426 
 427   STEP("printing fatal error message")
 428 
 429     st->print_cr("#");
 430     if (should_report_bug(_id)) {
 431       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 432     } else {
 433       st->print_cr("# There is insufficient memory for the Java "
 434                    "Runtime Environment to continue.");
 435     }
 436 
 437 #ifndef PRODUCT
 438   // Error handler self tests
 439 
 440   // test secondary error handling. Test it twice, to test that resetting
 441   // error handler after a secondary crash works.
 442   STEP("test secondary crash 1")
 443     if (_verbose && TestCrashInErrorHandler != 0) {
 444       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 445         TestCrashInErrorHandler);
 446       controlled_crash(TestCrashInErrorHandler);
 447     }
 448 
 449   STEP("test secondary crash 2")
 450     if (_verbose && TestCrashInErrorHandler != 0) {
 451       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 452         TestCrashInErrorHandler);
 453       controlled_crash(TestCrashInErrorHandler);
 454     }
 455 
 456   // TestUnresponsiveErrorHandler: We want to test both step timeouts and global timeout.
 457   // Step to global timeout ratio is 4:1, so in order to be absolutely sure we hit the
 458   // global timeout, let's execute the timeout step five times.
 459   // See corresponding test in test/runtime/ErrorHandling/TimeoutInErrorHandlingTest.java
 460   #define TIMEOUT_TEST_STEP STEP("test unresponsive error reporting step") \
 461     if (_verbose && TestUnresponsiveErrorHandler) { os::infinite_sleep(); }
 462   TIMEOUT_TEST_STEP
 463   TIMEOUT_TEST_STEP
 464   TIMEOUT_TEST_STEP
 465   TIMEOUT_TEST_STEP
 466   TIMEOUT_TEST_STEP
 467 
 468   STEP("test safefetch in error handler")
 469     // test whether it is safe to use SafeFetch32 in Crash Handler. Test twice
 470     // to test that resetting the signal handler works correctly.
 471     if (_verbose && TestSafeFetchInErrorHandler) {
 472       st->print_cr("Will test SafeFetch...");
 473       if (CanUseSafeFetch32()) {
 474         int* const invalid_pointer = (int*) get_segfault_address();
 475         const int x = 0x76543210;
 476         int i1 = SafeFetch32(invalid_pointer, x);
 477         int i2 = SafeFetch32(invalid_pointer, x);
 478         if (i1 == x && i2 == x) {
 479           st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern
 480         } else {
 481           st->print_cr("??");
 482         }
 483       } else {
 484         st->print_cr("not possible; skipped.");
 485       }
 486     }
 487 #endif // PRODUCT
 488 
 489   STEP("printing type of error")
 490 
 491      switch(_id) {
 492        case OOM_MALLOC_ERROR:
 493        case OOM_MMAP_ERROR:
 494          if (_size) {
 495            st->print("# Native memory allocation ");
 496            st->print((_id == (int)OOM_MALLOC_ERROR) ? "(malloc) failed to allocate " :
 497                                                  "(mmap) failed to map ");
 498            jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
 499            st->print("%s", buf);
 500            st->print(" bytes");
 501            if (strlen(_detail_msg) > 0) {
 502              st->print(" for ");
 503              st->print("%s", _detail_msg);
 504            }
 505            st->cr();
 506          } else {
 507            if (strlen(_detail_msg) > 0) {
 508              st->print("# ");
 509              st->print_cr("%s", _detail_msg);
 510            }
 511          }
 512          // In error file give some solutions
 513          if (_verbose) {
 514            print_oom_reasons(st);
 515          } else {
 516            return;  // that's enough for the screen
 517          }
 518          break;
 519        case INTERNAL_ERROR:
 520        default:
 521          break;
 522      }
 523 
 524   STEP("printing exception/signal name")
 525 
 526      st->print_cr("#");
 527      st->print("#  ");
 528      // Is it an OS exception/signal?
 529      if (os::exception_name(_id, buf, sizeof(buf))) {
 530        st->print("%s", buf);
 531        st->print(" (0x%x)", _id);                // signal number
 532        st->print(" at pc=" PTR_FORMAT, p2i(_pc));
 533      } else {
 534        if (should_report_bug(_id)) {
 535          st->print("Internal Error");
 536        } else {
 537          st->print("Out of Memory Error");
 538        }
 539        if (_filename != NULL && _lineno > 0) {
 540 #ifdef PRODUCT
 541          // In product mode chop off pathname?
 542          char separator = os::file_separator()[0];
 543          const char *p = strrchr(_filename, separator);
 544          const char *file = p ? p+1 : _filename;
 545 #else
 546          const char *file = _filename;
 547 #endif
 548          st->print(" (%s:%d)", file, _lineno);
 549        } else {
 550          st->print(" (0x%x)", _id);
 551        }
 552      }
 553 
 554   STEP("printing current thread and pid")
 555 
 556      // process id, thread id
 557      st->print(", pid=%d", os::current_process_id());
 558      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 559      st->cr();
 560 
 561   STEP("printing error message")
 562 
 563      if (should_report_bug(_id)) {  // already printed the message.
 564        // error message
 565        if (strlen(_detail_msg) > 0) {
 566          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
 567        } else if (_message) {
 568          st->print_cr("#  Error: %s", _message);
 569        }
 570      }
 571 
 572   STEP("printing Java version string")
 573 
 574      report_vm_version(st, buf, sizeof(buf));
 575 
 576   STEP("printing problematic frame")
 577 
 578      // Print current frame if we have a context (i.e. it's a crash)
 579      if (_context) {
 580        st->print_cr("# Problematic frame:");
 581        st->print("# ");
 582        frame fr = os::fetch_frame_from_context(_context);
 583        fr.print_on_error(st, buf, sizeof(buf));
 584        st->cr();
 585        st->print_cr("#");
 586      }
 587 
 588   STEP("printing core file information")
 589     st->print("# ");
 590     if (CreateCoredumpOnCrash) {
 591       if (coredump_status) {
 592         st->print("Core dump will be written. Default location: %s", coredump_message);
 593       } else {
 594         st->print("No core dump will be written. %s", coredump_message);
 595       }
 596     } else {
 597       st->print("CreateCoredumpOnCrash turned off, no core file dumped");
 598     }
 599     st->cr();
 600     st->print_cr("#");
 601 
 602   STEP("printing bug submit message")
 603 
 604      if (should_report_bug(_id) && _verbose) {
 605        print_bug_submit_message(st, _thread);
 606      }
 607 
 608   STEP("printing summary")
 609 
 610      if (_verbose) {
 611        st->cr();
 612        st->print_cr("---------------  S U M M A R Y ------------");
 613        st->cr();
 614      }
 615 
 616   STEP("printing VM option summary")
 617 
 618      if (_verbose) {
 619        // VM options
 620        Arguments::print_summary_on(st);
 621        st->cr();
 622      }
 623 
 624   STEP("printing summary machine and OS info")
 625 
 626      if (_verbose) {
 627        os::print_summary_info(st, buf, sizeof(buf));
 628      }
 629 
 630 
 631   STEP("printing date and time")
 632 
 633      if (_verbose) {
 634        os::print_date_and_time(st, buf, sizeof(buf));
 635      }
 636 
 637   STEP("printing thread")
 638 
 639      if (_verbose) {
 640        st->cr();
 641        st->print_cr("---------------  T H R E A D  ---------------");
 642        st->cr();
 643      }
 644 
 645   STEP("printing current thread")
 646 
 647      // current thread
 648      if (_verbose) {
 649        if (_thread) {
 650          st->print("Current thread (" PTR_FORMAT "):  ", p2i(_thread));
 651          _thread->print_on_error(st, buf, sizeof(buf));
 652          st->cr();
 653        } else {
 654          st->print_cr("Current thread is native thread");
 655        }
 656        st->cr();
 657      }
 658 
 659   STEP("printing current compile task")
 660 
 661      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 662         CompilerThread* t = (CompilerThread*)_thread;
 663         if (t->task()) {
 664            st->cr();
 665            st->print_cr("Current CompileTask:");
 666            t->task()->print_line_on_error(st, buf, sizeof(buf));
 667            st->cr();
 668         }
 669      }
 670 
 671 
 672   STEP("printing stack bounds")
 673 
 674      if (_verbose) {
 675        st->print("Stack: ");
 676 
 677        address stack_top;
 678        size_t stack_size;
 679 
 680        if (_thread) {
 681           stack_top = _thread->stack_base();
 682           stack_size = _thread->stack_size();
 683        } else {
 684           stack_top = os::current_stack_base();
 685           stack_size = os::current_stack_size();
 686        }
 687 
 688        address stack_bottom = stack_top - stack_size;
 689        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", p2i(stack_bottom), p2i(stack_top));
 690 
 691        frame fr = _context ? os::fetch_frame_from_context(_context)
 692                            : os::current_frame();
 693 
 694        if (fr.sp()) {
 695          st->print(",  sp=" PTR_FORMAT, p2i(fr.sp()));
 696          size_t free_stack_size = pointer_delta(fr.sp(), stack_bottom, 1024);
 697          st->print(",  free space=" SIZE_FORMAT "k", free_stack_size);
 698        }
 699 
 700        st->cr();
 701      }
 702 
 703   STEP("printing native stack")
 704 
 705    if (_verbose) {
 706      if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) {
 707        // We have printed the native stack in platform-specific code
 708        // Windows/x64 needs special handling.
 709      } else {
 710        frame fr = _context ? os::fetch_frame_from_context(_context)
 711                            : os::current_frame();
 712 
 713        print_native_stack(st, fr, _thread, buf, sizeof(buf));
 714      }
 715    }
 716 
 717   STEP("printing Java stack")
 718 
 719      if (_verbose && _thread && _thread->is_Java_thread()) {
 720        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 721      }
 722 
 723   STEP("printing target Java thread stack")
 724 
 725      // printing Java thread stack trace if it is involved in GC crash
 726      if (_verbose && _thread && (_thread->is_Named_thread())) {
 727        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 728        if (jt != NULL) {
 729          st->print_cr("JavaThread " PTR_FORMAT " (nid = %d) was being processed", p2i(jt), jt->osthread()->thread_id());
 730          print_stack_trace(st, jt, buf, sizeof(buf), true);
 731        }
 732      }
 733 
 734   STEP("printing siginfo")
 735 
 736      // signal no, signal code, address that caused the fault
 737      if (_verbose && _siginfo) {
 738        st->cr();
 739        os::print_siginfo(st, _siginfo);
 740        st->cr();
 741      }
 742 
 743   STEP("CDS archive access warning")
 744 
 745      // Print an explicit hint if we crashed on access to the CDS archive.
 746      if (_verbose && _siginfo) {
 747        check_failing_cds_access(st, _siginfo);
 748        st->cr();
 749      }
 750 
 751   STEP("printing register info")
 752 
 753      // decode register contents if possible
 754      if (_verbose && _context && Universe::is_fully_initialized()) {
 755        os::print_register_info(st, _context);
 756        st->cr();
 757      }
 758 
 759   STEP("printing registers, top of stack, instructions near pc")
 760 
 761      // registers, top of stack, instructions near pc
 762      if (_verbose && _context) {
 763        os::print_context(st, _context);
 764        st->cr();
 765      }
 766 
 767   STEP("printing code blob if possible")
 768 
 769      if (_verbose && _context) {
 770        CodeBlob* cb = CodeCache::find_blob(_pc);
 771        if (cb != NULL) {
 772          if (Interpreter::contains(_pc)) {
 773            // The interpreter CodeBlob is very large so try to print the codelet instead.
 774            InterpreterCodelet* codelet = Interpreter::codelet_containing(_pc);
 775            if (codelet != NULL) {
 776              codelet->print_on(st);
 777              Disassembler::decode(codelet->code_begin(), codelet->code_end(), st);
 778            }
 779          } else {
 780            StubCodeDesc* desc = StubCodeDesc::desc_for(_pc);
 781            if (desc != NULL) {
 782              desc->print_on(st);
 783              Disassembler::decode(desc->begin(), desc->end(), st);
 784            } else {
 785              Disassembler::decode(cb, st);
 786              st->cr();
 787            }
 788          }
 789        }
 790      }
 791 
 792   STEP("printing VM operation")
 793 
 794      if (_verbose && _thread && _thread->is_VM_thread()) {
 795         VMThread* t = (VMThread*)_thread;
 796         VM_Operation* op = t->vm_operation();
 797         if (op) {
 798           op->print_on_error(st);
 799           st->cr();
 800           st->cr();
 801         }
 802      }
 803 
 804   STEP("printing process")
 805 
 806      if (_verbose) {
 807        st->cr();
 808        st->print_cr("---------------  P R O C E S S  ---------------");
 809        st->cr();
 810      }
 811 
 812   STEP("printing all threads")
 813 
 814      // all threads
 815      if (_verbose && _thread) {
 816        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 817        st->cr();
 818      }
 819 
 820   STEP("printing VM state")
 821 
 822      if (_verbose) {
 823        // Safepoint state
 824        st->print("VM state:");
 825 
 826        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 827        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 828        else st->print("not at safepoint");
 829 
 830        // Also see if error occurred during initialization or shutdown
 831        if (!Universe::is_fully_initialized()) {
 832          st->print(" (not fully initialized)");
 833        } else if (VM_Exit::vm_exited()) {
 834          st->print(" (shutting down)");
 835        } else {
 836          st->print(" (normal execution)");
 837        }
 838        st->cr();
 839        st->cr();
 840      }
 841 
 842   STEP("printing owned locks on error")
 843 
 844      // mutexes/monitors that currently have an owner
 845      if (_verbose) {
 846        print_owned_locks_on_error(st);
 847        st->cr();
 848      }
 849 
 850   STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
 851 
 852      if (_verbose && Exceptions::has_exception_counts()) {
 853        st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 854        Exceptions::print_exception_counts_on_error(st);
 855        st->cr();
 856      }
 857 
 858   STEP("printing compressed oops mode")
 859 
 860      if (_verbose && UseCompressedOops) {
 861        Universe::print_compressed_oops_mode(st);
 862        if (UseCompressedClassPointers) {
 863          Metaspace::print_compressed_class_space(st);
 864        }
 865        st->cr();
 866      }
 867 
 868   STEP("printing heap information")
 869 
 870      if (_verbose && Universe::is_fully_initialized()) {
 871        Universe::heap()->print_on_error(st);
 872        st->cr();
 873        st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
 874        st->cr();
 875      }
 876 
 877   STEP("printing code cache information")
 878 
 879      if (_verbose && Universe::is_fully_initialized()) {
 880        // print code cache information before vm abort
 881        CodeCache::print_summary(st);
 882        st->cr();
 883      }
 884 
 885   STEP("printing ring buffers")
 886 
 887      if (_verbose) {
 888        Events::print_all(st);
 889        st->cr();
 890      }
 891 
 892   STEP("printing dynamic libraries")
 893 
 894      if (_verbose) {
 895        // dynamic libraries, or memory map
 896        os::print_dll_info(st);
 897        st->cr();
 898      }
 899 
 900   STEP("printing native decoder state")
 901 
 902      if (_verbose) {
 903        Decoder::print_state_on(st);
 904        st->cr();
 905      }
 906 
 907   STEP("printing VM options")
 908 
 909      if (_verbose) {
 910        // VM options
 911        Arguments::print_on(st);
 912        st->cr();
 913      }
 914 
 915   STEP("printing warning if internal testing API used")
 916 
 917      if (WhiteBox::used()) {
 918        st->print_cr("Unsupported internal testing APIs have been used.");
 919        st->cr();
 920      }
 921 
 922   STEP("printing log configuration")
 923     if (_verbose){
 924       st->print_cr("Logging:");
 925       LogConfiguration::describe_current_configuration(st);
 926       st->cr();
 927     }
 928 
 929   STEP("printing all environment variables")
 930 
 931      if (_verbose) {
 932        os::print_environment_variables(st, env_list);
 933        st->cr();
 934      }
 935 
 936   STEP("printing signal handlers")
 937 
 938      if (_verbose) {
 939        os::print_signal_handlers(st, buf, sizeof(buf));
 940        st->cr();
 941      }
 942 
 943   STEP("Native Memory Tracking")
 944      if (_verbose) {
 945        MemTracker::error_report(st);
 946      }
 947 
 948   STEP("printing system")
 949 
 950      if (_verbose) {
 951        st->cr();
 952        st->print_cr("---------------  S Y S T E M  ---------------");
 953        st->cr();
 954      }
 955 
 956   STEP("printing OS information")
 957 
 958      if (_verbose) {
 959        os::print_os_info(st);
 960        st->cr();
 961      }
 962 
 963   STEP("printing CPU info")
 964      if (_verbose) {
 965        os::print_cpu_info(st, buf, sizeof(buf));
 966        st->cr();
 967      }
 968 
 969   STEP("printing memory info")
 970 
 971      if (_verbose) {
 972        os::print_memory_info(st);
 973        st->cr();
 974      }
 975 
 976   STEP("printing internal vm info")
 977 
 978      if (_verbose) {
 979        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 980        st->cr();
 981      }
 982 
 983   // print a defined marker to show that error handling finished correctly.
 984   STEP("printing end marker")
 985 
 986      if (_verbose) {
 987        st->print_cr("END.");
 988      }
 989 
 990   END
 991 
 992 # undef BEGIN
 993 # undef STEP
 994 # undef END
 995 }
 996 
 997 // Report for the vm_info_cmd. This prints out the information above omitting
 998 // crash and thread specific information.  If output is added above, it should be added
 999 // here also, if it is safe to call during a running process.
1000 void VMError::print_vm_info(outputStream* st) {
1001 
1002   char buf[O_BUFLEN];
1003   report_vm_version(st, buf, sizeof(buf));
1004 
1005   // STEP("printing summary")
1006 
1007   st->cr();
1008   st->print_cr("---------------  S U M M A R Y ------------");
1009   st->cr();
1010 
1011   // STEP("printing VM option summary")
1012 
1013   // VM options
1014   Arguments::print_summary_on(st);
1015   st->cr();
1016 
1017   // STEP("printing summary machine and OS info")
1018 
1019   os::print_summary_info(st, buf, sizeof(buf));
1020 
1021   // STEP("printing date and time")
1022 
1023   os::print_date_and_time(st, buf, sizeof(buf));
1024 
1025   // Skip: STEP("printing thread")
1026 
1027   // STEP("printing process")
1028 
1029   st->cr();
1030   st->print_cr("---------------  P R O C E S S  ---------------");
1031   st->cr();
1032 
1033   // STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
1034 
1035   if (Exceptions::has_exception_counts()) {
1036     st->print_cr("OutOfMemory and StackOverflow Exception counts:");
1037     Exceptions::print_exception_counts_on_error(st);
1038     st->cr();
1039   }
1040 
1041   // STEP("printing compressed oops mode")
1042 
1043   if (UseCompressedOops) {
1044     Universe::print_compressed_oops_mode(st);
1045     if (UseCompressedClassPointers) {
1046       Metaspace::print_compressed_class_space(st);
1047     }
1048     st->cr();
1049   }
1050 
1051   // STEP("printing heap information")
1052 
1053   if (Universe::is_fully_initialized()) {
1054     MutexLocker hl(Heap_lock);
1055     Universe::heap()->print_on_error(st);
1056     st->cr();
1057     st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
1058     st->cr();
1059   }
1060 
1061   // STEP("printing code cache information")
1062 
1063   if (Universe::is_fully_initialized()) {
1064     // print code cache information before vm abort
1065     CodeCache::print_summary(st);
1066     st->cr();
1067   }
1068 
1069   // STEP("printing ring buffers")
1070 
1071   Events::print_all(st);
1072   st->cr();
1073 
1074   // STEP("printing dynamic libraries")
1075 
1076   // dynamic libraries, or memory map
1077   os::print_dll_info(st);
1078   st->cr();
1079 
1080   // STEP("printing VM options")
1081 
1082   // VM options
1083   Arguments::print_on(st);
1084   st->cr();
1085 
1086   // STEP("printing warning if internal testing API used")
1087 
1088   if (WhiteBox::used()) {
1089     st->print_cr("Unsupported internal testing APIs have been used.");
1090     st->cr();
1091   }
1092 
1093   // STEP("printing log configuration")
1094   st->print_cr("Logging:");
1095   LogConfiguration::describe(st);
1096   st->cr();
1097 
1098   // STEP("printing all environment variables")
1099 
1100   os::print_environment_variables(st, env_list);
1101   st->cr();
1102 
1103   // STEP("printing signal handlers")
1104 
1105   os::print_signal_handlers(st, buf, sizeof(buf));
1106   st->cr();
1107 
1108   // STEP("Native Memory Tracking")
1109 
1110   MemTracker::error_report(st);
1111 
1112   // STEP("printing system")
1113 
1114   st->cr();
1115   st->print_cr("---------------  S Y S T E M  ---------------");
1116   st->cr();
1117 
1118   // STEP("printing OS information")
1119 
1120   os::print_os_info(st);
1121   st->cr();
1122 
1123   // STEP("printing CPU info")
1124 
1125   os::print_cpu_info(st, buf, sizeof(buf));
1126   st->cr();
1127 
1128   // STEP("printing memory info")
1129 
1130   os::print_memory_info(st);
1131   st->cr();
1132 
1133   // STEP("printing internal vm info")
1134 
1135   st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
1136   st->cr();
1137 
1138   // print a defined marker to show that error handling finished correctly.
1139   // STEP("printing end marker")
1140 
1141   st->print_cr("END.");
1142 }
1143 
1144 volatile intptr_t VMError::first_error_tid = -1;
1145 
1146 // An error could happen before tty is initialized or after it has been
1147 // destroyed.
1148 // Please note: to prevent large stack allocations, the log- and
1149 // output-stream use a global scratch buffer for format printing.
1150 // (see VmError::report_and_die(). Access to those streams is synchronized
1151 // in  VmError::report_and_die() - there is only one reporting thread at
1152 // any given time.
1153 fdStream VMError::out(defaultStream::output_fd());
1154 fdStream VMError::log; // error log used by VMError::report_and_die()
1155 
1156 /** Expand a pattern into a buffer starting at pos and open a file using constructed path */
1157 static int expand_and_open(const char* pattern, char* buf, size_t buflen, size_t pos) {
1158   int fd = -1;
1159   if (Arguments::copy_expand_pid(pattern, strlen(pattern), &buf[pos], buflen - pos)) {
1160     // the O_EXCL flag will cause the open to fail if the file exists
1161     fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0666);
1162   }
1163   return fd;
1164 }
1165 
1166 /**
1167  * Construct file name for a log file and return it's file descriptor.
1168  * Name and location depends on pattern, default_pattern params and access
1169  * permissions.
1170  */
1171 static int prepare_log_file(const char* pattern, const char* default_pattern, char* buf, size_t buflen) {
1172   int fd = -1;
1173 
1174   // If possible, use specified pattern to construct log file name
1175   if (pattern != NULL) {
1176     fd = expand_and_open(pattern, buf, buflen, 0);
1177   }
1178 
1179   // Either user didn't specify, or the user's location failed,
1180   // so use the default name in the current directory
1181   if (fd == -1) {
1182     const char* cwd = os::get_current_directory(buf, buflen);
1183     if (cwd != NULL) {
1184       size_t pos = strlen(cwd);
1185       int fsep_len = jio_snprintf(&buf[pos], buflen-pos, "%s", os::file_separator());
1186       pos += fsep_len;
1187       if (fsep_len > 0) {
1188         fd = expand_and_open(default_pattern, buf, buflen, pos);
1189       }
1190     }
1191   }
1192 
1193    // try temp directory if it exists.
1194    if (fd == -1) {
1195      const char* tmpdir = os::get_temp_directory();
1196      if (tmpdir != NULL && strlen(tmpdir) > 0) {
1197        int pos = jio_snprintf(buf, buflen, "%s%s", tmpdir, os::file_separator());
1198        if (pos > 0) {
1199          fd = expand_and_open(default_pattern, buf, buflen, pos);
1200        }
1201      }
1202    }
1203 
1204   return fd;
1205 }
1206 
1207 int         VMError::_id;
1208 const char* VMError::_message;
1209 char        VMError::_detail_msg[1024];
1210 Thread*     VMError::_thread;
1211 address     VMError::_pc;
1212 void*       VMError::_siginfo;
1213 void*       VMError::_context;
1214 const char* VMError::_filename;
1215 int         VMError::_lineno;
1216 size_t      VMError::_size;
1217 
1218 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo,
1219                              void* context, const char* detail_fmt, ...)
1220 {
1221   va_list detail_args;
1222   va_start(detail_args, detail_fmt);
1223   report_and_die(sig, NULL, detail_fmt, detail_args, thread, pc, siginfo, context, NULL, 0, 0);
1224   va_end(detail_args);
1225 }
1226 
1227 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context)
1228 {
1229   report_and_die(thread, sig, pc, siginfo, context, "%s", "");
1230 }
1231 
1232 void VMError::report_and_die(const char* message, const char* detail_fmt, ...)
1233 {
1234   va_list detail_args;
1235   va_start(detail_args, detail_fmt);
1236   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, NULL, NULL, NULL, NULL, NULL, 0, 0);
1237   va_end(detail_args);
1238 }
1239 
1240 void VMError::report_and_die(const char* message)
1241 {
1242   report_and_die(message, "%s", "");
1243 }
1244 
1245 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, const char* message,
1246                              const char* detail_fmt, va_list detail_args)
1247 {
1248   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, 0);
1249 }
1250 
1251 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, size_t size,
1252                              VMErrorType vm_err_type, const char* detail_fmt, va_list detail_args) {
1253   report_and_die(vm_err_type, NULL, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, size);
1254 }
1255 
1256 void VMError::report_and_die(int id, const char* message, const char* detail_fmt, va_list detail_args,
1257                              Thread* thread, address pc, void* siginfo, void* context, const char* filename,
1258                              int lineno, size_t size)
1259 {
1260   // Don't allocate large buffer on stack
1261   static char buffer[O_BUFLEN];
1262   out.set_scratch_buffer(buffer, sizeof(buffer));
1263   log.set_scratch_buffer(buffer, sizeof(buffer));
1264 
1265   // How many errors occurred in error handler when reporting first_error.
1266   static int recursive_error_count;
1267 
1268   // We will first print a brief message to standard out (verbose = false),
1269   // then save detailed information in log file (verbose = true).
1270   static bool out_done = false;         // done printing to standard out
1271   static bool log_done = false;         // done saving error log
1272   static bool transmit_report_done = false; // done error reporting
1273 
1274   if (SuppressFatalErrorMessage) {
1275       os::abort(CreateCoredumpOnCrash);
1276   }
1277   intptr_t mytid = os::current_thread_id();
1278   if (first_error_tid == -1 &&
1279       Atomic::cmpxchg_ptr(mytid, &first_error_tid, -1) == -1) {
1280 
1281     // Initialize time stamps to use the same base.
1282     out.time_stamp().update_to(1);
1283     log.time_stamp().update_to(1);
1284 
1285     _id = id;
1286     _message = message;
1287     _thread = thread;
1288     _pc = pc;
1289     _siginfo = siginfo;
1290     _context = context;
1291     _filename = filename;
1292     _lineno = lineno;
1293     _size = size;
1294     jio_vsnprintf(_detail_msg, sizeof(_detail_msg), detail_fmt, detail_args);
1295 
1296     // first time
1297     _error_reported = true;
1298 
1299     reporting_started();
1300     record_reporting_start_time();
1301 
1302     if (ShowMessageBoxOnError || PauseAtExit) {
1303       show_message_box(buffer, sizeof(buffer));
1304 
1305       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
1306       // WatcherThread can kill JVM if the error handler hangs.
1307       ShowMessageBoxOnError = false;
1308     }
1309 
1310     os::check_dump_limit(buffer, sizeof(buffer));
1311 
1312     // reset signal handlers or exception filter; make sure recursive crashes
1313     // are handled properly.
1314     reset_signal_handlers();
1315 
1316     TRACE_VM_ERROR();
1317 
1318   } else {
1319     // If UseOsErrorReporting we call this for each level of the call stack
1320     // while searching for the exception handler.  Only the first level needs
1321     // to be reported.
1322     if (UseOSErrorReporting && log_done) return;
1323 
1324     // This is not the first error, see if it happened in a different thread
1325     // or in the same thread during error reporting.
1326     if (first_error_tid != mytid) {
1327       char msgbuf[64];
1328       jio_snprintf(msgbuf, sizeof(msgbuf),
1329                    "[thread " INTX_FORMAT " also had an error]",
1330                    mytid);
1331       out.print_raw_cr(msgbuf);
1332 
1333       // error reporting is not MT-safe, block current thread
1334       os::infinite_sleep();
1335 
1336     } else {
1337       if (recursive_error_count++ > 30) {
1338         out.print_raw_cr("[Too many errors, abort]");
1339         os::die();
1340       }
1341 
1342       outputStream* const st = log.is_open() ? &log : &out;
1343       st->cr();
1344 
1345       // Timeout handling.
1346       if (_step_did_timeout) {
1347         // The current step had a timeout. Lets continue reporting with the next step.
1348         st->print_raw("[timeout occurred during error reporting in step \"");
1349         st->print_raw(_current_step_info);
1350         st->print_cr("\"] after " INT64_FORMAT " s.",
1351                      (int64_t)
1352                      ((get_current_timestamp() - _step_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1353       } else if (_reporting_did_timeout) {
1354         // We hit ErrorLogTimeout. Reporting will stop altogether. Let's wrap things
1355         // up, the process is about to be stopped by the WatcherThread.
1356         st->print_cr("------ Timeout during error reporting after " INT64_FORMAT " s. ------",
1357                      (int64_t)
1358                      ((get_current_timestamp() - _reporting_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1359         st->flush();
1360         // Watcherthread is about to call os::die. Lets just wait.
1361         os::infinite_sleep();
1362       } else {
1363         // Crash or assert during error reporting. Lets continue reporting with the next step.
1364         jio_snprintf(buffer, sizeof(buffer),
1365            "[error occurred during error reporting (%s), id 0x%x]",
1366                    _current_step_info, _id);
1367         st->print_raw_cr(buffer);
1368         st->cr();
1369       }
1370     }
1371   }
1372 
1373   // print to screen
1374   if (!out_done) {
1375     report(&out, false);
1376 
1377     out_done = true;
1378 
1379     _current_step = 0;
1380     _current_step_info = "";
1381   }
1382 
1383   // print to error log file
1384   if (!log_done) {
1385     // see if log file is already open
1386     if (!log.is_open()) {
1387       // open log file
1388       int fd = prepare_log_file(ErrorFile, "hs_err_pid%p.log", buffer, sizeof(buffer));
1389       if (fd != -1) {
1390         out.print_raw("# An error report file with more information is saved as:\n# ");
1391         out.print_raw_cr(buffer);
1392 
1393         log.set_fd(fd);
1394       } else {
1395         out.print_raw_cr("# Can not save log file, dump to screen..");
1396         log.set_fd(defaultStream::output_fd());
1397         /* Error reporting currently needs dumpfile.
1398          * Maybe implement direct streaming in the future.*/
1399         transmit_report_done = true;
1400       }
1401     }
1402 
1403     report(&log, true);
1404     log_done = true;
1405     _current_step = 0;
1406     _current_step_info = "";
1407 
1408     // Run error reporting to determine whether or not to report the crash.
1409     if (!transmit_report_done && should_report_bug(_id)) {
1410       transmit_report_done = true;
1411       const int fd2 = ::dup(log.fd());
1412       if (fd2 != -1) {
1413         FILE* const hs_err = ::fdopen(fd2, "r");
1414         if (NULL != hs_err) {
1415           ErrorReporter er;
1416           er.call(hs_err, buffer, O_BUFLEN);
1417           ::fclose(hs_err);
1418         }
1419       }
1420     }
1421 
1422     if (log.fd() != defaultStream::output_fd()) {
1423       close(log.fd());
1424     }
1425 
1426     log.set_fd(-1);
1427   }
1428 
1429   static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
1430   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
1431     skip_replay = true;
1432     ciEnv* env = ciEnv::current();
1433     if (env != NULL) {
1434       int fd = prepare_log_file(ReplayDataFile, "replay_pid%p.log", buffer, sizeof(buffer));
1435       if (fd != -1) {
1436         FILE* replay_data_file = os::open(fd, "w");
1437         if (replay_data_file != NULL) {
1438           fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
1439           env->dump_replay_data_unsafe(&replay_data_stream);
1440           out.print_raw("#\n# Compiler replay data is saved as:\n# ");
1441           out.print_raw_cr(buffer);
1442         } else {
1443           int e = errno;
1444           out.print_raw("#\n# Can't open file to dump replay data. Error: ");
1445           out.print_raw_cr(os::strerror(e));
1446         }
1447       }
1448     }
1449   }
1450 
1451   static bool skip_bug_url = !should_report_bug(_id);
1452   if (!skip_bug_url) {
1453     skip_bug_url = true;
1454 
1455     out.print_raw_cr("#");
1456     print_bug_submit_message(&out, _thread);
1457   }
1458 
1459   static bool skip_OnError = false;
1460   if (!skip_OnError && OnError && OnError[0]) {
1461     skip_OnError = true;
1462 
1463     // Flush output and finish logs before running OnError commands.
1464     ostream_abort();
1465 
1466     out.print_raw_cr("#");
1467     out.print_raw   ("# -XX:OnError=\"");
1468     out.print_raw   (OnError);
1469     out.print_raw_cr("\"");
1470 
1471     char* cmd;
1472     const char* ptr = OnError;
1473     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1474       out.print_raw   ("#   Executing ");
1475 #if defined(LINUX) || defined(_ALLBSD_SOURCE)
1476       out.print_raw   ("/bin/sh -c ");
1477 #elif defined(SOLARIS)
1478       out.print_raw   ("/usr/bin/sh -c ");
1479 #elif defined(WINDOWS)
1480       out.print_raw   ("cmd /C ");
1481 #endif
1482       out.print_raw   ("\"");
1483       out.print_raw   (cmd);
1484       out.print_raw_cr("\" ...");
1485 
1486       if (os::fork_and_exec(cmd) < 0) {
1487         out.print_cr("os::fork_and_exec failed: %s (%s=%d)",
1488                      os::strerror(errno), os::errno_name(errno), errno);
1489       }
1490     }
1491 
1492     // done with OnError
1493     OnError = NULL;
1494   }
1495 
1496   if (!UseOSErrorReporting) {
1497     // os::abort() will call abort hooks, try it first.
1498     static bool skip_os_abort = false;
1499     if (!skip_os_abort) {
1500       skip_os_abort = true;
1501       bool dump_core = should_report_bug(_id);
1502       os::abort(dump_core && CreateCoredumpOnCrash, _siginfo, _context);
1503     }
1504 
1505     // if os::abort() doesn't abort, try os::die();
1506     os::die();
1507   }
1508 }
1509 
1510 /*
1511  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
1512  * ensures utilities such as jmap can observe the process is a consistent state.
1513  */
1514 class VM_ReportJavaOutOfMemory : public VM_Operation {
1515  private:
1516   const char* _message;
1517  public:
1518   VM_ReportJavaOutOfMemory(const char* message) { _message = message; }
1519   VMOp_Type type() const                        { return VMOp_ReportJavaOutOfMemory; }
1520   void doit();
1521 };
1522 
1523 void VM_ReportJavaOutOfMemory::doit() {
1524   // Don't allocate large buffer on stack
1525   static char buffer[O_BUFLEN];
1526 
1527   tty->print_cr("#");
1528   tty->print_cr("# java.lang.OutOfMemoryError: %s", _message);
1529   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
1530 
1531   // make heap parsability
1532   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1533 
1534   char* cmd;
1535   const char* ptr = OnOutOfMemoryError;
1536   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1537     tty->print("#   Executing ");
1538 #if defined(LINUX)
1539     tty->print  ("/bin/sh -c ");
1540 #elif defined(SOLARIS)
1541     tty->print  ("/usr/bin/sh -c ");
1542 #endif
1543     tty->print_cr("\"%s\"...", cmd);
1544 
1545     if (os::fork_and_exec(cmd) < 0) {
1546       tty->print_cr("os::fork_and_exec failed: %s (%s=%d)",
1547                      os::strerror(errno), os::errno_name(errno), errno);
1548     }
1549   }
1550 }
1551 
1552 void VMError::report_java_out_of_memory(const char* message) {
1553   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
1554     MutexLocker ml(Heap_lock);
1555     VM_ReportJavaOutOfMemory op(message);
1556     VMThread::execute(&op);
1557   }
1558 }
1559 
1560 void VMError::show_message_box(char *buf, int buflen) {
1561   bool yes;
1562   do {
1563     error_string(buf, buflen);
1564     yes = os::start_debugging(buf,buflen);
1565   } while (yes);
1566 }
1567 
1568 // Timeout handling: check if a timeout happened (either a single step did
1569 // timeout or the whole of error reporting hit ErrorLogTimeout). Interrupt
1570 // the reporting thread if that is the case.
1571 bool VMError::check_timeout() {
1572 
1573   if (ErrorLogTimeout == 0) {
1574     return false;
1575   }
1576 
1577   // Do not check for timeouts if we still have a message box to show to the
1578   // user or if there are OnError handlers to be run.
1579   if (ShowMessageBoxOnError
1580       || (OnError != NULL && OnError[0] != '\0')
1581       || Arguments::abort_hook() != NULL) {
1582     return false;
1583   }
1584 
1585   const jlong reporting_start_time_l = get_reporting_start_time();
1586   const jlong now = get_current_timestamp();
1587   // Timestamp is stored in nanos.
1588   if (reporting_start_time_l > 0) {
1589     const jlong end = reporting_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR;
1590     if (end <= now) {
1591       _reporting_did_timeout = true;
1592       interrupt_reporting_thread();
1593       return true; // global timeout
1594     }
1595   }
1596 
1597   const jlong step_start_time_l = get_step_start_time();
1598   if (step_start_time_l > 0) {
1599     // A step times out after a quarter of the total timeout. Steps are mostly fast unless they
1600     // hang for some reason, so this simple rule allows for three hanging step and still
1601     // hopefully leaves time enough for the rest of the steps to finish.
1602     const jlong end = step_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR / 4;
1603     if (end <= now) {
1604       _step_did_timeout = true;
1605       interrupt_reporting_thread();
1606       return false; // (Not a global timeout)
1607     }
1608   }
1609 
1610   return false;
1611 
1612 }
1613 
1614 #ifndef PRODUCT
1615 typedef void (*voidfun_t)();
1616 // Crash with an authentic sigfpe
1617 static void crash_with_sigfpe() {
1618   // generate a native synchronous SIGFPE where possible;
1619   // if that did not cause a signal (e.g. on ppc), just
1620   // raise the signal.
1621   volatile int x = 0;
1622   volatile int y = 1/x;
1623 #ifndef _WIN32
1624   // OSX implements raise(sig) incorrectly so we need to
1625   // explicitly target the current thread
1626   pthread_kill(pthread_self(), SIGFPE);
1627 #endif
1628 } // end: crash_with_sigfpe
1629 
1630 // crash with sigsegv at non-null address.
1631 static void crash_with_segfault() {
1632 
1633   char* const crash_addr = (char*) VMError::get_segfault_address();
1634   *crash_addr = 'X';
1635 
1636 } // end: crash_with_segfault
1637 
1638 void VMError::test_error_handler() {
1639   controlled_crash(ErrorHandlerTest);
1640 }
1641 
1642 // crash in a controlled way:
1643 // how can be one of:
1644 // 1,2 - asserts
1645 // 3,4 - guarantee
1646 // 5-7 - fatal
1647 // 8 - vm_exit_out_of_memory
1648 // 9 - ShouldNotCallThis
1649 // 10 - ShouldNotReachHere
1650 // 11 - Unimplemented
1651 // 12,13 - (not guaranteed) crashes
1652 // 14 - SIGSEGV
1653 // 15 - SIGFPE
1654 void VMError::controlled_crash(int how) {
1655   if (how == 0) return;
1656 
1657   // If asserts are disabled, use the corresponding guarantee instead.
1658   NOT_DEBUG(if (how <= 2) how += 2);
1659 
1660   const char* const str = "hello";
1661   const size_t      num = (size_t)os::vm_page_size();
1662 
1663   const char* const eol = os::line_separator();
1664   const char* const msg = "this message should be truncated during formatting";
1665   char * const dataPtr = NULL;  // bad data pointer
1666   const void (*funcPtr)(void) = (const void(*)()) 0xF;  // bad function pointer
1667 
1668   // Keep this in sync with test/runtime/ErrorHandling/ErrorHandler.java
1669   switch (how) {
1670     case  1: vmassert(str == NULL, "expected null");
1671     case  2: vmassert(num == 1023 && *str == 'X',
1672                       "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1673     case  3: guarantee(str == NULL, "expected null");
1674     case  4: guarantee(num == 1023 && *str == 'X',
1675                        "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1676     case  5: fatal("expected null");
1677     case  6: fatal("num=" SIZE_FORMAT " str=\"%s\"", num, str);
1678     case  7: fatal("%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1679                    "%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1680                    "%s%s#    %s%s#    %s%s#    %s%s#    %s",
1681                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1682                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1683                    msg, eol, msg, eol, msg, eol, msg, eol, msg);
1684     case  8: vm_exit_out_of_memory(num, OOM_MALLOC_ERROR, "ChunkPool::allocate");
1685     case  9: ShouldNotCallThis();
1686     case 10: ShouldNotReachHere();
1687     case 11: Unimplemented();
1688     // There's no guarantee the bad data pointer will crash us
1689     // so "break" out to the ShouldNotReachHere().
1690     case 12: *dataPtr = '\0'; break;
1691     // There's no guarantee the bad function pointer will crash us
1692     // so "break" out to the ShouldNotReachHere().
1693     case 13: (*funcPtr)(); break;
1694     case 14: crash_with_segfault(); break;
1695     case 15: crash_with_sigfpe(); break;
1696 
1697     default: tty->print_cr("ERROR: %d: unexpected test_num value.", how);
1698   }
1699   ShouldNotReachHere();
1700 }
1701 #endif // !PRODUCT
1702