1 /*
   2  * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "code/codeCache.hpp"
  27 #include "compiler/compileBroker.hpp"
  28 #include "compiler/disassembler.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "logging/logConfiguration.hpp"
  31 #include "prims/jvm.h"
  32 #include "prims/whitebox.hpp"
  33 #include "runtime/arguments.hpp"
  34 #include "runtime/atomic.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/init.hpp"
  37 #include "runtime/os.hpp"
  38 #include "runtime/thread.inline.hpp"
  39 #include "runtime/vmThread.hpp"
  40 #include "runtime/vm_operations.hpp"
  41 #include "runtime/vm_version.hpp"
  42 #include "services/memTracker.hpp"
  43 #include "trace/traceMacros.hpp"
  44 #include "utilities/debug.hpp"
  45 #include "utilities/decoder.hpp"
  46 #include "utilities/defaultStream.hpp"
  47 #include "utilities/errorReporter.hpp"
  48 #include "utilities/events.hpp"
  49 #include "utilities/vmError.hpp"
  50 
  51 #ifndef PRODUCT
  52 #include <signal.h>
  53 #endif // PRODUCT
  54 
  55 bool VMError::_error_reported = false;
  56 
  57 // call this when the VM is dying--it might loosen some asserts
  58 bool VMError::is_error_reported() { return _error_reported; }
  59 
  60 // returns an address which is guaranteed to generate a SIGSEGV on read,
  61 // for test purposes, which is not NULL and contains bits in every word
  62 void* VMError::get_segfault_address() {
  63   return (void*)
  64 #ifdef _LP64
  65     0xABC0000000000ABCULL;
  66 #else
  67     0x00000ABC;
  68 #endif
  69 }
  70 
  71 // List of environment variables that should be reported in error log file.
  72 const char *env_list[] = {
  73   // All platforms
  74   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  75   "JAVA_COMPILER", "PATH", "USERNAME",
  76 
  77   // Env variables that are defined on Solaris/Linux/BSD
  78   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  79   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  80 
  81   // defined on Linux
  82   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  83 
  84   // defined on Darwin
  85   "DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH",
  86   "DYLD_FRAMEWORK_PATH", "DYLD_FALLBACK_FRAMEWORK_PATH",
  87   "DYLD_INSERT_LIBRARIES",
  88 
  89   // defined on Windows
  90   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  91 
  92   (const char *)0
  93 };
  94 
  95 // A simple parser for -XX:OnError, usage:
  96 //  ptr = OnError;
  97 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
  98 //     ... ...
  99 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
 100   if (ptr == NULL || *ptr == NULL) return NULL;
 101 
 102   const char* cmd = *ptr;
 103 
 104   // skip leading blanks or ';'
 105   while (*cmd == ' ' || *cmd == ';') cmd++;
 106 
 107   if (*cmd == '\0') return NULL;
 108 
 109   const char * cmdend = cmd;
 110   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
 111 
 112   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
 113 
 114   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
 115   return buf;
 116 }
 117 
 118 static void print_bug_submit_message(outputStream *out, Thread *thread) {
 119   if (out == NULL) return;
 120   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 121   out->print_raw   ("#   ");
 122   out->print_raw_cr(Arguments::java_vendor_url_bug());
 123   // If the crash is in native code, encourage user to submit a bug to the
 124   // provider of that code.
 125   if (thread && thread->is_Java_thread() &&
 126       !thread->is_hidden_from_external_view()) {
 127     JavaThread* jt = (JavaThread*)thread;
 128     if (jt->thread_state() == _thread_in_native) {
 129       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 130     }
 131   }
 132   out->print_raw_cr("#");
 133 }
 134 
 135 bool VMError::coredump_status;
 136 char VMError::coredump_message[O_BUFLEN];
 137 
 138 void VMError::record_coredump_status(const char* message, bool status) {
 139   coredump_status = status;
 140   strncpy(coredump_message, message, sizeof(coredump_message));
 141   coredump_message[sizeof(coredump_message)-1] = 0;
 142 }
 143 
 144 // Return a string to describe the error
 145 char* VMError::error_string(char* buf, int buflen) {
 146   char signame_buf[64];
 147   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 148 
 149   if (signame) {
 150     jio_snprintf(buf, buflen,
 151                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 152                  signame, _id, _pc,
 153                  os::current_process_id(), os::current_thread_id());
 154   } else if (_filename != NULL && _lineno > 0) {
 155     // skip directory names
 156     char separator = os::file_separator()[0];
 157     const char *p = strrchr(_filename, separator);
 158     int n = jio_snprintf(buf, buflen,
 159                          "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT,
 160                          p ? p + 1 : _filename, _lineno,
 161                          os::current_process_id(), os::current_thread_id());
 162     if (n >= 0 && n < buflen && _message) {
 163       if (strlen(_detail_msg) > 0) {
 164         jio_snprintf(buf + n, buflen - n, "%s%s: %s",
 165         os::line_separator(), _message, _detail_msg);
 166       } else {
 167         jio_snprintf(buf + n, buflen - n, "%sError: %s",
 168                      os::line_separator(), _message);
 169       }
 170     }
 171   } else {
 172     jio_snprintf(buf, buflen,
 173                  "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 174                  _id, os::current_process_id(), os::current_thread_id());
 175   }
 176 
 177   return buf;
 178 }
 179 
 180 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 181                                 char* buf, int buflen, bool verbose) {
 182 #ifdef ZERO
 183   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 184     // StackFrameStream uses the frame anchor, which may not have
 185     // been set up.  This can be done at any time in Zero, however,
 186     // so if it hasn't been set up then we just set it up now and
 187     // clear it again when we're done.
 188     bool has_last_Java_frame = jt->has_last_Java_frame();
 189     if (!has_last_Java_frame)
 190       jt->set_last_Java_frame();
 191     st->print("Java frames:");
 192 
 193     // If the top frame is a Shark frame and the frame anchor isn't
 194     // set up then it's possible that the information in the frame
 195     // is garbage: it could be from a previous decache, or it could
 196     // simply have never been written.  So we print a warning...
 197     StackFrameStream sfs(jt);
 198     if (!has_last_Java_frame && !sfs.is_done()) {
 199       if (sfs.current()->zeroframe()->is_shark_frame()) {
 200         st->print(" (TOP FRAME MAY BE JUNK)");
 201       }
 202     }
 203     st->cr();
 204 
 205     // Print the frames
 206     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 207       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 208       st->cr();
 209     }
 210 
 211     // Reset the frame anchor if necessary
 212     if (!has_last_Java_frame)
 213       jt->reset_last_Java_frame();
 214   }
 215 #else
 216   if (jt->has_last_Java_frame()) {
 217     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 218     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 219       sfs.current()->print_on_error(st, buf, buflen, verbose);
 220       st->cr();
 221     }
 222   }
 223 #endif // ZERO
 224 }
 225 
 226 void VMError::print_native_stack(outputStream* st, frame fr, Thread* t, char* buf, int buf_size) {
 227 
 228   // see if it's a valid frame
 229   if (fr.pc()) {
 230     st->print_cr("Native frames: (J=compiled Java code, A=aot compiled Java code, j=interpreted, Vv=VM code, C=native code)");
 231 
 232     int count = 0;
 233     while (count++ < StackPrintLimit) {
 234       fr.print_on_error(st, buf, buf_size);
 235       st->cr();
 236       // Compiled code may use EBP register on x86 so it looks like
 237       // non-walkable C frame. Use frame.sender() for java frames.
 238       if (t && t->is_Java_thread()) {
 239         // Catch very first native frame by using stack address.
 240         // For JavaThread stack_base and stack_size should be set.
 241         if (!t->on_local_stack((address)(fr.real_fp() + 1))) {
 242           break;
 243         }
 244         if (fr.is_java_frame() || fr.is_native_frame() || fr.is_runtime_frame()) {
 245           RegisterMap map((JavaThread*)t, false); // No update
 246           fr = fr.sender(&map);
 247         } else {
 248           fr = os::get_sender_for_C_frame(&fr);
 249         }
 250       } else {
 251         // is_first_C_frame() does only simple checks for frame pointer,
 252         // it will pass if java compiled code has a pointer in EBP.
 253         if (os::is_first_C_frame(&fr)) break;
 254         fr = os::get_sender_for_C_frame(&fr);
 255       }
 256     }
 257 
 258     if (count > StackPrintLimit) {
 259       st->print_cr("...<more frames>...");
 260     }
 261 
 262     st->cr();
 263   }
 264 }
 265 
 266 static void print_oom_reasons(outputStream* st) {
 267   st->print_cr("# Possible reasons:");
 268   st->print_cr("#   The system is out of physical RAM or swap space");
 269   if (UseCompressedOops) {
 270     st->print_cr("#   The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap");
 271   }
 272   if (LogBytesPerWord == 2) {
 273     st->print_cr("#   In 32 bit mode, the process size limit was hit");
 274   }
 275   st->print_cr("# Possible solutions:");
 276   st->print_cr("#   Reduce memory load on the system");
 277   st->print_cr("#   Increase physical memory or swap space");
 278   st->print_cr("#   Check if swap backing store is full");
 279   if (LogBytesPerWord == 2) {
 280     st->print_cr("#   Use 64 bit Java on a 64 bit OS");
 281   }
 282   st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
 283   st->print_cr("#   Decrease number of Java threads");
 284   st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
 285   st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
 286   if (UseCompressedOops) {
 287     switch (Universe::narrow_oop_mode()) {
 288       case Universe::UnscaledNarrowOop:
 289         st->print_cr("#   JVM is running with Unscaled Compressed Oops mode in which the Java heap is");
 290         st->print_cr("#     placed in the first 4GB address space. The Java Heap base address is the");
 291         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 292         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 4GB virtual address.");
 293         break;
 294       case Universe::ZeroBasedNarrowOop:
 295         st->print_cr("#   JVM is running with Zero Based Compressed Oops mode in which the Java heap is");
 296         st->print_cr("#     placed in the first 32GB address space. The Java Heap base address is the");
 297         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 298         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 32GB virtual address.");
 299         break;
 300       default:
 301         break;
 302     }
 303   }
 304   st->print_cr("# This output file may be truncated or incomplete.");
 305 }
 306 
 307 static const char* gc_mode() {
 308   if (UseG1GC)            return "g1 gc";
 309   if (UseParallelGC)      return "parallel gc";
 310   if (UseConcMarkSweepGC) return "concurrent mark sweep gc";
 311   if (UseSerialGC)        return "serial gc";
 312   return "ERROR in GC mode";
 313 }
 314 
 315 static void report_vm_version(outputStream* st, char* buf, int buflen) {
 316    // VM version
 317    st->print_cr("#");
 318    JDK_Version::current().to_string(buf, buflen);
 319    const char* runtime_name = JDK_Version::runtime_name() != NULL ?
 320                                 JDK_Version::runtime_name() : "";
 321    const char* runtime_version = JDK_Version::runtime_version() != NULL ?
 322                                    JDK_Version::runtime_version() : "";
 323    const char* jdk_debug_level = Abstract_VM_Version::printable_jdk_debug_level() != NULL ?
 324                                    Abstract_VM_Version::printable_jdk_debug_level() : "";
 325 
 326    st->print_cr("# JRE version: %s (%s) (%sbuild %s)", runtime_name, buf,
 327                  jdk_debug_level, runtime_version);
 328 
 329    // This is the long version with some default settings added
 330    st->print_cr("# Java VM: %s (%s%s, %s%s%s%s%s, %s, %s)",
 331                  Abstract_VM_Version::vm_name(),
 332                  jdk_debug_level,
 333                  Abstract_VM_Version::vm_release(),
 334                  Abstract_VM_Version::vm_info_string(),
 335                  TieredCompilation ? ", tiered" : "",
 336 #if INCLUDE_JVMCI
 337                  EnableJVMCI ? ", jvmci" : "",
 338                  UseJVMCICompiler ? ", jvmci compiler" : "",
 339 #else
 340                  "", "",
 341 #endif
 342                  UseCompressedOops ? ", compressed oops" : "",
 343                  gc_mode(),
 344                  Abstract_VM_Version::vm_platform_string()
 345                );
 346 }
 347 
 348 // This is the main function to report a fatal error. Only one thread can
 349 // call this function, so we don't need to worry about MT-safety. But it's
 350 // possible that the error handler itself may crash or die on an internal
 351 // error, for example, when the stack/heap is badly damaged. We must be
 352 // able to handle recursive errors that happen inside error handler.
 353 //
 354 // Error reporting is done in several steps. If a crash or internal error
 355 // occurred when reporting an error, the nested signal/exception handler
 356 // can skip steps that are already (or partially) done. Error reporting will
 357 // continue from the next step. This allows us to retrieve and print
 358 // information that may be unsafe to get after a fatal error. If it happens,
 359 // you may find nested report_and_die() frames when you look at the stack
 360 // in a debugger.
 361 //
 362 // In general, a hang in error handler is much worse than a crash or internal
 363 // error, as it's harder to recover from a hang. Deadlock can happen if we
 364 // try to grab a lock that is already owned by current thread, or if the
 365 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 366 // error handler and all the functions it called should avoid grabbing any
 367 // lock. An important thing to notice is that memory allocation needs a lock.
 368 //
 369 // We should avoid using large stack allocated buffers. Many errors happen
 370 // when stack space is already low. Making things even worse is that there
 371 // could be nested report_and_die() calls on stack (see above). Only one
 372 // thread can report error, so large buffers are statically allocated in data
 373 // segment.
 374 
 375 int          VMError::_current_step;
 376 const char*  VMError::_current_step_info;
 377 
 378 volatile jlong VMError::_reporting_start_time = -1;
 379 volatile bool VMError::_reporting_did_timeout = false;
 380 volatile jlong VMError::_step_start_time = -1;
 381 volatile bool VMError::_step_did_timeout = false;
 382 
 383 // Helper, return current timestamp for timeout handling.
 384 jlong VMError::get_current_timestamp() {
 385   return os::javaTimeNanos();
 386 }
 387 // Factor to translate the timestamp to seconds.
 388 #define TIMESTAMP_TO_SECONDS_FACTOR (1000 * 1000 * 1000)
 389 
 390 void VMError::record_reporting_start_time() {
 391   const jlong now = get_current_timestamp();
 392   Atomic::store(now, &_reporting_start_time);
 393 }
 394 
 395 jlong VMError::get_reporting_start_time() {
 396   return Atomic::load(&_reporting_start_time);
 397 }
 398 
 399 void VMError::record_step_start_time() {
 400   const jlong now = get_current_timestamp();
 401   Atomic::store(now, &_step_start_time);
 402 }
 403 
 404 jlong VMError::get_step_start_time() {
 405   return Atomic::load(&_step_start_time);
 406 }
 407 
 408 void VMError::report(outputStream* st, bool _verbose) {
 409 
 410 # define BEGIN if (_current_step == 0) { _current_step = __LINE__;
 411 # define STEP(s) } if (_current_step < __LINE__) { _current_step = __LINE__; _current_step_info = s; \
 412   record_step_start_time(); _step_did_timeout = false;
 413 # define END }
 414 
 415   // don't allocate large buffer on stack
 416   static char buf[O_BUFLEN];
 417 
 418   BEGIN
 419 
 420   STEP("printing fatal error message")
 421 
 422     st->print_cr("#");
 423     if (should_report_bug(_id)) {
 424       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 425     } else {
 426       st->print_cr("# There is insufficient memory for the Java "
 427                    "Runtime Environment to continue.");
 428     }
 429 
 430 #ifndef PRODUCT
 431   // Error handler self tests
 432 
 433   // test secondary error handling. Test it twice, to test that resetting
 434   // error handler after a secondary crash works.
 435   STEP("test secondary crash 1")
 436     if (_verbose && TestCrashInErrorHandler != 0) {
 437       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 438         TestCrashInErrorHandler);
 439       controlled_crash(TestCrashInErrorHandler);
 440     }
 441 
 442   STEP("test secondary crash 2")
 443     if (_verbose && TestCrashInErrorHandler != 0) {
 444       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 445         TestCrashInErrorHandler);
 446       controlled_crash(TestCrashInErrorHandler);
 447     }
 448 
 449   // TestUnresponsiveErrorHandler: We want to test both step timeouts and global timeout.
 450   // Step to global timeout ratio is 4:1, so in order to be absolutely sure we hit the
 451   // global timeout, let's execute the timeout step five times.
 452   // See corresponding test in test/runtime/ErrorHandling/TimeoutInErrorHandlingTest.java
 453   #define TIMEOUT_TEST_STEP STEP("test unresponsive error reporting step") \
 454     if (_verbose && TestUnresponsiveErrorHandler) { os::infinite_sleep(); }
 455   TIMEOUT_TEST_STEP
 456   TIMEOUT_TEST_STEP
 457   TIMEOUT_TEST_STEP
 458   TIMEOUT_TEST_STEP
 459   TIMEOUT_TEST_STEP
 460 
 461   STEP("test safefetch in error handler")
 462     // test whether it is safe to use SafeFetch32 in Crash Handler. Test twice
 463     // to test that resetting the signal handler works correctly.
 464     if (_verbose && TestSafeFetchInErrorHandler) {
 465       st->print_cr("Will test SafeFetch...");
 466       if (CanUseSafeFetch32()) {
 467         int* const invalid_pointer = (int*) get_segfault_address();
 468         const int x = 0x76543210;
 469         int i1 = SafeFetch32(invalid_pointer, x);
 470         int i2 = SafeFetch32(invalid_pointer, x);
 471         if (i1 == x && i2 == x) {
 472           st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern
 473         } else {
 474           st->print_cr("??");
 475         }
 476       } else {
 477         st->print_cr("not possible; skipped.");
 478       }
 479     }
 480 #endif // PRODUCT
 481 
 482   STEP("printing type of error")
 483 
 484      switch(_id) {
 485        case OOM_MALLOC_ERROR:
 486        case OOM_MMAP_ERROR:
 487          if (_size) {
 488            st->print("# Native memory allocation ");
 489            st->print((_id == (int)OOM_MALLOC_ERROR) ? "(malloc) failed to allocate " :
 490                                                  "(mmap) failed to map ");
 491            jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
 492            st->print("%s", buf);
 493            st->print(" bytes");
 494            if (strlen(_detail_msg) > 0) {
 495              st->print(" for ");
 496              st->print("%s", _detail_msg);
 497            }
 498            st->cr();
 499          } else {
 500            if (strlen(_detail_msg) > 0) {
 501              st->print("# ");
 502              st->print_cr("%s", _detail_msg);
 503            }
 504          }
 505          // In error file give some solutions
 506          if (_verbose) {
 507            print_oom_reasons(st);
 508          } else {
 509            return;  // that's enough for the screen
 510          }
 511          break;
 512        case INTERNAL_ERROR:
 513        default:
 514          break;
 515      }
 516 
 517   STEP("printing exception/signal name")
 518 
 519      st->print_cr("#");
 520      st->print("#  ");
 521      // Is it an OS exception/signal?
 522      if (os::exception_name(_id, buf, sizeof(buf))) {
 523        st->print("%s", buf);
 524        st->print(" (0x%x)", _id);                // signal number
 525        st->print(" at pc=" PTR_FORMAT, p2i(_pc));
 526      } else {
 527        if (should_report_bug(_id)) {
 528          st->print("Internal Error");
 529        } else {
 530          st->print("Out of Memory Error");
 531        }
 532        if (_filename != NULL && _lineno > 0) {
 533 #ifdef PRODUCT
 534          // In product mode chop off pathname?
 535          char separator = os::file_separator()[0];
 536          const char *p = strrchr(_filename, separator);
 537          const char *file = p ? p+1 : _filename;
 538 #else
 539          const char *file = _filename;
 540 #endif
 541          st->print(" (%s:%d)", file, _lineno);
 542        } else {
 543          st->print(" (0x%x)", _id);
 544        }
 545      }
 546 
 547   STEP("printing current thread and pid")
 548 
 549      // process id, thread id
 550      st->print(", pid=%d", os::current_process_id());
 551      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 552      st->cr();
 553 
 554   STEP("printing error message")
 555 
 556      if (should_report_bug(_id)) {  // already printed the message.
 557        // error message
 558        if (strlen(_detail_msg) > 0) {
 559          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
 560        } else if (_message) {
 561          st->print_cr("#  Error: %s", _message);
 562        }
 563      }
 564 
 565   STEP("printing Java version string")
 566 
 567      report_vm_version(st, buf, sizeof(buf));
 568 
 569   STEP("printing problematic frame")
 570 
 571      // Print current frame if we have a context (i.e. it's a crash)
 572      if (_context) {
 573        st->print_cr("# Problematic frame:");
 574        st->print("# ");
 575        frame fr = os::fetch_frame_from_context(_context);
 576        fr.print_on_error(st, buf, sizeof(buf));
 577        st->cr();
 578        st->print_cr("#");
 579      }
 580 
 581   STEP("printing core file information")
 582     st->print("# ");
 583     if (CreateCoredumpOnCrash) {
 584       if (coredump_status) {
 585         st->print("Core dump will be written. Default location: %s", coredump_message);
 586       } else {
 587         st->print("No core dump will be written. %s", coredump_message);
 588       }
 589     } else {
 590       st->print("CreateCoredumpOnCrash turned off, no core file dumped");
 591     }
 592     st->cr();
 593     st->print_cr("#");
 594 
 595   STEP("printing bug submit message")
 596 
 597      if (should_report_bug(_id) && _verbose) {
 598        print_bug_submit_message(st, _thread);
 599      }
 600 
 601   STEP("printing summary")
 602 
 603      if (_verbose) {
 604        st->cr();
 605        st->print_cr("---------------  S U M M A R Y ------------");
 606        st->cr();
 607      }
 608 
 609   STEP("printing VM option summary")
 610 
 611      if (_verbose) {
 612        // VM options
 613        Arguments::print_summary_on(st);
 614        st->cr();
 615      }
 616 
 617   STEP("printing summary machine and OS info")
 618 
 619      if (_verbose) {
 620        os::print_summary_info(st, buf, sizeof(buf));
 621      }
 622 
 623 
 624   STEP("printing date and time")
 625 
 626      if (_verbose) {
 627        os::print_date_and_time(st, buf, sizeof(buf));
 628      }
 629 
 630   STEP("printing thread")
 631 
 632      if (_verbose) {
 633        st->cr();
 634        st->print_cr("---------------  T H R E A D  ---------------");
 635        st->cr();
 636      }
 637 
 638   STEP("printing current thread")
 639 
 640      // current thread
 641      if (_verbose) {
 642        if (_thread) {
 643          st->print("Current thread (" PTR_FORMAT "):  ", p2i(_thread));
 644          _thread->print_on_error(st, buf, sizeof(buf));
 645          st->cr();
 646        } else {
 647          st->print_cr("Current thread is native thread");
 648        }
 649        st->cr();
 650      }
 651 
 652   STEP("printing current compile task")
 653 
 654      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 655         CompilerThread* t = (CompilerThread*)_thread;
 656         if (t->task()) {
 657            st->cr();
 658            st->print_cr("Current CompileTask:");
 659            t->task()->print_line_on_error(st, buf, sizeof(buf));
 660            st->cr();
 661         }
 662      }
 663 
 664 
 665   STEP("printing stack bounds")
 666 
 667      if (_verbose) {
 668        st->print("Stack: ");
 669 
 670        address stack_top;
 671        size_t stack_size;
 672 
 673        if (_thread) {
 674           stack_top = _thread->stack_base();
 675           stack_size = _thread->stack_size();
 676        } else {
 677           stack_top = os::current_stack_base();
 678           stack_size = os::current_stack_size();
 679        }
 680 
 681        address stack_bottom = stack_top - stack_size;
 682        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", p2i(stack_bottom), p2i(stack_top));
 683 
 684        frame fr = _context ? os::fetch_frame_from_context(_context)
 685                            : os::current_frame();
 686 
 687        if (fr.sp()) {
 688          st->print(",  sp=" PTR_FORMAT, p2i(fr.sp()));
 689          size_t free_stack_size = pointer_delta(fr.sp(), stack_bottom, 1024);
 690          st->print(",  free space=" SIZE_FORMAT "k", free_stack_size);
 691        }
 692 
 693        st->cr();
 694      }
 695 
 696   STEP("printing native stack")
 697 
 698    if (_verbose) {
 699      if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) {
 700        // We have printed the native stack in platform-specific code
 701        // Windows/x64 needs special handling.
 702      } else {
 703        frame fr = _context ? os::fetch_frame_from_context(_context)
 704                            : os::current_frame();
 705 
 706        print_native_stack(st, fr, _thread, buf, sizeof(buf));
 707      }
 708    }
 709 
 710   STEP("printing Java stack")
 711 
 712      if (_verbose && _thread && _thread->is_Java_thread()) {
 713        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 714      }
 715 
 716   STEP("printing target Java thread stack")
 717 
 718      // printing Java thread stack trace if it is involved in GC crash
 719      if (_verbose && _thread && (_thread->is_Named_thread())) {
 720        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 721        if (jt != NULL) {
 722          st->print_cr("JavaThread " PTR_FORMAT " (nid = %d) was being processed", p2i(jt), jt->osthread()->thread_id());
 723          print_stack_trace(st, jt, buf, sizeof(buf), true);
 724        }
 725      }
 726 
 727   STEP("printing siginfo")
 728 
 729      // signal no, signal code, address that caused the fault
 730      if (_verbose && _siginfo) {
 731        st->cr();
 732        os::print_siginfo(st, _siginfo);
 733        st->cr();
 734      }
 735 
 736   STEP("CDS archive access warning")
 737 
 738      // Print an explicit hint if we crashed on access to the CDS archive.
 739      if (_verbose && _siginfo) {
 740        check_failing_cds_access(st, _siginfo);
 741        st->cr();
 742      }
 743 
 744   STEP("printing register info")
 745 
 746      // decode register contents if possible
 747      if (_verbose && _context && Universe::is_fully_initialized()) {
 748        os::print_register_info(st, _context);
 749        st->cr();
 750      }
 751 
 752   STEP("printing registers, top of stack, instructions near pc")
 753 
 754      // registers, top of stack, instructions near pc
 755      if (_verbose && _context) {
 756        os::print_context(st, _context);
 757        st->cr();
 758      }
 759 
 760   STEP("printing code blob if possible")
 761 
 762      if (_verbose && _context) {
 763        CodeBlob* cb = CodeCache::find_blob(_pc);
 764        if (cb != NULL) {
 765          if (Interpreter::contains(_pc)) {
 766            // The interpreter CodeBlob is very large so try to print the codelet instead.
 767            InterpreterCodelet* codelet = Interpreter::codelet_containing(_pc);
 768            if (codelet != NULL) {
 769              codelet->print_on(st);
 770              Disassembler::decode(codelet->code_begin(), codelet->code_end(), st);
 771            }
 772          } else {
 773            StubCodeDesc* desc = StubCodeDesc::desc_for(_pc);
 774            if (desc != NULL) {
 775              desc->print_on(st);
 776              Disassembler::decode(desc->begin(), desc->end(), st);
 777            } else {
 778              Disassembler::decode(cb, st);
 779              st->cr();
 780            }
 781          }
 782        }
 783      }
 784 
 785   STEP("printing VM operation")
 786 
 787      if (_verbose && _thread && _thread->is_VM_thread()) {
 788         VMThread* t = (VMThread*)_thread;
 789         VM_Operation* op = t->vm_operation();
 790         if (op) {
 791           op->print_on_error(st);
 792           st->cr();
 793           st->cr();
 794         }
 795      }
 796 
 797   STEP("printing process")
 798 
 799      if (_verbose) {
 800        st->cr();
 801        st->print_cr("---------------  P R O C E S S  ---------------");
 802        st->cr();
 803      }
 804 
 805   STEP("printing all threads")
 806 
 807      // all threads
 808      if (_verbose && _thread) {
 809        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 810        st->cr();
 811      }
 812 
 813   STEP("printing VM state")
 814 
 815      if (_verbose) {
 816        // Safepoint state
 817        st->print("VM state:");
 818 
 819        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 820        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 821        else st->print("not at safepoint");
 822 
 823        // Also see if error occurred during initialization or shutdown
 824        if (!Universe::is_fully_initialized()) {
 825          st->print(" (not fully initialized)");
 826        } else if (VM_Exit::vm_exited()) {
 827          st->print(" (shutting down)");
 828        } else {
 829          st->print(" (normal execution)");
 830        }
 831        st->cr();
 832        st->cr();
 833      }
 834 
 835   STEP("printing owned locks on error")
 836 
 837      // mutexes/monitors that currently have an owner
 838      if (_verbose) {
 839        print_owned_locks_on_error(st);
 840        st->cr();
 841      }
 842 
 843   STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
 844 
 845      if (_verbose && Exceptions::has_exception_counts()) {
 846        st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 847        Exceptions::print_exception_counts_on_error(st);
 848        st->cr();
 849      }
 850 
 851   STEP("printing compressed oops mode")
 852 
 853      if (_verbose && UseCompressedOops) {
 854        Universe::print_compressed_oops_mode(st);
 855        if (UseCompressedClassPointers) {
 856          Metaspace::print_compressed_class_space(st);
 857        }
 858        st->cr();
 859      }
 860 
 861   STEP("printing heap information")
 862 
 863      if (_verbose && Universe::is_fully_initialized()) {
 864        Universe::heap()->print_on_error(st);
 865        st->cr();
 866        st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
 867        st->cr();
 868      }
 869 
 870   STEP("printing code cache information")
 871 
 872      if (_verbose && Universe::is_fully_initialized()) {
 873        // print code cache information before vm abort
 874        CodeCache::print_summary(st);
 875        st->cr();
 876      }
 877 
 878   STEP("printing ring buffers")
 879 
 880      if (_verbose) {
 881        Events::print_all(st);
 882        st->cr();
 883      }
 884 
 885   STEP("printing dynamic libraries")
 886 
 887      if (_verbose) {
 888        // dynamic libraries, or memory map
 889        os::print_dll_info(st);
 890        st->cr();
 891      }
 892 
 893   STEP("printing native decoder state")
 894 
 895      if (_verbose) {
 896        Decoder::print_state_on(st);
 897        st->cr();
 898      }
 899 
 900   STEP("printing VM options")
 901 
 902      if (_verbose) {
 903        // VM options
 904        Arguments::print_on(st);
 905        st->cr();
 906      }
 907 
 908   STEP("printing warning if internal testing API used")
 909 
 910      if (WhiteBox::used()) {
 911        st->print_cr("Unsupported internal testing APIs have been used.");
 912        st->cr();
 913      }
 914 
 915   STEP("printing log configuration")
 916     if (_verbose){
 917       st->print_cr("Logging:");
 918       LogConfiguration::describe_current_configuration(st);
 919       st->cr();
 920     }
 921 
 922   STEP("printing all environment variables")
 923 
 924      if (_verbose) {
 925        os::print_environment_variables(st, env_list);
 926        st->cr();
 927      }
 928 
 929   STEP("printing signal handlers")
 930 
 931      if (_verbose) {
 932        os::print_signal_handlers(st, buf, sizeof(buf));
 933        st->cr();
 934      }
 935 
 936   STEP("Native Memory Tracking")
 937      if (_verbose) {
 938        MemTracker::error_report(st);
 939      }
 940 
 941   STEP("printing system")
 942 
 943      if (_verbose) {
 944        st->cr();
 945        st->print_cr("---------------  S Y S T E M  ---------------");
 946        st->cr();
 947      }
 948 
 949   STEP("printing OS information")
 950 
 951      if (_verbose) {
 952        os::print_os_info(st);
 953        st->cr();
 954      }
 955 
 956   STEP("printing CPU info")
 957      if (_verbose) {
 958        os::print_cpu_info(st, buf, sizeof(buf));
 959        st->cr();
 960      }
 961 
 962   STEP("printing memory info")
 963 
 964      if (_verbose) {
 965        os::print_memory_info(st);
 966        st->cr();
 967      }
 968 
 969   STEP("printing internal vm info")
 970 
 971      if (_verbose) {
 972        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 973        st->cr();
 974      }
 975 
 976   // print a defined marker to show that error handling finished correctly.
 977   STEP("printing end marker")
 978 
 979      if (_verbose) {
 980        st->print_cr("END.");
 981      }
 982 
 983   END
 984 
 985 # undef BEGIN
 986 # undef STEP
 987 # undef END
 988 }
 989 
 990 // Report for the vm_info_cmd. This prints out the information above omitting
 991 // crash and thread specific information.  If output is added above, it should be added
 992 // here also, if it is safe to call during a running process.
 993 void VMError::print_vm_info(outputStream* st) {
 994 
 995   char buf[O_BUFLEN];
 996   report_vm_version(st, buf, sizeof(buf));
 997 
 998   // STEP("printing summary")
 999 
1000   st->cr();
1001   st->print_cr("---------------  S U M M A R Y ------------");
1002   st->cr();
1003 
1004   // STEP("printing VM option summary")
1005 
1006   // VM options
1007   Arguments::print_summary_on(st);
1008   st->cr();
1009 
1010   // STEP("printing summary machine and OS info")
1011 
1012   os::print_summary_info(st, buf, sizeof(buf));
1013 
1014   // STEP("printing date and time")
1015 
1016   os::print_date_and_time(st, buf, sizeof(buf));
1017 
1018   // Skip: STEP("printing thread")
1019 
1020   // STEP("printing process")
1021 
1022   st->cr();
1023   st->print_cr("---------------  P R O C E S S  ---------------");
1024   st->cr();
1025 
1026   // STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
1027 
1028   if (Exceptions::has_exception_counts()) {
1029     st->print_cr("OutOfMemory and StackOverflow Exception counts:");
1030     Exceptions::print_exception_counts_on_error(st);
1031     st->cr();
1032   }
1033 
1034   // STEP("printing compressed oops mode")
1035 
1036   if (UseCompressedOops) {
1037     Universe::print_compressed_oops_mode(st);
1038     if (UseCompressedClassPointers) {
1039       Metaspace::print_compressed_class_space(st);
1040     }
1041     st->cr();
1042   }
1043 
1044   // STEP("printing heap information")
1045 
1046   if (Universe::is_fully_initialized()) {
1047     MutexLocker hl(Heap_lock);
1048     Universe::heap()->print_on_error(st);
1049     st->cr();
1050     st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
1051     st->cr();
1052   }
1053 
1054   // STEP("printing code cache information")
1055 
1056   if (Universe::is_fully_initialized()) {
1057     // print code cache information before vm abort
1058     CodeCache::print_summary(st);
1059     st->cr();
1060   }
1061 
1062   // STEP("printing ring buffers")
1063 
1064   Events::print_all(st);
1065   st->cr();
1066 
1067   // STEP("printing dynamic libraries")
1068 
1069   // dynamic libraries, or memory map
1070   os::print_dll_info(st);
1071   st->cr();
1072 
1073   // STEP("printing VM options")
1074 
1075   // VM options
1076   Arguments::print_on(st);
1077   st->cr();
1078 
1079   // STEP("printing warning if internal testing API used")
1080 
1081   if (WhiteBox::used()) {
1082     st->print_cr("Unsupported internal testing APIs have been used.");
1083     st->cr();
1084   }
1085 
1086   // STEP("printing log configuration")
1087   st->print_cr("Logging:");
1088   LogConfiguration::describe(st);
1089   st->cr();
1090 
1091   // STEP("printing all environment variables")
1092 
1093   os::print_environment_variables(st, env_list);
1094   st->cr();
1095 
1096   // STEP("printing signal handlers")
1097 
1098   os::print_signal_handlers(st, buf, sizeof(buf));
1099   st->cr();
1100 
1101   // STEP("Native Memory Tracking")
1102 
1103   MemTracker::error_report(st);
1104 
1105   // STEP("printing system")
1106 
1107   st->cr();
1108   st->print_cr("---------------  S Y S T E M  ---------------");
1109   st->cr();
1110 
1111   // STEP("printing OS information")
1112 
1113   os::print_os_info(st);
1114   st->cr();
1115 
1116   // STEP("printing CPU info")
1117 
1118   os::print_cpu_info(st, buf, sizeof(buf));
1119   st->cr();
1120 
1121   // STEP("printing memory info")
1122 
1123   os::print_memory_info(st);
1124   st->cr();
1125 
1126   // STEP("printing internal vm info")
1127 
1128   st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
1129   st->cr();
1130 
1131   // print a defined marker to show that error handling finished correctly.
1132   // STEP("printing end marker")
1133 
1134   st->print_cr("END.");
1135 }
1136 
1137 volatile intptr_t VMError::first_error_tid = -1;
1138 
1139 // An error could happen before tty is initialized or after it has been
1140 // destroyed.
1141 // Please note: to prevent large stack allocations, the log- and
1142 // output-stream use a global scratch buffer for format printing.
1143 // (see VmError::report_and_die(). Access to those streams is synchronized
1144 // in  VmError::report_and_die() - there is only one reporting thread at
1145 // any given time.
1146 fdStream VMError::out(defaultStream::output_fd());
1147 fdStream VMError::log; // error log used by VMError::report_and_die()
1148 
1149 /** Expand a pattern into a buffer starting at pos and open a file using constructed path */
1150 static int expand_and_open(const char* pattern, char* buf, size_t buflen, size_t pos) {
1151   int fd = -1;
1152   if (Arguments::copy_expand_pid(pattern, strlen(pattern), &buf[pos], buflen - pos)) {
1153     // the O_EXCL flag will cause the open to fail if the file exists
1154     fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0666);
1155   }
1156   return fd;
1157 }
1158 
1159 /**
1160  * Construct file name for a log file and return it's file descriptor.
1161  * Name and location depends on pattern, default_pattern params and access
1162  * permissions.
1163  */
1164 static int prepare_log_file(const char* pattern, const char* default_pattern, char* buf, size_t buflen) {
1165   int fd = -1;
1166 
1167   // If possible, use specified pattern to construct log file name
1168   if (pattern != NULL) {
1169     fd = expand_and_open(pattern, buf, buflen, 0);
1170   }
1171 
1172   // Either user didn't specify, or the user's location failed,
1173   // so use the default name in the current directory
1174   if (fd == -1) {
1175     const char* cwd = os::get_current_directory(buf, buflen);
1176     if (cwd != NULL) {
1177       size_t pos = strlen(cwd);
1178       int fsep_len = jio_snprintf(&buf[pos], buflen-pos, "%s", os::file_separator());
1179       pos += fsep_len;
1180       if (fsep_len > 0) {
1181         fd = expand_and_open(default_pattern, buf, buflen, pos);
1182       }
1183     }
1184   }
1185 
1186    // try temp directory if it exists.
1187    if (fd == -1) {
1188      const char* tmpdir = os::get_temp_directory();
1189      if (tmpdir != NULL && strlen(tmpdir) > 0) {
1190        int pos = jio_snprintf(buf, buflen, "%s%s", tmpdir, os::file_separator());
1191        if (pos > 0) {
1192          fd = expand_and_open(default_pattern, buf, buflen, pos);
1193        }
1194      }
1195    }
1196 
1197   return fd;
1198 }
1199 
1200 int         VMError::_id;
1201 const char* VMError::_message;
1202 char        VMError::_detail_msg[1024];
1203 Thread*     VMError::_thread;
1204 address     VMError::_pc;
1205 void*       VMError::_siginfo;
1206 void*       VMError::_context;
1207 const char* VMError::_filename;
1208 int         VMError::_lineno;
1209 size_t      VMError::_size;
1210 
1211 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo,
1212                              void* context, const char* detail_fmt, ...)
1213 {
1214   va_list detail_args;
1215   va_start(detail_args, detail_fmt);
1216   report_and_die(sig, NULL, detail_fmt, detail_args, thread, pc, siginfo, context, NULL, 0, 0);
1217   va_end(detail_args);
1218 }
1219 
1220 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context)
1221 {
1222   report_and_die(thread, sig, pc, siginfo, context, "%s", "");
1223 }
1224 
1225 void VMError::report_and_die(const char* message, const char* detail_fmt, ...)
1226 {
1227   va_list detail_args;
1228   va_start(detail_args, detail_fmt);
1229   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, NULL, NULL, NULL, NULL, NULL, 0, 0);
1230   va_end(detail_args);
1231 }
1232 
1233 void VMError::report_and_die(const char* message)
1234 {
1235   report_and_die(message, "%s", "");
1236 }
1237 
1238 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, const char* message,
1239                              const char* detail_fmt, va_list detail_args)
1240 {
1241   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, 0);
1242 }
1243 
1244 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, size_t size,
1245                              VMErrorType vm_err_type, const char* detail_fmt, va_list detail_args) {
1246   report_and_die(vm_err_type, NULL, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, size);
1247 }
1248 
1249 void VMError::report_and_die(int id, const char* message, const char* detail_fmt, va_list detail_args,
1250                              Thread* thread, address pc, void* siginfo, void* context, const char* filename,
1251                              int lineno, size_t size)
1252 {
1253   // Don't allocate large buffer on stack
1254   static char buffer[O_BUFLEN];
1255   out.set_scratch_buffer(buffer, sizeof(buffer));
1256   log.set_scratch_buffer(buffer, sizeof(buffer));
1257 
1258   // How many errors occurred in error handler when reporting first_error.
1259   static int recursive_error_count;
1260 
1261   // We will first print a brief message to standard out (verbose = false),
1262   // then save detailed information in log file (verbose = true).
1263   static bool out_done = false;         // done printing to standard out
1264   static bool log_done = false;         // done saving error log
1265   static bool transmit_report_done = false; // done error reporting
1266 
1267   if (SuppressFatalErrorMessage) {
1268       os::abort(CreateCoredumpOnCrash);
1269   }
1270   intptr_t mytid = os::current_thread_id();
1271   if (first_error_tid == -1 &&
1272       Atomic::cmpxchg_ptr(mytid, &first_error_tid, -1) == -1) {
1273 
1274     // Initialize time stamps to use the same base.
1275     out.time_stamp().update_to(1);
1276     log.time_stamp().update_to(1);
1277 
1278     _id = id;
1279     _message = message;
1280     _thread = thread;
1281     _pc = pc;
1282     _siginfo = siginfo;
1283     _context = context;
1284     _filename = filename;
1285     _lineno = lineno;
1286     _size = size;
1287     jio_vsnprintf(_detail_msg, sizeof(_detail_msg), detail_fmt, detail_args);
1288 
1289     // first time
1290     _error_reported = true;
1291 
1292     reporting_started();
1293     record_reporting_start_time();
1294 
1295     if (ShowMessageBoxOnError || PauseAtExit) {
1296       show_message_box(buffer, sizeof(buffer));
1297 
1298       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
1299       // WatcherThread can kill JVM if the error handler hangs.
1300       ShowMessageBoxOnError = false;
1301     }
1302 
1303     os::check_dump_limit(buffer, sizeof(buffer));
1304 
1305     // reset signal handlers or exception filter; make sure recursive crashes
1306     // are handled properly.
1307     reset_signal_handlers();
1308 
1309     TRACE_VM_ERROR();
1310 
1311   } else {
1312     // If UseOsErrorReporting we call this for each level of the call stack
1313     // while searching for the exception handler.  Only the first level needs
1314     // to be reported.
1315     if (UseOSErrorReporting && log_done) return;
1316 
1317     // This is not the first error, see if it happened in a different thread
1318     // or in the same thread during error reporting.
1319     if (first_error_tid != mytid) {
1320       char msgbuf[64];
1321       jio_snprintf(msgbuf, sizeof(msgbuf),
1322                    "[thread " INTX_FORMAT " also had an error]",
1323                    mytid);
1324       out.print_raw_cr(msgbuf);
1325 
1326       // error reporting is not MT-safe, block current thread
1327       os::infinite_sleep();
1328 
1329     } else {
1330       if (recursive_error_count++ > 30) {
1331         out.print_raw_cr("[Too many errors, abort]");
1332         os::die();
1333       }
1334 
1335       outputStream* const st = log.is_open() ? &log : &out;
1336       st->cr();
1337 
1338       // Timeout handling.
1339       if (_step_did_timeout) {
1340         // The current step had a timeout. Lets continue reporting with the next step.
1341         st->print_raw("[timeout occurred during error reporting in step \"");
1342         st->print_raw(_current_step_info);
1343         st->print_cr("\"] after " INT64_FORMAT " s.",
1344                      (int64_t)
1345                      ((get_current_timestamp() - _step_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1346       } else if (_reporting_did_timeout) {
1347         // We hit ErrorLogTimeout. Reporting will stop altogether. Let's wrap things
1348         // up, the process is about to be stopped by the WatcherThread.
1349         st->print_cr("------ Timeout during error reporting after " INT64_FORMAT " s. ------",
1350                      (int64_t)
1351                      ((get_current_timestamp() - _reporting_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1352         st->flush();
1353         // Watcherthread is about to call os::die. Lets just wait.
1354         os::infinite_sleep();
1355       } else {
1356         // Crash or assert during error reporting. Lets continue reporting with the next step.
1357         jio_snprintf(buffer, sizeof(buffer),
1358            "[error occurred during error reporting (%s), id 0x%x]",
1359                    _current_step_info, _id);
1360         st->print_raw_cr(buffer);
1361         st->cr();
1362       }
1363     }
1364   }
1365 
1366   // print to screen
1367   if (!out_done) {
1368     report(&out, false);
1369 
1370     out_done = true;
1371 
1372     _current_step = 0;
1373     _current_step_info = "";
1374   }
1375 
1376   // print to error log file
1377   if (!log_done) {
1378     // see if log file is already open
1379     if (!log.is_open()) {
1380       // open log file
1381       int fd = prepare_log_file(ErrorFile, "hs_err_pid%p.log", buffer, sizeof(buffer));
1382       if (fd != -1) {
1383         out.print_raw("# An error report file with more information is saved as:\n# ");
1384         out.print_raw_cr(buffer);
1385 
1386         log.set_fd(fd);
1387       } else {
1388         out.print_raw_cr("# Can not save log file, dump to screen..");
1389         log.set_fd(defaultStream::output_fd());
1390         /* Error reporting currently needs dumpfile.
1391          * Maybe implement direct streaming in the future.*/
1392         transmit_report_done = true;
1393       }
1394     }
1395 
1396     report(&log, true);
1397     log_done = true;
1398     _current_step = 0;
1399     _current_step_info = "";
1400 
1401     // Run error reporting to determine whether or not to report the crash.
1402     if (!transmit_report_done && should_report_bug(_id)) {
1403       transmit_report_done = true;
1404       const int fd2 = ::dup(log.fd());
1405       if (fd2 != -1) {
1406         FILE* const hs_err = ::fdopen(fd2, "r");
1407         if (NULL != hs_err) {
1408           ErrorReporter er;
1409           er.call(hs_err, buffer, O_BUFLEN);
1410           ::fclose(hs_err);
1411         }
1412       }
1413     }
1414 
1415     if (log.fd() != defaultStream::output_fd()) {
1416       close(log.fd());
1417     }
1418 
1419     log.set_fd(-1);
1420   }
1421 
1422   static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
1423   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
1424     skip_replay = true;
1425     ciEnv* env = ciEnv::current();
1426     if (env != NULL) {
1427       int fd = prepare_log_file(ReplayDataFile, "replay_pid%p.log", buffer, sizeof(buffer));
1428       if (fd != -1) {
1429         FILE* replay_data_file = os::open(fd, "w");
1430         if (replay_data_file != NULL) {
1431           fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
1432           env->dump_replay_data_unsafe(&replay_data_stream);
1433           out.print_raw("#\n# Compiler replay data is saved as:\n# ");
1434           out.print_raw_cr(buffer);
1435         } else {
1436           int e = errno;
1437           out.print_raw("#\n# Can't open file to dump replay data. Error: ");
1438           out.print_raw_cr(os::strerror(e));
1439         }
1440       }
1441     }
1442   }
1443 
1444   static bool skip_bug_url = !should_report_bug(_id);
1445   if (!skip_bug_url) {
1446     skip_bug_url = true;
1447 
1448     out.print_raw_cr("#");
1449     print_bug_submit_message(&out, _thread);
1450   }
1451 
1452   static bool skip_OnError = false;
1453   if (!skip_OnError && OnError && OnError[0]) {
1454     skip_OnError = true;
1455 
1456     // Flush output and finish logs before running OnError commands.
1457     ostream_abort();
1458 
1459     out.print_raw_cr("#");
1460     out.print_raw   ("# -XX:OnError=\"");
1461     out.print_raw   (OnError);
1462     out.print_raw_cr("\"");
1463 
1464     char* cmd;
1465     const char* ptr = OnError;
1466     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1467       out.print_raw   ("#   Executing ");
1468 #if defined(LINUX) || defined(_ALLBSD_SOURCE)
1469       out.print_raw   ("/bin/sh -c ");
1470 #elif defined(SOLARIS)
1471       out.print_raw   ("/usr/bin/sh -c ");
1472 #elif defined(WINDOWS)
1473       out.print_raw   ("cmd /C ");
1474 #endif
1475       out.print_raw   ("\"");
1476       out.print_raw   (cmd);
1477       out.print_raw_cr("\" ...");
1478 
1479       if (os::fork_and_exec(cmd) < 0) {
1480         out.print_cr("os::fork_and_exec failed: %s (%s=%d)",
1481                      os::strerror(errno), os::errno_name(errno), errno);
1482       }
1483     }
1484 
1485     // done with OnError
1486     OnError = NULL;
1487   }
1488 
1489   if (!UseOSErrorReporting) {
1490     // os::abort() will call abort hooks, try it first.
1491     static bool skip_os_abort = false;
1492     if (!skip_os_abort) {
1493       skip_os_abort = true;
1494       bool dump_core = should_report_bug(_id);
1495       os::abort(dump_core && CreateCoredumpOnCrash, _siginfo, _context);
1496     }
1497 
1498     // if os::abort() doesn't abort, try os::die();
1499     os::die();
1500   }
1501 }
1502 
1503 /*
1504  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
1505  * ensures utilities such as jmap can observe the process is a consistent state.
1506  */
1507 class VM_ReportJavaOutOfMemory : public VM_Operation {
1508  private:
1509   const char* _message;
1510  public:
1511   VM_ReportJavaOutOfMemory(const char* message) { _message = message; }
1512   VMOp_Type type() const                        { return VMOp_ReportJavaOutOfMemory; }
1513   void doit();
1514 };
1515 
1516 void VM_ReportJavaOutOfMemory::doit() {
1517   // Don't allocate large buffer on stack
1518   static char buffer[O_BUFLEN];
1519 
1520   tty->print_cr("#");
1521   tty->print_cr("# java.lang.OutOfMemoryError: %s", _message);
1522   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
1523 
1524   // make heap parsability
1525   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1526 
1527   char* cmd;
1528   const char* ptr = OnOutOfMemoryError;
1529   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1530     tty->print("#   Executing ");
1531 #if defined(LINUX)
1532     tty->print  ("/bin/sh -c ");
1533 #elif defined(SOLARIS)
1534     tty->print  ("/usr/bin/sh -c ");
1535 #endif
1536     tty->print_cr("\"%s\"...", cmd);
1537 
1538     if (os::fork_and_exec(cmd) < 0) {
1539       tty->print_cr("os::fork_and_exec failed: %s (%s=%d)",
1540                      os::strerror(errno), os::errno_name(errno), errno);
1541     }
1542   }
1543 }
1544 
1545 void VMError::report_java_out_of_memory(const char* message) {
1546   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
1547     MutexLocker ml(Heap_lock);
1548     VM_ReportJavaOutOfMemory op(message);
1549     VMThread::execute(&op);
1550   }
1551 }
1552 
1553 void VMError::show_message_box(char *buf, int buflen) {
1554   bool yes;
1555   do {
1556     error_string(buf, buflen);
1557     yes = os::start_debugging(buf,buflen);
1558   } while (yes);
1559 }
1560 
1561 // Timeout handling: check if a timeout happened (either a single step did
1562 // timeout or the whole of error reporting hit ErrorLogTimeout). Interrupt
1563 // the reporting thread if that is the case.
1564 bool VMError::check_timeout() {
1565 
1566   if (ErrorLogTimeout == 0) {
1567     return false;
1568   }
1569 
1570   // Do not check for timeouts if we still have a message box to show to the
1571   // user or if there are OnError handlers to be run.
1572   if (ShowMessageBoxOnError
1573       || (OnError != NULL && OnError[0] != '\0')
1574       || Arguments::abort_hook() != NULL) {
1575     return false;
1576   }
1577 
1578   const jlong reporting_start_time_l = get_reporting_start_time();
1579   const jlong now = get_current_timestamp();
1580   // Timestamp is stored in nanos.
1581   if (reporting_start_time_l > 0) {
1582     const jlong end = reporting_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR;
1583     if (end <= now) {
1584       _reporting_did_timeout = true;
1585       interrupt_reporting_thread();
1586       return true; // global timeout
1587     }
1588   }
1589 
1590   const jlong step_start_time_l = get_step_start_time();
1591   if (step_start_time_l > 0) {
1592     // A step times out after a quarter of the total timeout. Steps are mostly fast unless they
1593     // hang for some reason, so this simple rule allows for three hanging step and still
1594     // hopefully leaves time enough for the rest of the steps to finish.
1595     const jlong end = step_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR / 4;
1596     if (end <= now) {
1597       _step_did_timeout = true;
1598       interrupt_reporting_thread();
1599       return false; // (Not a global timeout)
1600     }
1601   }
1602 
1603   return false;
1604 
1605 }
1606 
1607 #ifndef PRODUCT
1608 typedef void (*voidfun_t)();
1609 // Crash with an authentic sigfpe
1610 static void crash_with_sigfpe() {
1611   // generate a native synchronous SIGFPE where possible;
1612   // if that did not cause a signal (e.g. on ppc), just
1613   // raise the signal.
1614   volatile int x = 0;
1615   volatile int y = 1/x;
1616 #ifndef _WIN32
1617   // OSX implements raise(sig) incorrectly so we need to
1618   // explicitly target the current thread
1619   pthread_kill(pthread_self(), SIGFPE);
1620 #endif
1621 } // end: crash_with_sigfpe
1622 
1623 // crash with sigsegv at non-null address.
1624 static void crash_with_segfault() {
1625 
1626   char* const crash_addr = (char*) VMError::get_segfault_address();
1627   *crash_addr = 'X';
1628 
1629 } // end: crash_with_segfault
1630 
1631 void VMError::test_error_handler() {
1632   controlled_crash(ErrorHandlerTest);
1633 }
1634 
1635 // crash in a controlled way:
1636 // how can be one of:
1637 // 1,2 - asserts
1638 // 3,4 - guarantee
1639 // 5-7 - fatal
1640 // 8 - vm_exit_out_of_memory
1641 // 9 - ShouldNotCallThis
1642 // 10 - ShouldNotReachHere
1643 // 11 - Unimplemented
1644 // 12,13 - (not guaranteed) crashes
1645 // 14 - SIGSEGV
1646 // 15 - SIGFPE
1647 void VMError::controlled_crash(int how) {
1648   if (how == 0) return;
1649 
1650   // If asserts are disabled, use the corresponding guarantee instead.
1651   NOT_DEBUG(if (how <= 2) how += 2);
1652 
1653   const char* const str = "hello";
1654   const size_t      num = (size_t)os::vm_page_size();
1655 
1656   const char* const eol = os::line_separator();
1657   const char* const msg = "this message should be truncated during formatting";
1658   char * const dataPtr = NULL;  // bad data pointer
1659   const void (*funcPtr)(void) = (const void(*)()) 0xF;  // bad function pointer
1660 
1661   // Keep this in sync with test/runtime/ErrorHandling/ErrorHandler.java
1662   switch (how) {
1663     case  1: vmassert(str == NULL, "expected null");
1664     case  2: vmassert(num == 1023 && *str == 'X',
1665                       "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1666     case  3: guarantee(str == NULL, "expected null");
1667     case  4: guarantee(num == 1023 && *str == 'X',
1668                        "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1669     case  5: fatal("expected null");
1670     case  6: fatal("num=" SIZE_FORMAT " str=\"%s\"", num, str);
1671     case  7: fatal("%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1672                    "%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1673                    "%s%s#    %s%s#    %s%s#    %s%s#    %s",
1674                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1675                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1676                    msg, eol, msg, eol, msg, eol, msg, eol, msg);
1677     case  8: vm_exit_out_of_memory(num, OOM_MALLOC_ERROR, "ChunkPool::allocate");
1678     case  9: ShouldNotCallThis();
1679     case 10: ShouldNotReachHere();
1680     case 11: Unimplemented();
1681     // There's no guarantee the bad data pointer will crash us
1682     // so "break" out to the ShouldNotReachHere().
1683     case 12: *dataPtr = '\0'; break;
1684     // There's no guarantee the bad function pointer will crash us
1685     // so "break" out to the ShouldNotReachHere().
1686     case 13: (*funcPtr)(); break;
1687     case 14: crash_with_segfault(); break;
1688     case 15: crash_with_sigfpe(); break;
1689 
1690     default: tty->print_cr("ERROR: %d: unexpected test_num value.", how);
1691   }
1692   ShouldNotReachHere();
1693 }
1694 #endif // !PRODUCT
1695