1 /*
   2  * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "code/codeCache.hpp"
  28 #include "compiler/compileBroker.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "logging/logConfiguration.hpp"
  32 #include "prims/whitebox.hpp"
  33 #include "runtime/arguments.hpp"
  34 #include "runtime/atomic.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/init.hpp"
  37 #include "runtime/os.hpp"
  38 #include "runtime/thread.inline.hpp"
  39 #include "runtime/threadSMR.hpp"
  40 #include "runtime/vmThread.hpp"
  41 #include "runtime/vm_operations.hpp"
  42 #include "runtime/vm_version.hpp"
  43 #include "services/memTracker.hpp"
  44 #include "trace/traceMacros.hpp"
  45 #include "utilities/debug.hpp"
  46 #include "utilities/decoder.hpp"
  47 #include "utilities/defaultStream.hpp"
  48 #include "utilities/errorReporter.hpp"
  49 #include "utilities/events.hpp"
  50 #include "utilities/vmError.hpp"
  51 
  52 #ifndef PRODUCT
  53 #include <signal.h>
  54 #endif // PRODUCT
  55 
  56 bool VMError::_error_reported = false;
  57 
  58 // call this when the VM is dying--it might loosen some asserts
  59 bool VMError::is_error_reported() { return _error_reported; }
  60 
  61 // returns an address which is guaranteed to generate a SIGSEGV on read,
  62 // for test purposes, which is not NULL and contains bits in every word
  63 void* VMError::get_segfault_address() {
  64   return (void*)
  65 #ifdef _LP64
  66     0xABC0000000000ABCULL;
  67 #else
  68     0x00000ABC;
  69 #endif
  70 }
  71 
  72 // List of environment variables that should be reported in error log file.
  73 const char *env_list[] = {
  74   // All platforms
  75   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  76   "JAVA_COMPILER", "PATH", "USERNAME",
  77 
  78   // Env variables that are defined on Solaris/Linux/BSD
  79   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  80   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  81 
  82   // defined on Linux
  83   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  84 
  85   // defined on Darwin
  86   "DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH",
  87   "DYLD_FRAMEWORK_PATH", "DYLD_FALLBACK_FRAMEWORK_PATH",
  88   "DYLD_INSERT_LIBRARIES",
  89 
  90   // defined on Windows
  91   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  92 
  93   (const char *)0
  94 };
  95 
  96 // A simple parser for -XX:OnError, usage:
  97 //  ptr = OnError;
  98 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
  99 //     ... ...
 100 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
 101   if (ptr == NULL || *ptr == NULL) return NULL;
 102 
 103   const char* cmd = *ptr;
 104 
 105   // skip leading blanks or ';'
 106   while (*cmd == ' ' || *cmd == ';') cmd++;
 107 
 108   if (*cmd == '\0') return NULL;
 109 
 110   const char * cmdend = cmd;
 111   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
 112 
 113   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
 114 
 115   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
 116   return buf;
 117 }
 118 
 119 static void print_bug_submit_message(outputStream *out, Thread *thread) {
 120   if (out == NULL) return;
 121   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 122   out->print_raw   ("#   ");
 123   out->print_raw_cr(Arguments::java_vendor_url_bug());
 124   // If the crash is in native code, encourage user to submit a bug to the
 125   // provider of that code.
 126   if (thread && thread->is_Java_thread() &&
 127       !thread->is_hidden_from_external_view()) {
 128     JavaThread* jt = (JavaThread*)thread;
 129     if (jt->thread_state() == _thread_in_native) {
 130       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 131     }
 132   }
 133   out->print_raw_cr("#");
 134 }
 135 
 136 bool VMError::coredump_status;
 137 char VMError::coredump_message[O_BUFLEN];
 138 
 139 void VMError::record_coredump_status(const char* message, bool status) {
 140   coredump_status = status;
 141   strncpy(coredump_message, message, sizeof(coredump_message));
 142   coredump_message[sizeof(coredump_message)-1] = 0;
 143 }
 144 
 145 // Return a string to describe the error
 146 char* VMError::error_string(char* buf, int buflen) {
 147   char signame_buf[64];
 148   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 149 
 150   if (signame) {
 151     jio_snprintf(buf, buflen,
 152                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 153                  signame, _id, _pc,
 154                  os::current_process_id(), os::current_thread_id());
 155   } else if (_filename != NULL && _lineno > 0) {
 156     // skip directory names
 157     char separator = os::file_separator()[0];
 158     const char *p = strrchr(_filename, separator);
 159     int n = jio_snprintf(buf, buflen,
 160                          "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT,
 161                          p ? p + 1 : _filename, _lineno,
 162                          os::current_process_id(), os::current_thread_id());
 163     if (n >= 0 && n < buflen && _message) {
 164       if (strlen(_detail_msg) > 0) {
 165         jio_snprintf(buf + n, buflen - n, "%s%s: %s",
 166         os::line_separator(), _message, _detail_msg);
 167       } else {
 168         jio_snprintf(buf + n, buflen - n, "%sError: %s",
 169                      os::line_separator(), _message);
 170       }
 171     }
 172   } else {
 173     jio_snprintf(buf, buflen,
 174                  "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 175                  _id, os::current_process_id(), os::current_thread_id());
 176   }
 177 
 178   return buf;
 179 }
 180 
 181 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 182                                 char* buf, int buflen, bool verbose) {
 183 #ifdef ZERO
 184   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 185     // StackFrameStream uses the frame anchor, which may not have
 186     // been set up.  This can be done at any time in Zero, however,
 187     // so if it hasn't been set up then we just set it up now and
 188     // clear it again when we're done.
 189     bool has_last_Java_frame = jt->has_last_Java_frame();
 190     if (!has_last_Java_frame)
 191       jt->set_last_Java_frame();
 192     st->print("Java frames:");
 193     st->cr();
 194 
 195     // Print the frames
 196     StackFrameStream sfs(jt);
 197     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 198       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 199       st->cr();
 200     }
 201 
 202     // Reset the frame anchor if necessary
 203     if (!has_last_Java_frame)
 204       jt->reset_last_Java_frame();
 205   }
 206 #else
 207   if (jt->has_last_Java_frame()) {
 208     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 209     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 210       sfs.current()->print_on_error(st, buf, buflen, verbose);
 211       st->cr();
 212     }
 213   }
 214 #endif // ZERO
 215 }
 216 
 217 void VMError::print_native_stack(outputStream* st, frame fr, Thread* t, char* buf, int buf_size) {
 218 
 219   // see if it's a valid frame
 220   if (fr.pc()) {
 221     st->print_cr("Native frames: (J=compiled Java code, A=aot compiled Java code, j=interpreted, Vv=VM code, C=native code)");
 222 
 223     int count = 0;
 224     while (count++ < StackPrintLimit) {
 225       fr.print_on_error(st, buf, buf_size);
 226       if (fr.pc()) { // print source file and line, if available
 227         char buf[128];
 228         int line_no;
 229         if (Decoder::get_source_info(fr.pc(), buf, sizeof(buf), &line_no)) {
 230           st->print("  (%s:%d)", buf, line_no);
 231         }
 232       }
 233       st->cr();
 234       // Compiled code may use EBP register on x86 so it looks like
 235       // non-walkable C frame. Use frame.sender() for java frames.
 236       if (t && t->is_Java_thread()) {
 237         // Catch very first native frame by using stack address.
 238         // For JavaThread stack_base and stack_size should be set.
 239         if (!t->on_local_stack((address)(fr.real_fp() + 1))) {
 240           break;
 241         }
 242         if (fr.is_java_frame() || fr.is_native_frame() || fr.is_runtime_frame()) {
 243           RegisterMap map((JavaThread*)t, false); // No update
 244           fr = fr.sender(&map);
 245         } else {
 246           // is_first_C_frame() does only simple checks for frame pointer,
 247           // it will pass if java compiled code has a pointer in EBP.
 248           if (os::is_first_C_frame(&fr)) break;
 249           fr = os::get_sender_for_C_frame(&fr);
 250         }
 251       } else {
 252         if (os::is_first_C_frame(&fr)) break;
 253         fr = os::get_sender_for_C_frame(&fr);
 254       }
 255     }
 256 
 257     if (count > StackPrintLimit) {
 258       st->print_cr("...<more frames>...");
 259     }
 260 
 261     st->cr();
 262   }
 263 }
 264 
 265 static void print_oom_reasons(outputStream* st) {
 266   st->print_cr("# Possible reasons:");
 267   st->print_cr("#   The system is out of physical RAM or swap space");
 268   if (UseCompressedOops) {
 269     st->print_cr("#   The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap");
 270   }
 271   if (LogBytesPerWord == 2) {
 272     st->print_cr("#   In 32 bit mode, the process size limit was hit");
 273   }
 274   st->print_cr("# Possible solutions:");
 275   st->print_cr("#   Reduce memory load on the system");
 276   st->print_cr("#   Increase physical memory or swap space");
 277   st->print_cr("#   Check if swap backing store is full");
 278   if (LogBytesPerWord == 2) {
 279     st->print_cr("#   Use 64 bit Java on a 64 bit OS");
 280   }
 281   st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
 282   st->print_cr("#   Decrease number of Java threads");
 283   st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
 284   st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
 285   if (UseCompressedOops) {
 286     switch (Universe::narrow_oop_mode()) {
 287       case Universe::UnscaledNarrowOop:
 288         st->print_cr("#   JVM is running with Unscaled Compressed Oops mode in which the Java heap is");
 289         st->print_cr("#     placed in the first 4GB address space. The Java Heap base address is the");
 290         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 291         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 4GB virtual address.");
 292         break;
 293       case Universe::ZeroBasedNarrowOop:
 294         st->print_cr("#   JVM is running with Zero Based Compressed Oops mode in which the Java heap is");
 295         st->print_cr("#     placed in the first 32GB address space. The Java Heap base address is the");
 296         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 297         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 32GB virtual address.");
 298         break;
 299       default:
 300         break;
 301     }
 302   }
 303   st->print_cr("# This output file may be truncated or incomplete.");
 304 }
 305 
 306 static const char* gc_mode() {
 307   if (UseG1GC)            return "g1 gc";
 308   if (UseParallelGC)      return "parallel gc";
 309   if (UseConcMarkSweepGC) return "concurrent mark sweep gc";
 310   if (UseSerialGC)        return "serial gc";
 311   return "ERROR in GC mode";
 312 }
 313 
 314 static void report_vm_version(outputStream* st, char* buf, int buflen) {
 315    // VM version
 316    st->print_cr("#");
 317    JDK_Version::current().to_string(buf, buflen);
 318    const char* runtime_name = JDK_Version::runtime_name() != NULL ?
 319                                 JDK_Version::runtime_name() : "";
 320    const char* runtime_version = JDK_Version::runtime_version() != NULL ?
 321                                    JDK_Version::runtime_version() : "";
 322    const char* jdk_debug_level = Abstract_VM_Version::printable_jdk_debug_level() != NULL ?
 323                                    Abstract_VM_Version::printable_jdk_debug_level() : "";
 324 
 325    st->print_cr("# JRE version: %s (%s) (%sbuild %s)", runtime_name, buf,
 326                  jdk_debug_level, runtime_version);
 327 
 328    // This is the long version with some default settings added
 329    st->print_cr("# Java VM: %s (%s%s, %s%s%s%s%s, %s, %s)",
 330                  Abstract_VM_Version::vm_name(),
 331                  jdk_debug_level,
 332                  Abstract_VM_Version::vm_release(),
 333                  Abstract_VM_Version::vm_info_string(),
 334                  TieredCompilation ? ", tiered" : "",
 335 #if INCLUDE_JVMCI
 336                  EnableJVMCI ? ", jvmci" : "",
 337                  UseJVMCICompiler ? ", jvmci compiler" : "",
 338 #else
 339                  "", "",
 340 #endif
 341                  UseCompressedOops ? ", compressed oops" : "",
 342                  gc_mode(),
 343                  Abstract_VM_Version::vm_platform_string()
 344                );
 345 }
 346 
 347 // This is the main function to report a fatal error. Only one thread can
 348 // call this function, so we don't need to worry about MT-safety. But it's
 349 // possible that the error handler itself may crash or die on an internal
 350 // error, for example, when the stack/heap is badly damaged. We must be
 351 // able to handle recursive errors that happen inside error handler.
 352 //
 353 // Error reporting is done in several steps. If a crash or internal error
 354 // occurred when reporting an error, the nested signal/exception handler
 355 // can skip steps that are already (or partially) done. Error reporting will
 356 // continue from the next step. This allows us to retrieve and print
 357 // information that may be unsafe to get after a fatal error. If it happens,
 358 // you may find nested report_and_die() frames when you look at the stack
 359 // in a debugger.
 360 //
 361 // In general, a hang in error handler is much worse than a crash or internal
 362 // error, as it's harder to recover from a hang. Deadlock can happen if we
 363 // try to grab a lock that is already owned by current thread, or if the
 364 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 365 // error handler and all the functions it called should avoid grabbing any
 366 // lock. An important thing to notice is that memory allocation needs a lock.
 367 //
 368 // We should avoid using large stack allocated buffers. Many errors happen
 369 // when stack space is already low. Making things even worse is that there
 370 // could be nested report_and_die() calls on stack (see above). Only one
 371 // thread can report error, so large buffers are statically allocated in data
 372 // segment.
 373 
 374 int          VMError::_current_step;
 375 const char*  VMError::_current_step_info;
 376 
 377 volatile jlong VMError::_reporting_start_time = -1;
 378 volatile bool VMError::_reporting_did_timeout = false;
 379 volatile jlong VMError::_step_start_time = -1;
 380 volatile bool VMError::_step_did_timeout = false;
 381 
 382 // Helper, return current timestamp for timeout handling.
 383 jlong VMError::get_current_timestamp() {
 384   return os::javaTimeNanos();
 385 }
 386 // Factor to translate the timestamp to seconds.
 387 #define TIMESTAMP_TO_SECONDS_FACTOR (1000 * 1000 * 1000)
 388 
 389 void VMError::record_reporting_start_time() {
 390   const jlong now = get_current_timestamp();
 391   Atomic::store(now, &_reporting_start_time);
 392 }
 393 
 394 jlong VMError::get_reporting_start_time() {
 395   return Atomic::load(&_reporting_start_time);
 396 }
 397 
 398 void VMError::record_step_start_time() {
 399   const jlong now = get_current_timestamp();
 400   Atomic::store(now, &_step_start_time);
 401 }
 402 
 403 jlong VMError::get_step_start_time() {
 404   return Atomic::load(&_step_start_time);
 405 }
 406 
 407 void VMError::report(outputStream* st, bool _verbose) {
 408 
 409 # define BEGIN if (_current_step == 0) { _current_step = __LINE__;
 410 # define STEP(s) } if (_current_step < __LINE__) { _current_step = __LINE__; _current_step_info = s; \
 411   record_step_start_time(); _step_did_timeout = false;
 412 # define END }
 413 
 414   // don't allocate large buffer on stack
 415   static char buf[O_BUFLEN];
 416 
 417   BEGIN
 418 
 419   STEP("printing fatal error message")
 420 
 421     st->print_cr("#");
 422     if (should_report_bug(_id)) {
 423       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 424     } else {
 425       st->print_cr("# There is insufficient memory for the Java "
 426                    "Runtime Environment to continue.");
 427     }
 428 
 429 #ifndef PRODUCT
 430   // Error handler self tests
 431 
 432   // test secondary error handling. Test it twice, to test that resetting
 433   // error handler after a secondary crash works.
 434   STEP("test secondary crash 1")
 435     if (_verbose && TestCrashInErrorHandler != 0) {
 436       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 437         TestCrashInErrorHandler);
 438       controlled_crash(TestCrashInErrorHandler);
 439     }
 440 
 441   STEP("test secondary crash 2")
 442     if (_verbose && TestCrashInErrorHandler != 0) {
 443       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 444         TestCrashInErrorHandler);
 445       controlled_crash(TestCrashInErrorHandler);
 446     }
 447 
 448   // TestUnresponsiveErrorHandler: We want to test both step timeouts and global timeout.
 449   // Step to global timeout ratio is 4:1, so in order to be absolutely sure we hit the
 450   // global timeout, let's execute the timeout step five times.
 451   // See corresponding test in test/runtime/ErrorHandling/TimeoutInErrorHandlingTest.java
 452   #define TIMEOUT_TEST_STEP STEP("test unresponsive error reporting step") \
 453     if (_verbose && TestUnresponsiveErrorHandler) { os::infinite_sleep(); }
 454   TIMEOUT_TEST_STEP
 455   TIMEOUT_TEST_STEP
 456   TIMEOUT_TEST_STEP
 457   TIMEOUT_TEST_STEP
 458   TIMEOUT_TEST_STEP
 459 
 460   STEP("test safefetch in error handler")
 461     // test whether it is safe to use SafeFetch32 in Crash Handler. Test twice
 462     // to test that resetting the signal handler works correctly.
 463     if (_verbose && TestSafeFetchInErrorHandler) {
 464       st->print_cr("Will test SafeFetch...");
 465       if (CanUseSafeFetch32()) {
 466         int* const invalid_pointer = (int*) get_segfault_address();
 467         const int x = 0x76543210;
 468         int i1 = SafeFetch32(invalid_pointer, x);
 469         int i2 = SafeFetch32(invalid_pointer, x);
 470         if (i1 == x && i2 == x) {
 471           st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern
 472         } else {
 473           st->print_cr("??");
 474         }
 475       } else {
 476         st->print_cr("not possible; skipped.");
 477       }
 478     }
 479 #endif // PRODUCT
 480 
 481   STEP("printing type of error")
 482 
 483      switch(_id) {
 484        case OOM_MALLOC_ERROR:
 485        case OOM_MMAP_ERROR:
 486          if (_size) {
 487            st->print("# Native memory allocation ");
 488            st->print((_id == (int)OOM_MALLOC_ERROR) ? "(malloc) failed to allocate " :
 489                                                  "(mmap) failed to map ");
 490            jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
 491            st->print("%s", buf);
 492            st->print(" bytes");
 493            if (strlen(_detail_msg) > 0) {
 494              st->print(" for ");
 495              st->print("%s", _detail_msg);
 496            }
 497            st->cr();
 498          } else {
 499            if (strlen(_detail_msg) > 0) {
 500              st->print("# ");
 501              st->print_cr("%s", _detail_msg);
 502            }
 503          }
 504          // In error file give some solutions
 505          if (_verbose) {
 506            print_oom_reasons(st);
 507          } else {
 508            return;  // that's enough for the screen
 509          }
 510          break;
 511        case INTERNAL_ERROR:
 512        default:
 513          break;
 514      }
 515 
 516   STEP("printing exception/signal name")
 517 
 518      st->print_cr("#");
 519      st->print("#  ");
 520      // Is it an OS exception/signal?
 521      if (os::exception_name(_id, buf, sizeof(buf))) {
 522        st->print("%s", buf);
 523        st->print(" (0x%x)", _id);                // signal number
 524        st->print(" at pc=" PTR_FORMAT, p2i(_pc));
 525      } else {
 526        if (should_report_bug(_id)) {
 527          st->print("Internal Error");
 528        } else {
 529          st->print("Out of Memory Error");
 530        }
 531        if (_filename != NULL && _lineno > 0) {
 532 #ifdef PRODUCT
 533          // In product mode chop off pathname?
 534          char separator = os::file_separator()[0];
 535          const char *p = strrchr(_filename, separator);
 536          const char *file = p ? p+1 : _filename;
 537 #else
 538          const char *file = _filename;
 539 #endif
 540          st->print(" (%s:%d)", file, _lineno);
 541        } else {
 542          st->print(" (0x%x)", _id);
 543        }
 544      }
 545 
 546   STEP("printing current thread and pid")
 547 
 548      // process id, thread id
 549      st->print(", pid=%d", os::current_process_id());
 550      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 551      st->cr();
 552 
 553   STEP("printing error message")
 554 
 555      if (should_report_bug(_id)) {  // already printed the message.
 556        // error message
 557        if (strlen(_detail_msg) > 0) {
 558          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
 559        } else if (_message) {
 560          st->print_cr("#  Error: %s", _message);
 561        }
 562      }
 563 
 564   STEP("printing Java version string")
 565 
 566      report_vm_version(st, buf, sizeof(buf));
 567 
 568   STEP("printing problematic frame")
 569 
 570      // Print current frame if we have a context (i.e. it's a crash)
 571      if (_context) {
 572        st->print_cr("# Problematic frame:");
 573        st->print("# ");
 574        frame fr = os::fetch_frame_from_context(_context);
 575        fr.print_on_error(st, buf, sizeof(buf));
 576        st->cr();
 577        st->print_cr("#");
 578      }
 579 
 580   STEP("printing core file information")
 581     st->print("# ");
 582     if (CreateCoredumpOnCrash) {
 583       if (coredump_status) {
 584         st->print("Core dump will be written. Default location: %s", coredump_message);
 585       } else {
 586         st->print("No core dump will be written. %s", coredump_message);
 587       }
 588     } else {
 589       st->print("CreateCoredumpOnCrash turned off, no core file dumped");
 590     }
 591     st->cr();
 592     st->print_cr("#");
 593 
 594   STEP("printing bug submit message")
 595 
 596      if (should_report_bug(_id) && _verbose) {
 597        print_bug_submit_message(st, _thread);
 598      }
 599 
 600   STEP("printing summary")
 601 
 602      if (_verbose) {
 603        st->cr();
 604        st->print_cr("---------------  S U M M A R Y ------------");
 605        st->cr();
 606      }
 607 
 608   STEP("printing VM option summary")
 609 
 610      if (_verbose) {
 611        // VM options
 612        Arguments::print_summary_on(st);
 613        st->cr();
 614      }
 615 
 616   STEP("printing summary machine and OS info")
 617 
 618      if (_verbose) {
 619        os::print_summary_info(st, buf, sizeof(buf));
 620      }
 621 
 622 
 623   STEP("printing date and time")
 624 
 625      if (_verbose) {
 626        os::print_date_and_time(st, buf, sizeof(buf));
 627      }
 628 
 629   STEP("printing thread")
 630 
 631      if (_verbose) {
 632        st->cr();
 633        st->print_cr("---------------  T H R E A D  ---------------");
 634        st->cr();
 635      }
 636 
 637   STEP("printing current thread")
 638 
 639      // current thread
 640      if (_verbose) {
 641        if (_thread) {
 642          st->print("Current thread (" PTR_FORMAT "):  ", p2i(_thread));
 643          _thread->print_on_error(st, buf, sizeof(buf));
 644          st->cr();
 645        } else {
 646          st->print_cr("Current thread is native thread");
 647        }
 648        st->cr();
 649      }
 650 
 651   STEP("printing current compile task")
 652 
 653      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 654         CompilerThread* t = (CompilerThread*)_thread;
 655         if (t->task()) {
 656            st->cr();
 657            st->print_cr("Current CompileTask:");
 658            t->task()->print_line_on_error(st, buf, sizeof(buf));
 659            st->cr();
 660         }
 661      }
 662 
 663 
 664   STEP("printing stack bounds")
 665 
 666      if (_verbose) {
 667        st->print("Stack: ");
 668 
 669        address stack_top;
 670        size_t stack_size;
 671 
 672        if (_thread) {
 673           stack_top = _thread->stack_base();
 674           stack_size = _thread->stack_size();
 675        } else {
 676           stack_top = os::current_stack_base();
 677           stack_size = os::current_stack_size();
 678        }
 679 
 680        address stack_bottom = stack_top - stack_size;
 681        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", p2i(stack_bottom), p2i(stack_top));
 682 
 683        frame fr = _context ? os::fetch_frame_from_context(_context)
 684                            : os::current_frame();
 685 
 686        if (fr.sp()) {
 687          st->print(",  sp=" PTR_FORMAT, p2i(fr.sp()));
 688          size_t free_stack_size = pointer_delta(fr.sp(), stack_bottom, 1024);
 689          st->print(",  free space=" SIZE_FORMAT "k", free_stack_size);
 690        }
 691 
 692        st->cr();
 693      }
 694 
 695   STEP("printing native stack")
 696 
 697    if (_verbose) {
 698      if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) {
 699        // We have printed the native stack in platform-specific code
 700        // Windows/x64 needs special handling.
 701      } else {
 702        frame fr = _context ? os::fetch_frame_from_context(_context)
 703                            : os::current_frame();
 704 
 705        print_native_stack(st, fr, _thread, buf, sizeof(buf));
 706      }
 707    }
 708 
 709   STEP("printing Java stack")
 710 
 711      if (_verbose && _thread && _thread->is_Java_thread()) {
 712        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 713      }
 714 
 715   STEP("printing target Java thread stack")
 716 
 717      // printing Java thread stack trace if it is involved in GC crash
 718      if (_verbose && _thread && (_thread->is_Named_thread())) {
 719        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 720        if (jt != NULL) {
 721          st->print_cr("JavaThread " PTR_FORMAT " (nid = %d) was being processed", p2i(jt), jt->osthread()->thread_id());
 722          print_stack_trace(st, jt, buf, sizeof(buf), true);
 723        }
 724      }
 725 
 726   STEP("printing siginfo")
 727 
 728      // signal no, signal code, address that caused the fault
 729      if (_verbose && _siginfo) {
 730        st->cr();
 731        os::print_siginfo(st, _siginfo);
 732        st->cr();
 733      }
 734 
 735   STEP("CDS archive access warning")
 736 
 737      // Print an explicit hint if we crashed on access to the CDS archive.
 738      if (_verbose && _siginfo) {
 739        check_failing_cds_access(st, _siginfo);
 740        st->cr();
 741      }
 742 
 743   STEP("printing register info")
 744 
 745      // decode register contents if possible
 746      if (_verbose && _context && Universe::is_fully_initialized()) {
 747        os::print_register_info(st, _context);
 748        st->cr();
 749      }
 750 
 751   STEP("printing registers, top of stack, instructions near pc")
 752 
 753      // registers, top of stack, instructions near pc
 754      if (_verbose && _context) {
 755        os::print_context(st, _context);
 756        st->cr();
 757      }
 758 
 759   STEP("printing code blob if possible")
 760 
 761      if (_verbose && _context) {
 762        CodeBlob* cb = CodeCache::find_blob(_pc);
 763        if (cb != NULL) {
 764          if (Interpreter::contains(_pc)) {
 765            // The interpreter CodeBlob is very large so try to print the codelet instead.
 766            InterpreterCodelet* codelet = Interpreter::codelet_containing(_pc);
 767            if (codelet != NULL) {
 768              codelet->print_on(st);
 769              Disassembler::decode(codelet->code_begin(), codelet->code_end(), st);
 770            }
 771          } else {
 772            StubCodeDesc* desc = StubCodeDesc::desc_for(_pc);
 773            if (desc != NULL) {
 774              desc->print_on(st);
 775              Disassembler::decode(desc->begin(), desc->end(), st);
 776            } else {
 777              Disassembler::decode(cb, st);
 778              st->cr();
 779            }
 780          }
 781        }
 782      }
 783 
 784   STEP("printing VM operation")
 785 
 786      if (_verbose && _thread && _thread->is_VM_thread()) {
 787         VMThread* t = (VMThread*)_thread;
 788         VM_Operation* op = t->vm_operation();
 789         if (op) {
 790           op->print_on_error(st);
 791           st->cr();
 792           st->cr();
 793         }
 794      }
 795 
 796   STEP("printing process")
 797 
 798      if (_verbose) {
 799        st->cr();
 800        st->print_cr("---------------  P R O C E S S  ---------------");
 801        st->cr();
 802      }
 803 
 804   STEP("printing all threads")
 805 
 806      // all threads
 807      if (_verbose && _thread) {
 808        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 809        st->cr();
 810      }
 811 
 812   STEP("printing VM state")
 813 
 814      if (_verbose) {
 815        // Safepoint state
 816        st->print("VM state:");
 817 
 818        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 819        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 820        else st->print("not at safepoint");
 821 
 822        // Also see if error occurred during initialization or shutdown
 823        if (!Universe::is_fully_initialized()) {
 824          st->print(" (not fully initialized)");
 825        } else if (VM_Exit::vm_exited()) {
 826          st->print(" (shutting down)");
 827        } else {
 828          st->print(" (normal execution)");
 829        }
 830        st->cr();
 831        st->cr();
 832      }
 833 
 834   STEP("printing owned locks on error")
 835 
 836      // mutexes/monitors that currently have an owner
 837      if (_verbose) {
 838        print_owned_locks_on_error(st);
 839        st->cr();
 840      }
 841 
 842   STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
 843 
 844      if (_verbose && Exceptions::has_exception_counts()) {
 845        st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 846        Exceptions::print_exception_counts_on_error(st);
 847        st->cr();
 848      }
 849 
 850   STEP("printing compressed oops mode")
 851 
 852      if (_verbose && UseCompressedOops) {
 853        Universe::print_compressed_oops_mode(st);
 854        if (UseCompressedClassPointers) {
 855          Metaspace::print_compressed_class_space(st);
 856        }
 857        st->cr();
 858      }
 859 
 860   STEP("printing heap information")
 861 
 862      if (_verbose && Universe::is_fully_initialized()) {
 863        Universe::heap()->print_on_error(st);
 864        st->cr();
 865        st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
 866        st->cr();
 867      }
 868 
 869   STEP("printing code cache information")
 870 
 871      if (_verbose && Universe::is_fully_initialized()) {
 872        // print code cache information before vm abort
 873        CodeCache::print_summary(st);
 874        st->cr();
 875      }
 876 
 877   STEP("printing ring buffers")
 878 
 879      if (_verbose) {
 880        Events::print_all(st);
 881        st->cr();
 882      }
 883 
 884   STEP("printing dynamic libraries")
 885 
 886      if (_verbose) {
 887        // dynamic libraries, or memory map
 888        os::print_dll_info(st);
 889        st->cr();
 890      }
 891 
 892   STEP("printing native decoder state")
 893 
 894      if (_verbose) {
 895        Decoder::print_state_on(st);
 896        st->cr();
 897      }
 898 
 899   STEP("printing VM options")
 900 
 901      if (_verbose) {
 902        // VM options
 903        Arguments::print_on(st);
 904        st->cr();
 905      }
 906 
 907   STEP("printing warning if internal testing API used")
 908 
 909      if (WhiteBox::used()) {
 910        st->print_cr("Unsupported internal testing APIs have been used.");
 911        st->cr();
 912      }
 913 
 914   STEP("printing log configuration")
 915     if (_verbose){
 916       st->print_cr("Logging:");
 917       LogConfiguration::describe_current_configuration(st);
 918       st->cr();
 919     }
 920 
 921   STEP("printing all environment variables")
 922 
 923      if (_verbose) {
 924        os::print_environment_variables(st, env_list);
 925        st->cr();
 926      }
 927 
 928   STEP("printing signal handlers")
 929 
 930      if (_verbose) {
 931        os::print_signal_handlers(st, buf, sizeof(buf));
 932        st->cr();
 933      }
 934 
 935   STEP("Native Memory Tracking")
 936      if (_verbose) {
 937        MemTracker::error_report(st);
 938      }
 939 
 940   STEP("printing system")
 941 
 942      if (_verbose) {
 943        st->cr();
 944        st->print_cr("---------------  S Y S T E M  ---------------");
 945        st->cr();
 946      }
 947 
 948   STEP("printing OS information")
 949 
 950      if (_verbose) {
 951        os::print_os_info(st);
 952        st->cr();
 953      }
 954 
 955   STEP("printing CPU info")
 956      if (_verbose) {
 957        os::print_cpu_info(st, buf, sizeof(buf));
 958        st->cr();
 959      }
 960 
 961   STEP("printing memory info")
 962 
 963      if (_verbose) {
 964        os::print_memory_info(st);
 965        st->cr();
 966      }
 967 
 968   STEP("printing internal vm info")
 969 
 970      if (_verbose) {
 971        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 972        st->cr();
 973      }
 974 
 975   // print a defined marker to show that error handling finished correctly.
 976   STEP("printing end marker")
 977 
 978      if (_verbose) {
 979        st->print_cr("END.");
 980      }
 981 
 982   END
 983 
 984 # undef BEGIN
 985 # undef STEP
 986 # undef END
 987 }
 988 
 989 // Report for the vm_info_cmd. This prints out the information above omitting
 990 // crash and thread specific information.  If output is added above, it should be added
 991 // here also, if it is safe to call during a running process.
 992 void VMError::print_vm_info(outputStream* st) {
 993 
 994   char buf[O_BUFLEN];
 995   report_vm_version(st, buf, sizeof(buf));
 996 
 997   // STEP("printing summary")
 998 
 999   st->cr();
1000   st->print_cr("---------------  S U M M A R Y ------------");
1001   st->cr();
1002 
1003   // STEP("printing VM option summary")
1004 
1005   // VM options
1006   Arguments::print_summary_on(st);
1007   st->cr();
1008 
1009   // STEP("printing summary machine and OS info")
1010 
1011   os::print_summary_info(st, buf, sizeof(buf));
1012 
1013   // STEP("printing date and time")
1014 
1015   os::print_date_and_time(st, buf, sizeof(buf));
1016 
1017   // Skip: STEP("printing thread")
1018 
1019   // STEP("printing process")
1020 
1021   st->cr();
1022   st->print_cr("---------------  P R O C E S S  ---------------");
1023   st->cr();
1024 
1025   // STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
1026 
1027   if (Exceptions::has_exception_counts()) {
1028     st->print_cr("OutOfMemory and StackOverflow Exception counts:");
1029     Exceptions::print_exception_counts_on_error(st);
1030     st->cr();
1031   }
1032 
1033   // STEP("printing compressed oops mode")
1034 
1035   if (UseCompressedOops) {
1036     Universe::print_compressed_oops_mode(st);
1037     if (UseCompressedClassPointers) {
1038       Metaspace::print_compressed_class_space(st);
1039     }
1040     st->cr();
1041   }
1042 
1043   // STEP("printing heap information")
1044 
1045   if (Universe::is_fully_initialized()) {
1046     MutexLocker hl(Heap_lock);
1047     Universe::heap()->print_on_error(st);
1048     st->cr();
1049     st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
1050     st->cr();
1051   }
1052 
1053   // STEP("printing code cache information")
1054 
1055   if (Universe::is_fully_initialized()) {
1056     // print code cache information before vm abort
1057     CodeCache::print_summary(st);
1058     st->cr();
1059   }
1060 
1061   // STEP("printing ring buffers")
1062 
1063   Events::print_all(st);
1064   st->cr();
1065 
1066   // STEP("printing dynamic libraries")
1067 
1068   // dynamic libraries, or memory map
1069   os::print_dll_info(st);
1070   st->cr();
1071 
1072   // STEP("printing VM options")
1073 
1074   // VM options
1075   Arguments::print_on(st);
1076   st->cr();
1077 
1078   // STEP("printing warning if internal testing API used")
1079 
1080   if (WhiteBox::used()) {
1081     st->print_cr("Unsupported internal testing APIs have been used.");
1082     st->cr();
1083   }
1084 
1085   // STEP("printing log configuration")
1086   st->print_cr("Logging:");
1087   LogConfiguration::describe(st);
1088   st->cr();
1089 
1090   // STEP("printing all environment variables")
1091 
1092   os::print_environment_variables(st, env_list);
1093   st->cr();
1094 
1095   // STEP("printing signal handlers")
1096 
1097   os::print_signal_handlers(st, buf, sizeof(buf));
1098   st->cr();
1099 
1100   // STEP("Native Memory Tracking")
1101 
1102   MemTracker::error_report(st);
1103 
1104   // STEP("printing system")
1105 
1106   st->cr();
1107   st->print_cr("---------------  S Y S T E M  ---------------");
1108   st->cr();
1109 
1110   // STEP("printing OS information")
1111 
1112   os::print_os_info(st);
1113   st->cr();
1114 
1115   // STEP("printing CPU info")
1116 
1117   os::print_cpu_info(st, buf, sizeof(buf));
1118   st->cr();
1119 
1120   // STEP("printing memory info")
1121 
1122   os::print_memory_info(st);
1123   st->cr();
1124 
1125   // STEP("printing internal vm info")
1126 
1127   st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
1128   st->cr();
1129 
1130   // print a defined marker to show that error handling finished correctly.
1131   // STEP("printing end marker")
1132 
1133   st->print_cr("END.");
1134 }
1135 
1136 volatile intptr_t VMError::first_error_tid = -1;
1137 
1138 // An error could happen before tty is initialized or after it has been
1139 // destroyed.
1140 // Please note: to prevent large stack allocations, the log- and
1141 // output-stream use a global scratch buffer for format printing.
1142 // (see VmError::report_and_die(). Access to those streams is synchronized
1143 // in  VmError::report_and_die() - there is only one reporting thread at
1144 // any given time.
1145 fdStream VMError::out(defaultStream::output_fd());
1146 fdStream VMError::log; // error log used by VMError::report_and_die()
1147 
1148 /** Expand a pattern into a buffer starting at pos and open a file using constructed path */
1149 static int expand_and_open(const char* pattern, char* buf, size_t buflen, size_t pos) {
1150   int fd = -1;
1151   if (Arguments::copy_expand_pid(pattern, strlen(pattern), &buf[pos], buflen - pos)) {
1152     // the O_EXCL flag will cause the open to fail if the file exists
1153     fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0666);
1154   }
1155   return fd;
1156 }
1157 
1158 /**
1159  * Construct file name for a log file and return it's file descriptor.
1160  * Name and location depends on pattern, default_pattern params and access
1161  * permissions.
1162  */
1163 static int prepare_log_file(const char* pattern, const char* default_pattern, char* buf, size_t buflen) {
1164   int fd = -1;
1165 
1166   // If possible, use specified pattern to construct log file name
1167   if (pattern != NULL) {
1168     fd = expand_and_open(pattern, buf, buflen, 0);
1169   }
1170 
1171   // Either user didn't specify, or the user's location failed,
1172   // so use the default name in the current directory
1173   if (fd == -1) {
1174     const char* cwd = os::get_current_directory(buf, buflen);
1175     if (cwd != NULL) {
1176       size_t pos = strlen(cwd);
1177       int fsep_len = jio_snprintf(&buf[pos], buflen-pos, "%s", os::file_separator());
1178       pos += fsep_len;
1179       if (fsep_len > 0) {
1180         fd = expand_and_open(default_pattern, buf, buflen, pos);
1181       }
1182     }
1183   }
1184 
1185    // try temp directory if it exists.
1186    if (fd == -1) {
1187      const char* tmpdir = os::get_temp_directory();
1188      if (tmpdir != NULL && strlen(tmpdir) > 0) {
1189        int pos = jio_snprintf(buf, buflen, "%s%s", tmpdir, os::file_separator());
1190        if (pos > 0) {
1191          fd = expand_and_open(default_pattern, buf, buflen, pos);
1192        }
1193      }
1194    }
1195 
1196   return fd;
1197 }
1198 
1199 int         VMError::_id;
1200 const char* VMError::_message;
1201 char        VMError::_detail_msg[1024];
1202 Thread*     VMError::_thread;
1203 address     VMError::_pc;
1204 void*       VMError::_siginfo;
1205 void*       VMError::_context;
1206 const char* VMError::_filename;
1207 int         VMError::_lineno;
1208 size_t      VMError::_size;
1209 
1210 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo,
1211                              void* context, const char* detail_fmt, ...)
1212 {
1213   va_list detail_args;
1214   va_start(detail_args, detail_fmt);
1215   report_and_die(sig, NULL, detail_fmt, detail_args, thread, pc, siginfo, context, NULL, 0, 0);
1216   va_end(detail_args);
1217 }
1218 
1219 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context)
1220 {
1221   report_and_die(thread, sig, pc, siginfo, context, "%s", "");
1222 }
1223 
1224 void VMError::report_and_die(const char* message, const char* detail_fmt, ...)
1225 {
1226   va_list detail_args;
1227   va_start(detail_args, detail_fmt);
1228   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, NULL, NULL, NULL, NULL, NULL, 0, 0);
1229   va_end(detail_args);
1230 }
1231 
1232 void VMError::report_and_die(const char* message)
1233 {
1234   report_and_die(message, "%s", "");
1235 }
1236 
1237 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, const char* message,
1238                              const char* detail_fmt, va_list detail_args)
1239 {
1240   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, 0);
1241 }
1242 
1243 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, size_t size,
1244                              VMErrorType vm_err_type, const char* detail_fmt, va_list detail_args) {
1245   report_and_die(vm_err_type, NULL, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, size);
1246 }
1247 
1248 void VMError::report_and_die(int id, const char* message, const char* detail_fmt, va_list detail_args,
1249                              Thread* thread, address pc, void* siginfo, void* context, const char* filename,
1250                              int lineno, size_t size)
1251 {
1252   // Don't allocate large buffer on stack
1253   static char buffer[O_BUFLEN];
1254   out.set_scratch_buffer(buffer, sizeof(buffer));
1255   log.set_scratch_buffer(buffer, sizeof(buffer));
1256 
1257   // How many errors occurred in error handler when reporting first_error.
1258   static int recursive_error_count;
1259 
1260   // We will first print a brief message to standard out (verbose = false),
1261   // then save detailed information in log file (verbose = true).
1262   static bool out_done = false;         // done printing to standard out
1263   static bool log_done = false;         // done saving error log
1264   static bool transmit_report_done = false; // done error reporting
1265 
1266   if (SuppressFatalErrorMessage) {
1267       os::abort(CreateCoredumpOnCrash);
1268   }
1269   intptr_t mytid = os::current_thread_id();
1270   if (first_error_tid == -1 &&
1271       Atomic::cmpxchg(mytid, &first_error_tid, (intptr_t)-1) == -1) {
1272 
1273     // Initialize time stamps to use the same base.
1274     out.time_stamp().update_to(1);
1275     log.time_stamp().update_to(1);
1276 
1277     _id = id;
1278     _message = message;
1279     _thread = thread;
1280     _pc = pc;
1281     _siginfo = siginfo;
1282     _context = context;
1283     _filename = filename;
1284     _lineno = lineno;
1285     _size = size;
1286     jio_vsnprintf(_detail_msg, sizeof(_detail_msg), detail_fmt, detail_args);
1287 
1288     // first time
1289     _error_reported = true;
1290 
1291     reporting_started();
1292     record_reporting_start_time();
1293 
1294     if (ShowMessageBoxOnError || PauseAtExit) {
1295       show_message_box(buffer, sizeof(buffer));
1296 
1297       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
1298       // WatcherThread can kill JVM if the error handler hangs.
1299       ShowMessageBoxOnError = false;
1300     }
1301 
1302     os::check_dump_limit(buffer, sizeof(buffer));
1303 
1304     // reset signal handlers or exception filter; make sure recursive crashes
1305     // are handled properly.
1306     reset_signal_handlers();
1307 
1308     TRACE_VM_ERROR();
1309 
1310   } else {
1311     // If UseOsErrorReporting we call this for each level of the call stack
1312     // while searching for the exception handler.  Only the first level needs
1313     // to be reported.
1314     if (UseOSErrorReporting && log_done) return;
1315 
1316     // This is not the first error, see if it happened in a different thread
1317     // or in the same thread during error reporting.
1318     if (first_error_tid != mytid) {
1319       char msgbuf[64];
1320       jio_snprintf(msgbuf, sizeof(msgbuf),
1321                    "[thread " INTX_FORMAT " also had an error]",
1322                    mytid);
1323       out.print_raw_cr(msgbuf);
1324 
1325       // error reporting is not MT-safe, block current thread
1326       os::infinite_sleep();
1327 
1328     } else {
1329       if (recursive_error_count++ > 30) {
1330         out.print_raw_cr("[Too many errors, abort]");
1331         os::die();
1332       }
1333 
1334       outputStream* const st = log.is_open() ? &log : &out;
1335       st->cr();
1336 
1337       // Timeout handling.
1338       if (_step_did_timeout) {
1339         // The current step had a timeout. Lets continue reporting with the next step.
1340         st->print_raw("[timeout occurred during error reporting in step \"");
1341         st->print_raw(_current_step_info);
1342         st->print_cr("\"] after " INT64_FORMAT " s.",
1343                      (int64_t)
1344                      ((get_current_timestamp() - _step_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1345       } else if (_reporting_did_timeout) {
1346         // We hit ErrorLogTimeout. Reporting will stop altogether. Let's wrap things
1347         // up, the process is about to be stopped by the WatcherThread.
1348         st->print_cr("------ Timeout during error reporting after " INT64_FORMAT " s. ------",
1349                      (int64_t)
1350                      ((get_current_timestamp() - _reporting_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1351         st->flush();
1352         // Watcherthread is about to call os::die. Lets just wait.
1353         os::infinite_sleep();
1354       } else {
1355         // Crash or assert during error reporting. Lets continue reporting with the next step.
1356         jio_snprintf(buffer, sizeof(buffer),
1357            "[error occurred during error reporting (%s), id 0x%x]",
1358                    _current_step_info, _id);
1359         st->print_raw_cr(buffer);
1360         st->cr();
1361       }
1362     }
1363   }
1364 
1365   // print to screen
1366   if (!out_done) {
1367     report(&out, false);
1368 
1369     out_done = true;
1370 
1371     _current_step = 0;
1372     _current_step_info = "";
1373   }
1374 
1375   // print to error log file
1376   if (!log_done) {
1377     // see if log file is already open
1378     if (!log.is_open()) {
1379       // open log file
1380       int fd = prepare_log_file(ErrorFile, "hs_err_pid%p.log", buffer, sizeof(buffer));
1381       if (fd != -1) {
1382         out.print_raw("# An error report file with more information is saved as:\n# ");
1383         out.print_raw_cr(buffer);
1384 
1385         log.set_fd(fd);
1386       } else {
1387         out.print_raw_cr("# Can not save log file, dump to screen..");
1388         log.set_fd(defaultStream::output_fd());
1389         /* Error reporting currently needs dumpfile.
1390          * Maybe implement direct streaming in the future.*/
1391         transmit_report_done = true;
1392       }
1393     }
1394 
1395     report(&log, true);
1396     log_done = true;
1397     _current_step = 0;
1398     _current_step_info = "";
1399 
1400     // Run error reporting to determine whether or not to report the crash.
1401     if (!transmit_report_done && should_report_bug(_id)) {
1402       transmit_report_done = true;
1403       const int fd2 = ::dup(log.fd());
1404       if (fd2 != -1) {
1405         FILE* const hs_err = ::fdopen(fd2, "r");
1406         if (NULL != hs_err) {
1407           ErrorReporter er;
1408           er.call(hs_err, buffer, O_BUFLEN);
1409           ::fclose(hs_err);
1410         }
1411       }
1412     }
1413 
1414     if (log.fd() != defaultStream::output_fd()) {
1415       close(log.fd());
1416     }
1417 
1418     log.set_fd(-1);
1419   }
1420 
1421   static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
1422   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
1423     skip_replay = true;
1424     ciEnv* env = ciEnv::current();
1425     if (env != NULL) {
1426       int fd = prepare_log_file(ReplayDataFile, "replay_pid%p.log", buffer, sizeof(buffer));
1427       if (fd != -1) {
1428         FILE* replay_data_file = os::open(fd, "w");
1429         if (replay_data_file != NULL) {
1430           fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
1431           env->dump_replay_data_unsafe(&replay_data_stream);
1432           out.print_raw("#\n# Compiler replay data is saved as:\n# ");
1433           out.print_raw_cr(buffer);
1434         } else {
1435           int e = errno;
1436           out.print_raw("#\n# Can't open file to dump replay data. Error: ");
1437           out.print_raw_cr(os::strerror(e));
1438         }
1439       }
1440     }
1441   }
1442 
1443   static bool skip_bug_url = !should_report_bug(_id);
1444   if (!skip_bug_url) {
1445     skip_bug_url = true;
1446 
1447     out.print_raw_cr("#");
1448     print_bug_submit_message(&out, _thread);
1449   }
1450 
1451   static bool skip_OnError = false;
1452   if (!skip_OnError && OnError && OnError[0]) {
1453     skip_OnError = true;
1454 
1455     // Flush output and finish logs before running OnError commands.
1456     ostream_abort();
1457 
1458     out.print_raw_cr("#");
1459     out.print_raw   ("# -XX:OnError=\"");
1460     out.print_raw   (OnError);
1461     out.print_raw_cr("\"");
1462 
1463     char* cmd;
1464     const char* ptr = OnError;
1465     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1466       out.print_raw   ("#   Executing ");
1467 #if defined(LINUX) || defined(_ALLBSD_SOURCE)
1468       out.print_raw   ("/bin/sh -c ");
1469 #elif defined(SOLARIS)
1470       out.print_raw   ("/usr/bin/sh -c ");
1471 #elif defined(WINDOWS)
1472       out.print_raw   ("cmd /C ");
1473 #endif
1474       out.print_raw   ("\"");
1475       out.print_raw   (cmd);
1476       out.print_raw_cr("\" ...");
1477 
1478       if (os::fork_and_exec(cmd) < 0) {
1479         out.print_cr("os::fork_and_exec failed: %s (%s=%d)",
1480                      os::strerror(errno), os::errno_name(errno), errno);
1481       }
1482     }
1483 
1484     // done with OnError
1485     OnError = NULL;
1486   }
1487 
1488   if (!UseOSErrorReporting) {
1489     // os::abort() will call abort hooks, try it first.
1490     static bool skip_os_abort = false;
1491     if (!skip_os_abort) {
1492       skip_os_abort = true;
1493       bool dump_core = should_report_bug(_id);
1494       os::abort(dump_core && CreateCoredumpOnCrash, _siginfo, _context);
1495     }
1496 
1497     // if os::abort() doesn't abort, try os::die();
1498     os::die();
1499   }
1500 }
1501 
1502 /*
1503  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
1504  * ensures utilities such as jmap can observe the process is a consistent state.
1505  */
1506 class VM_ReportJavaOutOfMemory : public VM_Operation {
1507  private:
1508   const char* _message;
1509  public:
1510   VM_ReportJavaOutOfMemory(const char* message) { _message = message; }
1511   VMOp_Type type() const                        { return VMOp_ReportJavaOutOfMemory; }
1512   void doit();
1513 };
1514 
1515 void VM_ReportJavaOutOfMemory::doit() {
1516   // Don't allocate large buffer on stack
1517   static char buffer[O_BUFLEN];
1518 
1519   tty->print_cr("#");
1520   tty->print_cr("# java.lang.OutOfMemoryError: %s", _message);
1521   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
1522 
1523   // make heap parsability
1524   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1525 
1526   char* cmd;
1527   const char* ptr = OnOutOfMemoryError;
1528   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1529     tty->print("#   Executing ");
1530 #if defined(LINUX)
1531     tty->print  ("/bin/sh -c ");
1532 #elif defined(SOLARIS)
1533     tty->print  ("/usr/bin/sh -c ");
1534 #endif
1535     tty->print_cr("\"%s\"...", cmd);
1536 
1537     if (os::fork_and_exec(cmd) < 0) {
1538       tty->print_cr("os::fork_and_exec failed: %s (%s=%d)",
1539                      os::strerror(errno), os::errno_name(errno), errno);
1540     }
1541   }
1542 }
1543 
1544 void VMError::report_java_out_of_memory(const char* message) {
1545   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
1546     MutexLocker ml(Heap_lock);
1547     VM_ReportJavaOutOfMemory op(message);
1548     VMThread::execute(&op);
1549   }
1550 }
1551 
1552 void VMError::show_message_box(char *buf, int buflen) {
1553   bool yes;
1554   do {
1555     error_string(buf, buflen);
1556     yes = os::start_debugging(buf,buflen);
1557   } while (yes);
1558 }
1559 
1560 // Timeout handling: check if a timeout happened (either a single step did
1561 // timeout or the whole of error reporting hit ErrorLogTimeout). Interrupt
1562 // the reporting thread if that is the case.
1563 bool VMError::check_timeout() {
1564 
1565   if (ErrorLogTimeout == 0) {
1566     return false;
1567   }
1568 
1569   // Do not check for timeouts if we still have a message box to show to the
1570   // user or if there are OnError handlers to be run.
1571   if (ShowMessageBoxOnError
1572       || (OnError != NULL && OnError[0] != '\0')
1573       || Arguments::abort_hook() != NULL) {
1574     return false;
1575   }
1576 
1577   const jlong reporting_start_time_l = get_reporting_start_time();
1578   const jlong now = get_current_timestamp();
1579   // Timestamp is stored in nanos.
1580   if (reporting_start_time_l > 0) {
1581     const jlong end = reporting_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR;
1582     if (end <= now) {
1583       _reporting_did_timeout = true;
1584       interrupt_reporting_thread();
1585       return true; // global timeout
1586     }
1587   }
1588 
1589   const jlong step_start_time_l = get_step_start_time();
1590   if (step_start_time_l > 0) {
1591     // A step times out after a quarter of the total timeout. Steps are mostly fast unless they
1592     // hang for some reason, so this simple rule allows for three hanging step and still
1593     // hopefully leaves time enough for the rest of the steps to finish.
1594     const jlong end = step_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR / 4;
1595     if (end <= now) {
1596       _step_did_timeout = true;
1597       interrupt_reporting_thread();
1598       return false; // (Not a global timeout)
1599     }
1600   }
1601 
1602   return false;
1603 
1604 }
1605 
1606 #ifndef PRODUCT
1607 typedef void (*voidfun_t)();
1608 // Crash with an authentic sigfpe
1609 static void crash_with_sigfpe() {
1610   // generate a native synchronous SIGFPE where possible;
1611   // if that did not cause a signal (e.g. on ppc), just
1612   // raise the signal.
1613   volatile int x = 0;
1614   volatile int y = 1/x;
1615 #ifndef _WIN32
1616   // OSX implements raise(sig) incorrectly so we need to
1617   // explicitly target the current thread
1618   pthread_kill(pthread_self(), SIGFPE);
1619 #endif
1620 } // end: crash_with_sigfpe
1621 
1622 // crash with sigsegv at non-null address.
1623 static void crash_with_segfault() {
1624 
1625   char* const crash_addr = (char*) VMError::get_segfault_address();
1626   *crash_addr = 'X';
1627 
1628 } // end: crash_with_segfault
1629 
1630 void VMError::test_error_handler() {
1631   controlled_crash(ErrorHandlerTest);
1632 }
1633 
1634 // crash in a controlled way:
1635 // how can be one of:
1636 // 1,2 - asserts
1637 // 3,4 - guarantee
1638 // 5-7 - fatal
1639 // 8 - vm_exit_out_of_memory
1640 // 9 - ShouldNotCallThis
1641 // 10 - ShouldNotReachHere
1642 // 11 - Unimplemented
1643 // 12,13 - (not guaranteed) crashes
1644 // 14 - SIGSEGV
1645 // 15 - SIGFPE
1646 void VMError::controlled_crash(int how) {
1647   if (how == 0) return;
1648 
1649   // If asserts are disabled, use the corresponding guarantee instead.
1650   NOT_DEBUG(if (how <= 2) how += 2);
1651 
1652   const char* const str = "hello";
1653   const size_t      num = (size_t)os::vm_page_size();
1654 
1655   const char* const eol = os::line_separator();
1656   const char* const msg = "this message should be truncated during formatting";
1657   char * const dataPtr = NULL;  // bad data pointer
1658   const void (*funcPtr)(void) = (const void(*)()) 0xF;  // bad function pointer
1659 
1660   // Keep this in sync with test/hotspot/jtreg/runtime/ErrorHandling/ErrorHandler.java
1661   // which tests cases 1 thru 13.
1662   // Case 14 is tested by test/hotspot/jtreg/runtime/ErrorHandling/SafeFetchInErrorHandlingTest.java.
1663   // Case 15 is tested by test/hotspot/jtreg/runtime/ErrorHandling/SecondaryErrorTest.java.
1664   // Case 16 is tested by test/hotspot/jtreg/runtime/ErrorHandling/ThreadsListHandleInErrorHandlingTest.java.
1665   // Case 17 is tested by test/hotspot/jtreg/runtime/ErrorHandling/NestedThreadsListHandleInErrorHandlingTest.java.
1666   switch (how) {
1667     case  1: vmassert(str == NULL, "expected null");
1668     case  2: vmassert(num == 1023 && *str == 'X',
1669                       "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1670     case  3: guarantee(str == NULL, "expected null");
1671     case  4: guarantee(num == 1023 && *str == 'X',
1672                        "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1673     case  5: fatal("expected null");
1674     case  6: fatal("num=" SIZE_FORMAT " str=\"%s\"", num, str);
1675     case  7: fatal("%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1676                    "%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1677                    "%s%s#    %s%s#    %s%s#    %s%s#    %s",
1678                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1679                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1680                    msg, eol, msg, eol, msg, eol, msg, eol, msg);
1681     case  8: vm_exit_out_of_memory(num, OOM_MALLOC_ERROR, "ChunkPool::allocate");
1682     case  9: ShouldNotCallThis();
1683     case 10: ShouldNotReachHere();
1684     case 11: Unimplemented();
1685     // There's no guarantee the bad data pointer will crash us
1686     // so "break" out to the ShouldNotReachHere().
1687     case 12: *dataPtr = '\0'; break;
1688     // There's no guarantee the bad function pointer will crash us
1689     // so "break" out to the ShouldNotReachHere().
1690     case 13: (*funcPtr)(); break;
1691     case 14: crash_with_segfault(); break;
1692     case 15: crash_with_sigfpe(); break;
1693     case 16: {
1694       ThreadsListHandle tlh;
1695       fatal("Force crash with an active ThreadsListHandle.");
1696     }
1697     case 17: {
1698       ThreadsListHandle tlh;
1699       {
1700         ThreadsListHandle tlh2;
1701         fatal("Force crash with a nested ThreadsListHandle.");
1702       }
1703     }
1704 
1705     default: tty->print_cr("ERROR: %d: unexpected test_num value.", how);
1706   }
1707   ShouldNotReachHere();
1708 }
1709 #endif // !PRODUCT
1710