1 /*
   2  * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "code/codeCache.hpp"
  28 #include "compiler/compileBroker.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "logging/logConfiguration.hpp"
  32 #include "prims/whitebox.hpp"
  33 #include "runtime/arguments.hpp"
  34 #include "runtime/atomic.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/init.hpp"
  37 #include "runtime/os.hpp"
  38 #include "runtime/thread.inline.hpp"
  39 #include "runtime/threadSMR.hpp"
  40 #include "runtime/vmThread.hpp"
  41 #include "runtime/vm_operations.hpp"
  42 #include "runtime/vm_version.hpp"
  43 #include "services/memTracker.hpp"
  44 #include "trace/traceMacros.hpp"
  45 #include "utilities/debug.hpp"
  46 #include "utilities/decoder.hpp"
  47 #include "utilities/defaultStream.hpp"
  48 #include "utilities/errorReporter.hpp"
  49 #include "utilities/events.hpp"
  50 #include "utilities/vmError.hpp"
  51 
  52 #ifndef PRODUCT
  53 #include <signal.h>
  54 #endif // PRODUCT
  55 
  56 bool VMError::_error_reported = false;
  57 
  58 // call this when the VM is dying--it might loosen some asserts
  59 bool VMError::is_error_reported() { return _error_reported; }
  60 
  61 // returns an address which is guaranteed to generate a SIGSEGV on read,
  62 // for test purposes, which is not NULL and contains bits in every word
  63 void* VMError::get_segfault_address() {
  64   return (void*)
  65 #ifdef _LP64
  66     0xABC0000000000ABCULL;
  67 #else
  68     0x00000ABC;
  69 #endif
  70 }
  71 
  72 // List of environment variables that should be reported in error log file.
  73 const char *env_list[] = {
  74   // All platforms
  75   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  76   "JAVA_COMPILER", "PATH", "USERNAME",
  77 
  78   // Env variables that are defined on Solaris/Linux/BSD
  79   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  80   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  81 
  82   // defined on Linux
  83   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  84 
  85   // defined on Darwin
  86   "DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH",
  87   "DYLD_FRAMEWORK_PATH", "DYLD_FALLBACK_FRAMEWORK_PATH",
  88   "DYLD_INSERT_LIBRARIES",
  89 
  90   // defined on Windows
  91   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  92 
  93   (const char *)0
  94 };
  95 
  96 // A simple parser for -XX:OnError, usage:
  97 //  ptr = OnError;
  98 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
  99 //     ... ...
 100 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
 101   if (ptr == NULL || *ptr == NULL) return NULL;
 102 
 103   const char* cmd = *ptr;
 104 
 105   // skip leading blanks or ';'
 106   while (*cmd == ' ' || *cmd == ';') cmd++;
 107 
 108   if (*cmd == '\0') return NULL;
 109 
 110   const char * cmdend = cmd;
 111   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
 112 
 113   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
 114 
 115   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
 116   return buf;
 117 }
 118 
 119 static void print_bug_submit_message(outputStream *out, Thread *thread) {
 120   if (out == NULL) return;
 121   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 122   out->print_raw   ("#   ");
 123   out->print_raw_cr(Arguments::java_vendor_url_bug());
 124   // If the crash is in native code, encourage user to submit a bug to the
 125   // provider of that code.
 126   if (thread && thread->is_Java_thread() &&
 127       !thread->is_hidden_from_external_view()) {
 128     JavaThread* jt = (JavaThread*)thread;
 129     if (jt->thread_state() == _thread_in_native) {
 130       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 131     }
 132   }
 133   out->print_raw_cr("#");
 134 }
 135 
 136 bool VMError::coredump_status;
 137 char VMError::coredump_message[O_BUFLEN];
 138 
 139 void VMError::record_coredump_status(const char* message, bool status) {
 140   coredump_status = status;
 141   strncpy(coredump_message, message, sizeof(coredump_message));
 142   coredump_message[sizeof(coredump_message)-1] = 0;
 143 }
 144 
 145 // Return a string to describe the error
 146 char* VMError::error_string(char* buf, int buflen) {
 147   char signame_buf[64];
 148   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 149 
 150   if (signame) {
 151     jio_snprintf(buf, buflen,
 152                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 153                  signame, _id, _pc,
 154                  os::current_process_id(), os::current_thread_id());
 155   } else if (_filename != NULL && _lineno > 0) {
 156     // skip directory names
 157     char separator = os::file_separator()[0];
 158     const char *p = strrchr(_filename, separator);
 159     int n = jio_snprintf(buf, buflen,
 160                          "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT,
 161                          p ? p + 1 : _filename, _lineno,
 162                          os::current_process_id(), os::current_thread_id());
 163     if (n >= 0 && n < buflen && _message) {
 164       if (strlen(_detail_msg) > 0) {
 165         jio_snprintf(buf + n, buflen - n, "%s%s: %s",
 166         os::line_separator(), _message, _detail_msg);
 167       } else {
 168         jio_snprintf(buf + n, buflen - n, "%sError: %s",
 169                      os::line_separator(), _message);
 170       }
 171     }
 172   } else {
 173     jio_snprintf(buf, buflen,
 174                  "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 175                  _id, os::current_process_id(), os::current_thread_id());
 176   }
 177 
 178   return buf;
 179 }
 180 
 181 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 182                                 char* buf, int buflen, bool verbose) {
 183 #ifdef ZERO
 184   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 185     // StackFrameStream uses the frame anchor, which may not have
 186     // been set up.  This can be done at any time in Zero, however,
 187     // so if it hasn't been set up then we just set it up now and
 188     // clear it again when we're done.
 189     bool has_last_Java_frame = jt->has_last_Java_frame();
 190     if (!has_last_Java_frame)
 191       jt->set_last_Java_frame();
 192     st->print("Java frames:");
 193     st->cr();
 194 
 195     // Print the frames
 196     StackFrameStream sfs(jt);
 197     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 198       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 199       st->cr();
 200     }
 201 
 202     // Reset the frame anchor if necessary
 203     if (!has_last_Java_frame)
 204       jt->reset_last_Java_frame();
 205   }
 206 #else
 207   if (jt->has_last_Java_frame()) {
 208     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 209     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 210       sfs.current()->print_on_error(st, buf, buflen, verbose);
 211       st->cr();
 212     }
 213   }
 214 #endif // ZERO
 215 }
 216 
 217 void VMError::print_native_stack(outputStream* st, frame fr, Thread* t, char* buf, int buf_size) {
 218 
 219   // see if it's a valid frame
 220   if (fr.pc()) {
 221     st->print_cr("Native frames: (J=compiled Java code, A=aot compiled Java code, j=interpreted, Vv=VM code, C=native code)");
 222 
 223     int count = 0;
 224     while (count++ < StackPrintLimit) {
 225       fr.print_on_error(st, buf, buf_size);
 226       if (fr.pc()) { // print source file and line, if available
 227         char buf[128];
 228         int line_no;
 229         if (Decoder::get_source_info(fr.pc(), buf, sizeof(buf), &line_no)) {
 230           st->print("  (%s:%d)", buf, line_no);
 231         }
 232       }
 233       st->cr();
 234       // Compiled code may use EBP register on x86 so it looks like
 235       // non-walkable C frame. Use frame.sender() for java frames.
 236       if (t && t->is_Java_thread()) {
 237         // Catch very first native frame by using stack address.
 238         // For JavaThread stack_base and stack_size should be set.
 239         if (!t->on_local_stack((address)(fr.real_fp() + 1))) {
 240           break;
 241         }
 242         if (fr.is_java_frame() || fr.is_native_frame() || fr.is_runtime_frame()) {
 243           RegisterMap map((JavaThread*)t, false); // No update
 244           fr = fr.sender(&map);
 245         } else {
 246           // is_first_C_frame() does only simple checks for frame pointer,
 247           // it will pass if java compiled code has a pointer in EBP.
 248           if (os::is_first_C_frame(&fr)) break;
 249           fr = os::get_sender_for_C_frame(&fr);
 250         }
 251       } else {
 252         if (os::is_first_C_frame(&fr)) break;
 253         fr = os::get_sender_for_C_frame(&fr);
 254       }
 255     }
 256 
 257     if (count > StackPrintLimit) {
 258       st->print_cr("...<more frames>...");
 259     }
 260 
 261     st->cr();
 262   }
 263 }
 264 
 265 static void print_oom_reasons(outputStream* st) {
 266   st->print_cr("# Possible reasons:");
 267   st->print_cr("#   The system is out of physical RAM or swap space");
 268   if (UseCompressedOops) {
 269     st->print_cr("#   The process is running with CompressedOops enabled, and the Java Heap may be blocking the growth of the native heap");
 270   }
 271   if (LogBytesPerWord == 2) {
 272     st->print_cr("#   In 32 bit mode, the process size limit was hit");
 273   }
 274   st->print_cr("# Possible solutions:");
 275   st->print_cr("#   Reduce memory load on the system");
 276   st->print_cr("#   Increase physical memory or swap space");
 277   st->print_cr("#   Check if swap backing store is full");
 278   if (LogBytesPerWord == 2) {
 279     st->print_cr("#   Use 64 bit Java on a 64 bit OS");
 280   }
 281   st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
 282   st->print_cr("#   Decrease number of Java threads");
 283   st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
 284   st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
 285   if (UseCompressedOops) {
 286     switch (Universe::narrow_oop_mode()) {
 287       case Universe::UnscaledNarrowOop:
 288         st->print_cr("#   JVM is running with Unscaled Compressed Oops mode in which the Java heap is");
 289         st->print_cr("#     placed in the first 4GB address space. The Java Heap base address is the");
 290         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 291         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 4GB virtual address.");
 292         break;
 293       case Universe::ZeroBasedNarrowOop:
 294         st->print_cr("#   JVM is running with Zero Based Compressed Oops mode in which the Java heap is");
 295         st->print_cr("#     placed in the first 32GB address space. The Java Heap base address is the");
 296         st->print_cr("#     maximum limit for the native heap growth. Please use -XX:HeapBaseMinAddress");
 297         st->print_cr("#     to set the Java Heap base and to place the Java Heap above 32GB virtual address.");
 298         break;
 299       default:
 300         break;
 301     }
 302   }
 303   st->print_cr("# This output file may be truncated or incomplete.");
 304 }
 305 
 306 static const char* gc_mode() {
 307   if (UseG1GC)            return "g1 gc";
 308   if (UseParallelGC)      return "parallel gc";
 309   if (UseConcMarkSweepGC) return "concurrent mark sweep gc";
 310   if (UseSerialGC)        return "serial gc";
 311   return "ERROR in GC mode";
 312 }
 313 
 314 static void report_vm_version(outputStream* st, char* buf, int buflen) {
 315    // VM version
 316    st->print_cr("#");
 317    JDK_Version::current().to_string(buf, buflen);
 318    const char* runtime_name = JDK_Version::runtime_name() != NULL ?
 319                                 JDK_Version::runtime_name() : "";
 320    const char* runtime_version = JDK_Version::runtime_version() != NULL ?
 321                                    JDK_Version::runtime_version() : "";
 322    const char* jdk_debug_level = Abstract_VM_Version::printable_jdk_debug_level() != NULL ?
 323                                    Abstract_VM_Version::printable_jdk_debug_level() : "";
 324 
 325    st->print_cr("# JRE version: %s (%s) (%sbuild %s)", runtime_name, buf,
 326                  jdk_debug_level, runtime_version);
 327 
 328    // This is the long version with some default settings added
 329    st->print_cr("# Java VM: %s (%s%s, %s%s%s%s%s, %s, %s)",
 330                  Abstract_VM_Version::vm_name(),
 331                  jdk_debug_level,
 332                  Abstract_VM_Version::vm_release(),
 333                  Abstract_VM_Version::vm_info_string(),
 334                  TieredCompilation ? ", tiered" : "",
 335 #if INCLUDE_JVMCI
 336                  EnableJVMCI ? ", jvmci" : "",
 337                  UseJVMCICompiler ? ", jvmci compiler" : "",
 338 #else
 339                  "", "",
 340 #endif
 341                  UseCompressedOops ? ", compressed oops" : "",
 342                  gc_mode(),
 343                  Abstract_VM_Version::vm_platform_string()
 344                );
 345 }
 346 
 347 // This is the main function to report a fatal error. Only one thread can
 348 // call this function, so we don't need to worry about MT-safety. But it's
 349 // possible that the error handler itself may crash or die on an internal
 350 // error, for example, when the stack/heap is badly damaged. We must be
 351 // able to handle recursive errors that happen inside error handler.
 352 //
 353 // Error reporting is done in several steps. If a crash or internal error
 354 // occurred when reporting an error, the nested signal/exception handler
 355 // can skip steps that are already (or partially) done. Error reporting will
 356 // continue from the next step. This allows us to retrieve and print
 357 // information that may be unsafe to get after a fatal error. If it happens,
 358 // you may find nested report_and_die() frames when you look at the stack
 359 // in a debugger.
 360 //
 361 // In general, a hang in error handler is much worse than a crash or internal
 362 // error, as it's harder to recover from a hang. Deadlock can happen if we
 363 // try to grab a lock that is already owned by current thread, or if the
 364 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 365 // error handler and all the functions it called should avoid grabbing any
 366 // lock. An important thing to notice is that memory allocation needs a lock.
 367 //
 368 // We should avoid using large stack allocated buffers. Many errors happen
 369 // when stack space is already low. Making things even worse is that there
 370 // could be nested report_and_die() calls on stack (see above). Only one
 371 // thread can report error, so large buffers are statically allocated in data
 372 // segment.
 373 
 374 int          VMError::_current_step;
 375 const char*  VMError::_current_step_info;
 376 
 377 volatile jlong VMError::_reporting_start_time = -1;
 378 volatile bool VMError::_reporting_did_timeout = false;
 379 volatile jlong VMError::_step_start_time = -1;
 380 volatile bool VMError::_step_did_timeout = false;
 381 
 382 // Helper, return current timestamp for timeout handling.
 383 jlong VMError::get_current_timestamp() {
 384   return os::javaTimeNanos();
 385 }
 386 // Factor to translate the timestamp to seconds.
 387 #define TIMESTAMP_TO_SECONDS_FACTOR (1000 * 1000 * 1000)
 388 
 389 void VMError::record_reporting_start_time() {
 390   const jlong now = get_current_timestamp();
 391   Atomic::store(now, &_reporting_start_time);
 392 }
 393 
 394 jlong VMError::get_reporting_start_time() {
 395   return Atomic::load(&_reporting_start_time);
 396 }
 397 
 398 void VMError::record_step_start_time() {
 399   const jlong now = get_current_timestamp();
 400   Atomic::store(now, &_step_start_time);
 401 }
 402 
 403 jlong VMError::get_step_start_time() {
 404   return Atomic::load(&_step_start_time);
 405 }
 406 
 407 void VMError::report(outputStream* st, bool _verbose) {
 408 
 409 # define BEGIN if (_current_step == 0) { _current_step = __LINE__;
 410 # define STEP(s) } if (_current_step < __LINE__) { _current_step = __LINE__; _current_step_info = s; \
 411   record_step_start_time(); _step_did_timeout = false;
 412 # define END }
 413 
 414   // don't allocate large buffer on stack
 415   static char buf[O_BUFLEN];
 416 
 417   BEGIN
 418 
 419   STEP("printing fatal error message")
 420 
 421     st->print_cr("#");
 422     if (should_report_bug(_id)) {
 423       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 424     } else {
 425       st->print_cr("# There is insufficient memory for the Java "
 426                    "Runtime Environment to continue.");
 427     }
 428 
 429 #ifndef PRODUCT
 430   // Error handler self tests
 431 
 432   // test secondary error handling. Test it twice, to test that resetting
 433   // error handler after a secondary crash works.
 434   STEP("test secondary crash 1")
 435     if (_verbose && TestCrashInErrorHandler != 0) {
 436       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 437         TestCrashInErrorHandler);
 438       controlled_crash(TestCrashInErrorHandler);
 439     }
 440 
 441   STEP("test secondary crash 2")
 442     if (_verbose && TestCrashInErrorHandler != 0) {
 443       st->print_cr("Will crash now (TestCrashInErrorHandler=" UINTX_FORMAT ")...",
 444         TestCrashInErrorHandler);
 445       controlled_crash(TestCrashInErrorHandler);
 446     }
 447 
 448   // TestUnresponsiveErrorHandler: We want to test both step timeouts and global timeout.
 449   // Step to global timeout ratio is 4:1, so in order to be absolutely sure we hit the
 450   // global timeout, let's execute the timeout step five times.
 451   // See corresponding test in test/runtime/ErrorHandling/TimeoutInErrorHandlingTest.java
 452   #define TIMEOUT_TEST_STEP STEP("test unresponsive error reporting step") \
 453     if (_verbose && TestUnresponsiveErrorHandler) { os::infinite_sleep(); }
 454   TIMEOUT_TEST_STEP
 455   TIMEOUT_TEST_STEP
 456   TIMEOUT_TEST_STEP
 457   TIMEOUT_TEST_STEP
 458   TIMEOUT_TEST_STEP
 459 
 460   STEP("test safefetch in error handler")
 461     // test whether it is safe to use SafeFetch32 in Crash Handler. Test twice
 462     // to test that resetting the signal handler works correctly.
 463     if (_verbose && TestSafeFetchInErrorHandler) {
 464       st->print_cr("Will test SafeFetch...");
 465       if (CanUseSafeFetch32()) {
 466         int* const invalid_pointer = (int*) get_segfault_address();
 467         const int x = 0x76543210;
 468         int i1 = SafeFetch32(invalid_pointer, x);
 469         int i2 = SafeFetch32(invalid_pointer, x);
 470         if (i1 == x && i2 == x) {
 471           st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern
 472         } else {
 473           st->print_cr("??");
 474         }
 475       } else {
 476         st->print_cr("not possible; skipped.");
 477       }
 478     }
 479 #endif // PRODUCT
 480 
 481   STEP("printing type of error")
 482 
 483      switch(static_cast<unsigned int>(_id)) {
 484        case OOM_MALLOC_ERROR:
 485        case OOM_MMAP_ERROR:
 486          if (_size) {
 487            st->print("# Native memory allocation ");
 488            st->print((_id == (int)OOM_MALLOC_ERROR) ? "(malloc) failed to allocate " :
 489                                                  "(mmap) failed to map ");
 490            jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
 491            st->print("%s", buf);
 492            st->print(" bytes");
 493            if (strlen(_detail_msg) > 0) {
 494              st->print(" for ");
 495              st->print("%s", _detail_msg);
 496            }
 497            st->cr();
 498          } else {
 499            if (strlen(_detail_msg) > 0) {
 500              st->print("# ");
 501              st->print_cr("%s", _detail_msg);
 502            }
 503          }
 504          // In error file give some solutions
 505          if (_verbose) {
 506            print_oom_reasons(st);
 507          } else {
 508            return;  // that's enough for the screen
 509          }
 510          break;
 511        case INTERNAL_ERROR:
 512        default:
 513          break;
 514      }
 515 
 516   STEP("printing exception/signal name")
 517 
 518      st->print_cr("#");
 519      st->print("#  ");
 520      // Is it an OS exception/signal?
 521      if (os::exception_name(_id, buf, sizeof(buf))) {
 522        st->print("%s", buf);
 523        st->print(" (0x%x)", _id);                // signal number
 524        st->print(" at pc=" PTR_FORMAT, p2i(_pc));
 525      } else {
 526        if (should_report_bug(_id)) {
 527          st->print("Internal Error");
 528        } else {
 529          st->print("Out of Memory Error");
 530        }
 531        if (_filename != NULL && _lineno > 0) {
 532 #ifdef PRODUCT
 533          // In product mode chop off pathname?
 534          char separator = os::file_separator()[0];
 535          const char *p = strrchr(_filename, separator);
 536          const char *file = p ? p+1 : _filename;
 537 #else
 538          const char *file = _filename;
 539 #endif
 540          st->print(" (%s:%d)", file, _lineno);
 541        } else {
 542          st->print(" (0x%x)", _id);
 543        }
 544      }
 545 
 546   STEP("printing current thread and pid")
 547 
 548      // process id, thread id
 549      st->print(", pid=%d", os::current_process_id());
 550      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 551      st->cr();
 552 
 553   STEP("printing error message")
 554 
 555      if (should_report_bug(_id)) {  // already printed the message.
 556        // error message
 557        if (strlen(_detail_msg) > 0) {
 558          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
 559        } else if (_message) {
 560          st->print_cr("#  Error: %s", _message);
 561        }
 562      }
 563 
 564   STEP("printing Java version string")
 565 
 566      report_vm_version(st, buf, sizeof(buf));
 567 
 568   STEP("printing problematic frame")
 569 
 570      // Print current frame if we have a context (i.e. it's a crash)
 571      if (_context) {
 572        st->print_cr("# Problematic frame:");
 573        st->print("# ");
 574        frame fr = os::fetch_frame_from_context(_context);
 575        fr.print_on_error(st, buf, sizeof(buf));
 576        st->cr();
 577        st->print_cr("#");
 578      }
 579 
 580   STEP("printing core file information")
 581     st->print("# ");
 582     if (CreateCoredumpOnCrash) {
 583       if (coredump_status) {
 584         st->print("Core dump will be written. Default location: %s", coredump_message);
 585       } else {
 586         st->print("No core dump will be written. %s", coredump_message);
 587       }
 588     } else {
 589       st->print("CreateCoredumpOnCrash turned off, no core file dumped");
 590     }
 591     st->cr();
 592     st->print_cr("#");
 593 
 594   STEP("printing bug submit message")
 595 
 596      if (should_report_bug(_id) && _verbose) {
 597        print_bug_submit_message(st, _thread);
 598      }
 599 
 600   STEP("printing summary")
 601 
 602      if (_verbose) {
 603        st->cr();
 604        st->print_cr("---------------  S U M M A R Y ------------");
 605        st->cr();
 606      }
 607 
 608   STEP("printing VM option summary")
 609 
 610      if (_verbose) {
 611        // VM options
 612        Arguments::print_summary_on(st);
 613        st->cr();
 614      }
 615 
 616   STEP("printing summary machine and OS info")
 617 
 618      if (_verbose) {
 619        os::print_summary_info(st, buf, sizeof(buf));
 620      }
 621 
 622 
 623   STEP("printing date and time")
 624 
 625      if (_verbose) {
 626        os::print_date_and_time(st, buf, sizeof(buf));
 627      }
 628 
 629   STEP("printing thread")
 630 
 631      if (_verbose) {
 632        st->cr();
 633        st->print_cr("---------------  T H R E A D  ---------------");
 634        st->cr();
 635      }
 636 
 637   STEP("printing current thread")
 638 
 639      // current thread
 640      if (_verbose) {
 641        if (_thread) {
 642          st->print("Current thread (" PTR_FORMAT "):  ", p2i(_thread));
 643          _thread->print_on_error(st, buf, sizeof(buf));
 644          st->cr();
 645        } else {
 646          st->print_cr("Current thread is native thread");
 647        }
 648        st->cr();
 649      }
 650 
 651   STEP("printing current compile task")
 652 
 653      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 654         CompilerThread* t = (CompilerThread*)_thread;
 655         if (t->task()) {
 656            st->cr();
 657            st->print_cr("Current CompileTask:");
 658            t->task()->print_line_on_error(st, buf, sizeof(buf));
 659            st->cr();
 660         }
 661      }
 662 
 663 
 664   STEP("printing stack bounds")
 665 
 666      if (_verbose) {
 667        st->print("Stack: ");
 668 
 669        address stack_top;
 670        size_t stack_size;
 671 
 672        if (_thread) {
 673           stack_top = _thread->stack_base();
 674           stack_size = _thread->stack_size();
 675        } else {
 676           stack_top = os::current_stack_base();
 677           stack_size = os::current_stack_size();
 678        }
 679 
 680        address stack_bottom = stack_top - stack_size;
 681        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", p2i(stack_bottom), p2i(stack_top));
 682 
 683        frame fr = _context ? os::fetch_frame_from_context(_context)
 684                            : os::current_frame();
 685 
 686        if (fr.sp()) {
 687          st->print(",  sp=" PTR_FORMAT, p2i(fr.sp()));
 688          size_t free_stack_size = pointer_delta(fr.sp(), stack_bottom, 1024);
 689          st->print(",  free space=" SIZE_FORMAT "k", free_stack_size);
 690        }
 691 
 692        st->cr();
 693      }
 694 
 695   STEP("printing native stack")
 696 
 697    if (_verbose) {
 698      if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) {
 699        // We have printed the native stack in platform-specific code
 700        // Windows/x64 needs special handling.
 701      } else {
 702        frame fr = _context ? os::fetch_frame_from_context(_context)
 703                            : os::current_frame();
 704 
 705        print_native_stack(st, fr, _thread, buf, sizeof(buf));
 706      }
 707    }
 708 
 709   STEP("printing Java stack")
 710 
 711      if (_verbose && _thread && _thread->is_Java_thread()) {
 712        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 713      }
 714 
 715   STEP("printing target Java thread stack")
 716 
 717      // printing Java thread stack trace if it is involved in GC crash
 718      if (_verbose && _thread && (_thread->is_Named_thread())) {
 719        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 720        if (jt != NULL) {
 721          st->print_cr("JavaThread " PTR_FORMAT " (nid = %d) was being processed", p2i(jt), jt->osthread()->thread_id());
 722          print_stack_trace(st, jt, buf, sizeof(buf), true);
 723        }
 724      }
 725 
 726   STEP("printing siginfo")
 727 
 728      // signal no, signal code, address that caused the fault
 729      if (_verbose && _siginfo) {
 730        st->cr();
 731        os::print_siginfo(st, _siginfo);
 732        st->cr();
 733      }
 734 
 735   STEP("CDS archive access warning")
 736 
 737      // Print an explicit hint if we crashed on access to the CDS archive.
 738      if (_verbose && _siginfo) {
 739        check_failing_cds_access(st, _siginfo);
 740        st->cr();
 741      }
 742 
 743   STEP("printing register info")
 744 
 745      // decode register contents if possible
 746      if (_verbose && _context && Universe::is_fully_initialized()) {
 747        os::print_register_info(st, _context);
 748        st->cr();
 749      }
 750 
 751   STEP("printing registers, top of stack, instructions near pc")
 752 
 753      // registers, top of stack, instructions near pc
 754      if (_verbose && _context) {
 755        os::print_context(st, _context);
 756        st->cr();
 757      }
 758 
 759   STEP("printing code blob if possible")
 760 
 761      if (_verbose && _context) {
 762        CodeBlob* cb = CodeCache::find_blob(_pc);
 763        if (cb != NULL) {
 764          if (Interpreter::contains(_pc)) {
 765            // The interpreter CodeBlob is very large so try to print the codelet instead.
 766            InterpreterCodelet* codelet = Interpreter::codelet_containing(_pc);
 767            if (codelet != NULL) {
 768              codelet->print_on(st);
 769              Disassembler::decode(codelet->code_begin(), codelet->code_end(), st);
 770            }
 771          } else {
 772            StubCodeDesc* desc = StubCodeDesc::desc_for(_pc);
 773            if (desc != NULL) {
 774              desc->print_on(st);
 775              Disassembler::decode(desc->begin(), desc->end(), st);
 776            } else {
 777              Disassembler::decode(cb, st);
 778              st->cr();
 779            }
 780          }
 781        }
 782      }
 783 
 784   STEP("printing VM operation")
 785 
 786      if (_verbose && _thread && _thread->is_VM_thread()) {
 787         VMThread* t = (VMThread*)_thread;
 788         VM_Operation* op = t->vm_operation();
 789         if (op) {
 790           op->print_on_error(st);
 791           st->cr();
 792           st->cr();
 793         }
 794      }
 795 
 796   STEP("printing process")
 797 
 798      if (_verbose) {
 799        st->cr();
 800        st->print_cr("---------------  P R O C E S S  ---------------");
 801        st->cr();
 802      }
 803 
 804   STEP("printing all threads")
 805 
 806      // all threads
 807      if (_verbose && _thread) {
 808        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 809        st->cr();
 810      }
 811 
 812   STEP("printing VM state")
 813 
 814      if (_verbose) {
 815        // Safepoint state
 816        st->print("VM state:");
 817 
 818        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 819        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 820        else st->print("not at safepoint");
 821 
 822        // Also see if error occurred during initialization or shutdown
 823        if (!Universe::is_fully_initialized()) {
 824          st->print(" (not fully initialized)");
 825        } else if (VM_Exit::vm_exited()) {
 826          st->print(" (shutting down)");
 827        } else {
 828          st->print(" (normal execution)");
 829        }
 830        st->cr();
 831        st->cr();
 832      }
 833 
 834   STEP("printing owned locks on error")
 835 
 836      // mutexes/monitors that currently have an owner
 837      if (_verbose) {
 838        print_owned_locks_on_error(st);
 839        st->cr();
 840      }
 841 
 842   STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
 843 
 844      if (_verbose && Exceptions::has_exception_counts()) {
 845        st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 846        Exceptions::print_exception_counts_on_error(st);
 847        st->cr();
 848      }
 849 
 850   STEP("printing compressed oops mode")
 851 
 852      if (_verbose && UseCompressedOops) {
 853        Universe::print_compressed_oops_mode(st);
 854        if (UseCompressedClassPointers) {
 855          Metaspace::print_compressed_class_space(st);
 856        }
 857        st->cr();
 858      }
 859 
 860   STEP("printing heap information")
 861 
 862      if (_verbose && Universe::is_fully_initialized()) {
 863        Universe::heap()->print_on_error(st);
 864        st->cr();
 865        st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
 866        st->cr();
 867      }
 868 
 869   STEP("printing code cache information")
 870 
 871      if (_verbose && Universe::is_fully_initialized()) {
 872        // print code cache information before vm abort
 873        CodeCache::print_summary(st);
 874        st->cr();
 875      }
 876 
 877   STEP("printing ring buffers")
 878 
 879      if (_verbose) {
 880        Events::print_all(st);
 881        st->cr();
 882      }
 883 
 884   STEP("printing dynamic libraries")
 885 
 886      if (_verbose) {
 887        // dynamic libraries, or memory map
 888        os::print_dll_info(st);
 889        st->cr();
 890      }
 891 
 892   STEP("printing native decoder state")
 893 
 894      if (_verbose) {
 895        Decoder::print_state_on(st);
 896        st->cr();
 897      }
 898 
 899   STEP("printing VM options")
 900 
 901      if (_verbose) {
 902        // VM options
 903        Arguments::print_on(st);
 904        st->cr();
 905      }
 906 
 907   STEP("printing warning if internal testing API used")
 908 
 909      if (WhiteBox::used()) {
 910        st->print_cr("Unsupported internal testing APIs have been used.");
 911        st->cr();
 912      }
 913 
 914   STEP("printing log configuration")
 915     if (_verbose){
 916       st->print_cr("Logging:");
 917       LogConfiguration::describe_current_configuration(st);
 918       st->cr();
 919     }
 920 
 921   STEP("printing all environment variables")
 922 
 923      if (_verbose) {
 924        os::print_environment_variables(st, env_list);
 925        st->cr();
 926      }
 927 
 928   STEP("printing signal handlers")
 929 
 930      if (_verbose) {
 931        os::print_signal_handlers(st, buf, sizeof(buf));
 932        st->cr();
 933      }
 934 
 935   STEP("Native Memory Tracking")
 936      if (_verbose) {
 937        MemTracker::error_report(st);
 938      }
 939 
 940   STEP("printing system")
 941 
 942      if (_verbose) {
 943        st->cr();
 944        st->print_cr("---------------  S Y S T E M  ---------------");
 945        st->cr();
 946      }
 947 
 948   STEP("printing OS information")
 949 
 950      if (_verbose) {
 951        os::print_os_info(st);
 952        st->cr();
 953      }
 954 
 955   STEP("printing CPU info")
 956      if (_verbose) {
 957        os::print_cpu_info(st, buf, sizeof(buf));
 958        st->cr();
 959      }
 960 
 961   STEP("printing memory info")
 962 
 963      if (_verbose) {
 964        os::print_memory_info(st);
 965        st->cr();
 966      }
 967 
 968   STEP("printing internal vm info")
 969 
 970      if (_verbose) {
 971        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 972        st->cr();
 973      }
 974 
 975   // print a defined marker to show that error handling finished correctly.
 976   STEP("printing end marker")
 977 
 978      if (_verbose) {
 979        st->print_cr("END.");
 980      }
 981 
 982   END
 983 
 984 # undef BEGIN
 985 # undef STEP
 986 # undef END
 987 }
 988 
 989 // Report for the vm_info_cmd. This prints out the information above omitting
 990 // crash and thread specific information.  If output is added above, it should be added
 991 // here also, if it is safe to call during a running process.
 992 void VMError::print_vm_info(outputStream* st) {
 993 
 994   char buf[O_BUFLEN];
 995   report_vm_version(st, buf, sizeof(buf));
 996 
 997   // STEP("printing summary")
 998 
 999   st->cr();
1000   st->print_cr("---------------  S U M M A R Y ------------");
1001   st->cr();
1002 
1003   // STEP("printing VM option summary")
1004 
1005   // VM options
1006   Arguments::print_summary_on(st);
1007   st->cr();
1008 
1009   // STEP("printing summary machine and OS info")
1010 
1011   os::print_summary_info(st, buf, sizeof(buf));
1012 
1013   // STEP("printing date and time")
1014 
1015   os::print_date_and_time(st, buf, sizeof(buf));
1016 
1017   // Skip: STEP("printing thread")
1018 
1019   // STEP("printing process")
1020 
1021   st->cr();
1022   st->print_cr("---------------  P R O C E S S  ---------------");
1023   st->cr();
1024 
1025   // STEP("printing number of OutOfMemoryError and StackOverflow exceptions")
1026 
1027   if (Exceptions::has_exception_counts()) {
1028     st->print_cr("OutOfMemory and StackOverflow Exception counts:");
1029     Exceptions::print_exception_counts_on_error(st);
1030     st->cr();
1031   }
1032 
1033   // STEP("printing compressed oops mode")
1034 
1035   if (UseCompressedOops) {
1036     Universe::print_compressed_oops_mode(st);
1037     if (UseCompressedClassPointers) {
1038       Metaspace::print_compressed_class_space(st);
1039     }
1040     st->cr();
1041   }
1042 
1043   // STEP("printing heap information")
1044 
1045   if (Universe::is_fully_initialized()) {
1046     MutexLocker hl(Heap_lock);
1047     Universe::heap()->print_on_error(st);
1048     st->cr();
1049     st->print_cr("Polling page: " INTPTR_FORMAT, p2i(os::get_polling_page()));
1050     st->cr();
1051   }
1052 
1053   // STEP("printing code cache information")
1054 
1055   if (Universe::is_fully_initialized()) {
1056     // print code cache information before vm abort
1057     CodeCache::print_summary(st);
1058     st->cr();
1059   }
1060 
1061   // STEP("printing ring buffers")
1062 
1063   Events::print_all(st);
1064   st->cr();
1065 
1066   // STEP("printing dynamic libraries")
1067 
1068   // dynamic libraries, or memory map
1069   os::print_dll_info(st);
1070   st->cr();
1071 
1072   // STEP("printing VM options")
1073 
1074   // VM options
1075   Arguments::print_on(st);
1076   st->cr();
1077 
1078   // STEP("printing warning if internal testing API used")
1079 
1080   if (WhiteBox::used()) {
1081     st->print_cr("Unsupported internal testing APIs have been used.");
1082     st->cr();
1083   }
1084 
1085   // STEP("printing log configuration")
1086   st->print_cr("Logging:");
1087   LogConfiguration::describe(st);
1088   st->cr();
1089 
1090   // STEP("printing all environment variables")
1091 
1092   os::print_environment_variables(st, env_list);
1093   st->cr();
1094 
1095   // STEP("printing signal handlers")
1096 
1097   os::print_signal_handlers(st, buf, sizeof(buf));
1098   st->cr();
1099 
1100   // STEP("Native Memory Tracking")
1101 
1102   MemTracker::error_report(st);
1103 
1104   // STEP("printing system")
1105 
1106   st->cr();
1107   st->print_cr("---------------  S Y S T E M  ---------------");
1108   st->cr();
1109 
1110   // STEP("printing OS information")
1111 
1112   os::print_os_info(st);
1113   st->cr();
1114 
1115   // STEP("printing CPU info")
1116 
1117   os::print_cpu_info(st, buf, sizeof(buf));
1118   st->cr();
1119 
1120   // STEP("printing memory info")
1121 
1122   os::print_memory_info(st);
1123   st->cr();
1124 
1125   // STEP("printing internal vm info")
1126 
1127   st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
1128   st->cr();
1129 
1130   // print a defined marker to show that error handling finished correctly.
1131   // STEP("printing end marker")
1132 
1133   st->print_cr("END.");
1134 }
1135 
1136 volatile intptr_t VMError::first_error_tid = -1;
1137 
1138 // An error could happen before tty is initialized or after it has been
1139 // destroyed.
1140 // Please note: to prevent large stack allocations, the log- and
1141 // output-stream use a global scratch buffer for format printing.
1142 // (see VmError::report_and_die(). Access to those streams is synchronized
1143 // in  VmError::report_and_die() - there is only one reporting thread at
1144 // any given time.
1145 fdStream VMError::out(defaultStream::output_fd());
1146 fdStream VMError::log; // error log used by VMError::report_and_die()
1147 
1148 /** Expand a pattern into a buffer starting at pos and open a file using constructed path */
1149 static int expand_and_open(const char* pattern, char* buf, size_t buflen, size_t pos) {
1150   int fd = -1;
1151   if (Arguments::copy_expand_pid(pattern, strlen(pattern), &buf[pos], buflen - pos)) {
1152     // the O_EXCL flag will cause the open to fail if the file exists
1153     fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0666);
1154   }
1155   return fd;
1156 }
1157 
1158 /**
1159  * Construct file name for a log file and return it's file descriptor.
1160  * Name and location depends on pattern, default_pattern params and access
1161  * permissions.
1162  */
1163 static int prepare_log_file(const char* pattern, const char* default_pattern, char* buf, size_t buflen) {
1164   int fd = -1;
1165 
1166   // If possible, use specified pattern to construct log file name
1167   if (pattern != NULL) {
1168     fd = expand_and_open(pattern, buf, buflen, 0);
1169   }
1170 
1171   // Either user didn't specify, or the user's location failed,
1172   // so use the default name in the current directory
1173   if (fd == -1) {
1174     const char* cwd = os::get_current_directory(buf, buflen);
1175     if (cwd != NULL) {
1176       size_t pos = strlen(cwd);
1177       int fsep_len = jio_snprintf(&buf[pos], buflen-pos, "%s", os::file_separator());
1178       pos += fsep_len;
1179       if (fsep_len > 0) {
1180         fd = expand_and_open(default_pattern, buf, buflen, pos);
1181       }
1182     }
1183   }
1184 
1185    // try temp directory if it exists.
1186    if (fd == -1) {
1187      const char* tmpdir = os::get_temp_directory();
1188      if (tmpdir != NULL && strlen(tmpdir) > 0) {
1189        int pos = jio_snprintf(buf, buflen, "%s%s", tmpdir, os::file_separator());
1190        if (pos > 0) {
1191          fd = expand_and_open(default_pattern, buf, buflen, pos);
1192        }
1193      }
1194    }
1195 
1196   return fd;
1197 }
1198 
1199 int         VMError::_id;
1200 const char* VMError::_message;
1201 char        VMError::_detail_msg[1024];
1202 Thread*     VMError::_thread;
1203 address     VMError::_pc;
1204 void*       VMError::_siginfo;
1205 void*       VMError::_context;
1206 const char* VMError::_filename;
1207 int         VMError::_lineno;
1208 size_t      VMError::_size;
1209 
1210 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo,
1211                              void* context, const char* detail_fmt, ...)
1212 {
1213   va_list detail_args;
1214   va_start(detail_args, detail_fmt);
1215   report_and_die(sig, NULL, detail_fmt, detail_args, thread, pc, siginfo, context, NULL, 0, 0);
1216   va_end(detail_args);
1217 }
1218 
1219 void VMError::report_and_die(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context)
1220 {
1221   report_and_die(thread, sig, pc, siginfo, context, "%s", "");
1222 }
1223 
1224 void VMError::report_and_die(const char* message, const char* detail_fmt, ...)
1225 {
1226   va_list detail_args;
1227   va_start(detail_args, detail_fmt);
1228   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, NULL, NULL, NULL, NULL, NULL, 0, 0);
1229   va_end(detail_args);
1230 }
1231 
1232 void VMError::report_and_die(const char* message)
1233 {
1234   report_and_die(message, "%s", "");
1235 }
1236 
1237 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, const char* message,
1238                              const char* detail_fmt, va_list detail_args)
1239 {
1240   report_and_die(INTERNAL_ERROR, message, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, 0);
1241 }
1242 
1243 void VMError::report_and_die(Thread* thread, const char* filename, int lineno, size_t size,
1244                              VMErrorType vm_err_type, const char* detail_fmt, va_list detail_args) {
1245   report_and_die(vm_err_type, NULL, detail_fmt, detail_args, thread, NULL, NULL, NULL, filename, lineno, size);
1246 }
1247 
1248 void VMError::report_and_die(int id, const char* message, const char* detail_fmt, va_list detail_args,
1249                              Thread* thread, address pc, void* siginfo, void* context, const char* filename,
1250                              int lineno, size_t size)
1251 {
1252   // Don't allocate large buffer on stack
1253   static char buffer[O_BUFLEN];
1254   out.set_scratch_buffer(buffer, sizeof(buffer));
1255   log.set_scratch_buffer(buffer, sizeof(buffer));
1256 
1257   // How many errors occurred in error handler when reporting first_error.
1258   static int recursive_error_count;
1259 
1260   // We will first print a brief message to standard out (verbose = false),
1261   // then save detailed information in log file (verbose = true).
1262   static bool out_done = false;         // done printing to standard out
1263   static bool log_done = false;         // done saving error log
1264   static bool transmit_report_done = false; // done error reporting
1265 
1266   if (SuppressFatalErrorMessage) {
1267       os::abort(CreateCoredumpOnCrash);
1268   }
1269   intptr_t mytid = os::current_thread_id();
1270   if (first_error_tid == -1 &&
1271       Atomic::cmpxchg(mytid, &first_error_tid, (intptr_t)-1) == -1) {
1272 
1273     // Initialize time stamps to use the same base.
1274     out.time_stamp().update_to(1);
1275     log.time_stamp().update_to(1);
1276 
1277     _id = id;
1278     _message = message;
1279     _thread = thread;
1280     _pc = pc;
1281     _siginfo = siginfo;
1282     _context = context;
1283     _filename = filename;
1284     _lineno = lineno;
1285     _size = size;
1286     jio_vsnprintf(_detail_msg, sizeof(_detail_msg), detail_fmt, detail_args);
1287 
1288     // first time
1289     _error_reported = true;
1290 
1291     reporting_started();
1292     record_reporting_start_time();
1293 
1294     if (ShowMessageBoxOnError || PauseAtExit) {
1295       show_message_box(buffer, sizeof(buffer));
1296 
1297       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
1298       // WatcherThread can kill JVM if the error handler hangs.
1299       ShowMessageBoxOnError = false;
1300     }
1301 
1302     os::check_dump_limit(buffer, sizeof(buffer));
1303 
1304     // reset signal handlers or exception filter; make sure recursive crashes
1305     // are handled properly.
1306     reset_signal_handlers();
1307 
1308     EventShutdown e;
1309     if (e.should_commit()) {
1310       e.set_reason("VM Error");
1311       e.commit();
1312     }
1313 
1314     TRACE_VM_ERROR();
1315 
1316   } else {
1317     // If UseOsErrorReporting we call this for each level of the call stack
1318     // while searching for the exception handler.  Only the first level needs
1319     // to be reported.
1320     if (UseOSErrorReporting && log_done) return;
1321 
1322     // This is not the first error, see if it happened in a different thread
1323     // or in the same thread during error reporting.
1324     if (first_error_tid != mytid) {
1325       char msgbuf[64];
1326       jio_snprintf(msgbuf, sizeof(msgbuf),
1327                    "[thread " INTX_FORMAT " also had an error]",
1328                    mytid);
1329       out.print_raw_cr(msgbuf);
1330 
1331       // error reporting is not MT-safe, block current thread
1332       os::infinite_sleep();
1333 
1334     } else {
1335       if (recursive_error_count++ > 30) {
1336         out.print_raw_cr("[Too many errors, abort]");
1337         os::die();
1338       }
1339 
1340       outputStream* const st = log.is_open() ? &log : &out;
1341       st->cr();
1342 
1343       // Timeout handling.
1344       if (_step_did_timeout) {
1345         // The current step had a timeout. Lets continue reporting with the next step.
1346         st->print_raw("[timeout occurred during error reporting in step \"");
1347         st->print_raw(_current_step_info);
1348         st->print_cr("\"] after " INT64_FORMAT " s.",
1349                      (int64_t)
1350                      ((get_current_timestamp() - _step_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1351       } else if (_reporting_did_timeout) {
1352         // We hit ErrorLogTimeout. Reporting will stop altogether. Let's wrap things
1353         // up, the process is about to be stopped by the WatcherThread.
1354         st->print_cr("------ Timeout during error reporting after " INT64_FORMAT " s. ------",
1355                      (int64_t)
1356                      ((get_current_timestamp() - _reporting_start_time) / TIMESTAMP_TO_SECONDS_FACTOR));
1357         st->flush();
1358         // Watcherthread is about to call os::die. Lets just wait.
1359         os::infinite_sleep();
1360       } else {
1361         // Crash or assert during error reporting. Lets continue reporting with the next step.
1362         jio_snprintf(buffer, sizeof(buffer),
1363            "[error occurred during error reporting (%s), id 0x%x]",
1364                    _current_step_info, _id);
1365         st->print_raw_cr(buffer);
1366         st->cr();
1367       }
1368     }
1369   }
1370 
1371   // print to screen
1372   if (!out_done) {
1373     report(&out, false);
1374 
1375     out_done = true;
1376 
1377     _current_step = 0;
1378     _current_step_info = "";
1379   }
1380 
1381   // print to error log file
1382   if (!log_done) {
1383     // see if log file is already open
1384     if (!log.is_open()) {
1385       // open log file
1386       int fd = prepare_log_file(ErrorFile, "hs_err_pid%p.log", buffer, sizeof(buffer));
1387       if (fd != -1) {
1388         out.print_raw("# An error report file with more information is saved as:\n# ");
1389         out.print_raw_cr(buffer);
1390 
1391         log.set_fd(fd);
1392       } else {
1393         out.print_raw_cr("# Can not save log file, dump to screen..");
1394         log.set_fd(defaultStream::output_fd());
1395         /* Error reporting currently needs dumpfile.
1396          * Maybe implement direct streaming in the future.*/
1397         transmit_report_done = true;
1398       }
1399     }
1400 
1401     report(&log, true);
1402     log_done = true;
1403     _current_step = 0;
1404     _current_step_info = "";
1405 
1406     // Run error reporting to determine whether or not to report the crash.
1407     if (!transmit_report_done && should_report_bug(_id)) {
1408       transmit_report_done = true;
1409       const int fd2 = ::dup(log.fd());
1410       if (fd2 != -1) {
1411         FILE* const hs_err = ::fdopen(fd2, "r");
1412         if (NULL != hs_err) {
1413           ErrorReporter er;
1414           er.call(hs_err, buffer, O_BUFLEN);
1415           ::fclose(hs_err);
1416         }
1417       }
1418     }
1419 
1420     if (log.fd() != defaultStream::output_fd()) {
1421       close(log.fd());
1422     }
1423 
1424     log.set_fd(-1);
1425   }
1426 
1427   static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
1428   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
1429     skip_replay = true;
1430     ciEnv* env = ciEnv::current();
1431     if (env != NULL) {
1432       int fd = prepare_log_file(ReplayDataFile, "replay_pid%p.log", buffer, sizeof(buffer));
1433       if (fd != -1) {
1434         FILE* replay_data_file = os::open(fd, "w");
1435         if (replay_data_file != NULL) {
1436           fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
1437           env->dump_replay_data_unsafe(&replay_data_stream);
1438           out.print_raw("#\n# Compiler replay data is saved as:\n# ");
1439           out.print_raw_cr(buffer);
1440         } else {
1441           int e = errno;
1442           out.print_raw("#\n# Can't open file to dump replay data. Error: ");
1443           out.print_raw_cr(os::strerror(e));
1444         }
1445       }
1446     }
1447   }
1448 
1449   static bool skip_bug_url = !should_report_bug(_id);
1450   if (!skip_bug_url) {
1451     skip_bug_url = true;
1452 
1453     out.print_raw_cr("#");
1454     print_bug_submit_message(&out, _thread);
1455   }
1456 
1457   static bool skip_OnError = false;
1458   if (!skip_OnError && OnError && OnError[0]) {
1459     skip_OnError = true;
1460 
1461     // Flush output and finish logs before running OnError commands.
1462     ostream_abort();
1463 
1464     out.print_raw_cr("#");
1465     out.print_raw   ("# -XX:OnError=\"");
1466     out.print_raw   (OnError);
1467     out.print_raw_cr("\"");
1468 
1469     char* cmd;
1470     const char* ptr = OnError;
1471     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1472       out.print_raw   ("#   Executing ");
1473 #if defined(LINUX) || defined(_ALLBSD_SOURCE)
1474       out.print_raw   ("/bin/sh -c ");
1475 #elif defined(SOLARIS)
1476       out.print_raw   ("/usr/bin/sh -c ");
1477 #elif defined(WINDOWS)
1478       out.print_raw   ("cmd /C ");
1479 #endif
1480       out.print_raw   ("\"");
1481       out.print_raw   (cmd);
1482       out.print_raw_cr("\" ...");
1483 
1484       if (os::fork_and_exec(cmd) < 0) {
1485         out.print_cr("os::fork_and_exec failed: %s (%s=%d)",
1486                      os::strerror(errno), os::errno_name(errno), errno);
1487       }
1488     }
1489 
1490     // done with OnError
1491     OnError = NULL;
1492   }
1493 
1494   if (!UseOSErrorReporting) {
1495     // os::abort() will call abort hooks, try it first.
1496     static bool skip_os_abort = false;
1497     if (!skip_os_abort) {
1498       skip_os_abort = true;
1499       bool dump_core = should_report_bug(_id);
1500       os::abort(dump_core && CreateCoredumpOnCrash, _siginfo, _context);
1501     }
1502 
1503     // if os::abort() doesn't abort, try os::die();
1504     os::die();
1505   }
1506 }
1507 
1508 /*
1509  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
1510  * ensures utilities such as jmap can observe the process is a consistent state.
1511  */
1512 class VM_ReportJavaOutOfMemory : public VM_Operation {
1513  private:
1514   const char* _message;
1515  public:
1516   VM_ReportJavaOutOfMemory(const char* message) { _message = message; }
1517   VMOp_Type type() const                        { return VMOp_ReportJavaOutOfMemory; }
1518   void doit();
1519 };
1520 
1521 void VM_ReportJavaOutOfMemory::doit() {
1522   // Don't allocate large buffer on stack
1523   static char buffer[O_BUFLEN];
1524 
1525   tty->print_cr("#");
1526   tty->print_cr("# java.lang.OutOfMemoryError: %s", _message);
1527   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
1528 
1529   // make heap parsability
1530   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1531 
1532   char* cmd;
1533   const char* ptr = OnOutOfMemoryError;
1534   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1535     tty->print("#   Executing ");
1536 #if defined(LINUX)
1537     tty->print  ("/bin/sh -c ");
1538 #elif defined(SOLARIS)
1539     tty->print  ("/usr/bin/sh -c ");
1540 #endif
1541     tty->print_cr("\"%s\"...", cmd);
1542 
1543     if (os::fork_and_exec(cmd) < 0) {
1544       tty->print_cr("os::fork_and_exec failed: %s (%s=%d)",
1545                      os::strerror(errno), os::errno_name(errno), errno);
1546     }
1547   }
1548 }
1549 
1550 void VMError::report_java_out_of_memory(const char* message) {
1551   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
1552     MutexLocker ml(Heap_lock);
1553     VM_ReportJavaOutOfMemory op(message);
1554     VMThread::execute(&op);
1555   }
1556 }
1557 
1558 void VMError::show_message_box(char *buf, int buflen) {
1559   bool yes;
1560   do {
1561     error_string(buf, buflen);
1562     yes = os::start_debugging(buf,buflen);
1563   } while (yes);
1564 }
1565 
1566 // Timeout handling: check if a timeout happened (either a single step did
1567 // timeout or the whole of error reporting hit ErrorLogTimeout). Interrupt
1568 // the reporting thread if that is the case.
1569 bool VMError::check_timeout() {
1570 
1571   if (ErrorLogTimeout == 0) {
1572     return false;
1573   }
1574 
1575   // Do not check for timeouts if we still have a message box to show to the
1576   // user or if there are OnError handlers to be run.
1577   if (ShowMessageBoxOnError
1578       || (OnError != NULL && OnError[0] != '\0')
1579       || Arguments::abort_hook() != NULL) {
1580     return false;
1581   }
1582 
1583   const jlong reporting_start_time_l = get_reporting_start_time();
1584   const jlong now = get_current_timestamp();
1585   // Timestamp is stored in nanos.
1586   if (reporting_start_time_l > 0) {
1587     const jlong end = reporting_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR;
1588     if (end <= now) {
1589       _reporting_did_timeout = true;
1590       interrupt_reporting_thread();
1591       return true; // global timeout
1592     }
1593   }
1594 
1595   const jlong step_start_time_l = get_step_start_time();
1596   if (step_start_time_l > 0) {
1597     // A step times out after a quarter of the total timeout. Steps are mostly fast unless they
1598     // hang for some reason, so this simple rule allows for three hanging step and still
1599     // hopefully leaves time enough for the rest of the steps to finish.
1600     const jlong end = step_start_time_l + (jlong)ErrorLogTimeout * TIMESTAMP_TO_SECONDS_FACTOR / 4;
1601     if (end <= now) {
1602       _step_did_timeout = true;
1603       interrupt_reporting_thread();
1604       return false; // (Not a global timeout)
1605     }
1606   }
1607 
1608   return false;
1609 
1610 }
1611 
1612 #ifndef PRODUCT
1613 typedef void (*voidfun_t)();
1614 // Crash with an authentic sigfpe
1615 static void crash_with_sigfpe() {
1616   // generate a native synchronous SIGFPE where possible;
1617   // if that did not cause a signal (e.g. on ppc), just
1618   // raise the signal.
1619   volatile int x = 0;
1620   volatile int y = 1/x;
1621 #ifndef _WIN32
1622   // OSX implements raise(sig) incorrectly so we need to
1623   // explicitly target the current thread
1624   pthread_kill(pthread_self(), SIGFPE);
1625 #endif
1626 } // end: crash_with_sigfpe
1627 
1628 // crash with sigsegv at non-null address.
1629 static void crash_with_segfault() {
1630 
1631   char* const crash_addr = (char*) VMError::get_segfault_address();
1632   *crash_addr = 'X';
1633 
1634 } // end: crash_with_segfault
1635 
1636 void VMError::test_error_handler() {
1637   controlled_crash(ErrorHandlerTest);
1638 }
1639 
1640 // crash in a controlled way:
1641 // how can be one of:
1642 // 1,2 - asserts
1643 // 3,4 - guarantee
1644 // 5-7 - fatal
1645 // 8 - vm_exit_out_of_memory
1646 // 9 - ShouldNotCallThis
1647 // 10 - ShouldNotReachHere
1648 // 11 - Unimplemented
1649 // 12,13 - (not guaranteed) crashes
1650 // 14 - SIGSEGV
1651 // 15 - SIGFPE
1652 void VMError::controlled_crash(int how) {
1653   if (how == 0) return;
1654 
1655   // If asserts are disabled, use the corresponding guarantee instead.
1656   NOT_DEBUG(if (how <= 2) how += 2);
1657 
1658   const char* const str = "hello";
1659   const size_t      num = (size_t)os::vm_page_size();
1660 
1661   const char* const eol = os::line_separator();
1662   const char* const msg = "this message should be truncated during formatting";
1663   char * const dataPtr = NULL;  // bad data pointer
1664   const void (*funcPtr)(void) = (const void(*)()) 0xF;  // bad function pointer
1665 
1666   // Keep this in sync with test/hotspot/jtreg/runtime/ErrorHandling/ErrorHandler.java
1667   // which tests cases 1 thru 13.
1668   // Case 14 is tested by test/hotspot/jtreg/runtime/ErrorHandling/SafeFetchInErrorHandlingTest.java.
1669   // Case 15 is tested by test/hotspot/jtreg/runtime/ErrorHandling/SecondaryErrorTest.java.
1670   // Case 16 is tested by test/hotspot/jtreg/runtime/ErrorHandling/ThreadsListHandleInErrorHandlingTest.java.
1671   // Case 17 is tested by test/hotspot/jtreg/runtime/ErrorHandling/NestedThreadsListHandleInErrorHandlingTest.java.
1672   switch (how) {
1673     case  1: vmassert(str == NULL, "expected null");
1674     case  2: vmassert(num == 1023 && *str == 'X',
1675                       "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1676     case  3: guarantee(str == NULL, "expected null");
1677     case  4: guarantee(num == 1023 && *str == 'X',
1678                        "num=" SIZE_FORMAT " str=\"%s\"", num, str);
1679     case  5: fatal("expected null");
1680     case  6: fatal("num=" SIZE_FORMAT " str=\"%s\"", num, str);
1681     case  7: fatal("%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1682                    "%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
1683                    "%s%s#    %s%s#    %s%s#    %s%s#    %s",
1684                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1685                    msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
1686                    msg, eol, msg, eol, msg, eol, msg, eol, msg);
1687     case  8: vm_exit_out_of_memory(num, OOM_MALLOC_ERROR, "ChunkPool::allocate");
1688     case  9: ShouldNotCallThis();
1689     case 10: ShouldNotReachHere();
1690     case 11: Unimplemented();
1691     // There's no guarantee the bad data pointer will crash us
1692     // so "break" out to the ShouldNotReachHere().
1693     case 12: *dataPtr = '\0'; break;
1694     // There's no guarantee the bad function pointer will crash us
1695     // so "break" out to the ShouldNotReachHere().
1696     case 13: (*funcPtr)(); break;
1697     case 14: crash_with_segfault(); break;
1698     case 15: crash_with_sigfpe(); break;
1699     case 16: {
1700       ThreadsListHandle tlh;
1701       fatal("Force crash with an active ThreadsListHandle.");
1702     }
1703     case 17: {
1704       ThreadsListHandle tlh;
1705       {
1706         ThreadsListHandle tlh2;
1707         fatal("Force crash with a nested ThreadsListHandle.");
1708       }
1709     }
1710 
1711     default: tty->print_cr("ERROR: %d: unexpected test_num value.", how);
1712   }
1713   ShouldNotReachHere();
1714 }
1715 #endif // !PRODUCT
1716