1 /*
   2  * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include <fcntl.h>
  26 #include "precompiled.hpp"
  27 #include "code/codeCache.hpp"
  28 #include "compiler/compileBroker.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "prims/whitebox.hpp"
  31 #include "runtime/arguments.hpp"
  32 #include "runtime/atomic.inline.hpp"
  33 #include "runtime/frame.inline.hpp"
  34 #include "runtime/init.hpp"
  35 #include "runtime/os.hpp"
  36 #include "runtime/thread.inline.hpp"
  37 #include "runtime/vmThread.hpp"
  38 #include "runtime/vm_operations.hpp"
  39 #include "services/memTracker.hpp"
  40 #include "utilities/debug.hpp"
  41 #include "utilities/decoder.hpp"
  42 #include "utilities/defaultStream.hpp"
  43 #include "utilities/errorReporter.hpp"
  44 #include "utilities/events.hpp"
  45 #include "utilities/top.hpp"
  46 #include "utilities/vmError.hpp"
  47 
  48 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
  49 
  50 // List of environment variables that should be reported in error log file.
  51 const char *env_list[] = {
  52   // All platforms
  53   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  54   "JAVA_COMPILER", "PATH", "USERNAME",
  55 
  56   // Env variables that are defined on Solaris/Linux/BSD
  57   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  58   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  59 
  60   // defined on Linux
  61   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  62 
  63   // defined on Darwin
  64   "DYLD_LIBRARY_PATH", "DYLD_FALLBACK_LIBRARY_PATH",
  65   "DYLD_FRAMEWORK_PATH", "DYLD_FALLBACK_FRAMEWORK_PATH",
  66   "DYLD_INSERT_LIBRARIES",
  67 
  68   // defined on Windows
  69   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  70 
  71   (const char *)0
  72 };
  73 
  74 // Fatal error handler for internal errors and crashes.
  75 //
  76 // The default behavior of fatal error handler is to print a brief message
  77 // to standard out (defaultStream::output_fd()), then save detailed information
  78 // into an error report file (hs_err_pid<pid>.log) and abort VM. If multiple
  79 // threads are having troubles at the same time, only one error is reported.
  80 // The thread that is reporting error will abort VM when it is done, all other
  81 // threads are blocked forever inside report_and_die().
  82 
  83 // Constructor for crashes
  84 VMError::VMError(Thread* thread, unsigned int sig, address pc, void* siginfo, void* context) {
  85     _thread = thread;
  86     _id = sig;
  87     _pc   = pc;
  88     _siginfo = siginfo;
  89     _context = context;
  90 
  91     _verbose = false;
  92     _current_step = 0;
  93     _current_step_info = NULL;
  94 
  95     _message = NULL;
  96     _detail_msg = NULL;
  97     _filename = NULL;
  98     _lineno = 0;
  99 
 100     _size = 0;
 101 }
 102 
 103 // Constructor for internal errors
 104 VMError::VMError(Thread* thread, const char* filename, int lineno,
 105                  const char* message, const char * detail_msg)
 106 {
 107   _thread = thread;
 108   _id = INTERNAL_ERROR;     // Value that's not an OS exception/signal
 109   _filename = filename;
 110   _lineno = lineno;
 111   _message = message;
 112   _detail_msg = detail_msg;
 113 
 114   _verbose = false;
 115   _current_step = 0;
 116   _current_step_info = NULL;
 117 
 118   _pc = NULL;
 119   _siginfo = NULL;
 120   _context = NULL;
 121 
 122   _size = 0;
 123 }
 124 
 125 // Constructor for OOM errors
 126 VMError::VMError(Thread* thread, const char* filename, int lineno, size_t size,
 127                  VMErrorType vm_err_type, const char* message) {
 128     _thread = thread;
 129     _id = vm_err_type; // Value that's not an OS exception/signal
 130     _filename = filename;
 131     _lineno = lineno;
 132     _message = message;
 133     _detail_msg = NULL;
 134 
 135     _verbose = false;
 136     _current_step = 0;
 137     _current_step_info = NULL;
 138 
 139     _pc = NULL;
 140     _siginfo = NULL;
 141     _context = NULL;
 142 
 143     _size = size;
 144 }
 145 
 146 
 147 // Constructor for non-fatal errors
 148 VMError::VMError(const char* message) {
 149     _thread = NULL;
 150     _id = INTERNAL_ERROR;     // Value that's not an OS exception/signal
 151     _filename = NULL;
 152     _lineno = 0;
 153     _message = message;
 154     _detail_msg = NULL;
 155 
 156     _verbose = false;
 157     _current_step = 0;
 158     _current_step_info = NULL;
 159 
 160     _pc = NULL;
 161     _siginfo = NULL;
 162     _context = NULL;
 163 
 164     _size = 0;
 165 }
 166 
 167 // -XX:OnError=<string>, where <string> can be a list of commands, separated
 168 // by ';'. "%p" is replaced by current process id (pid); "%%" is replaced by
 169 // a single "%". Some examples:
 170 //
 171 // -XX:OnError="pmap %p"                // show memory map
 172 // -XX:OnError="gcore %p; dbx - %p"     // dump core and launch debugger
 173 // -XX:OnError="cat hs_err_pid%p.log | mail my_email@sun.com"
 174 // -XX:OnError="kill -9 %p"             // ?#!@#
 175 
 176 // A simple parser for -XX:OnError, usage:
 177 //  ptr = OnError;
 178 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
 179 //     ... ...
 180 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
 181   if (ptr == NULL || *ptr == NULL) return NULL;
 182 
 183   const char* cmd = *ptr;
 184 
 185   // skip leading blanks or ';'
 186   while (*cmd == ' ' || *cmd == ';') cmd++;
 187 
 188   if (*cmd == '\0') return NULL;
 189 
 190   const char * cmdend = cmd;
 191   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
 192 
 193   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
 194 
 195   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
 196   return buf;
 197 }
 198 
 199 
 200 static void print_bug_submit_message(outputStream *out, Thread *thread) {
 201   if (out == NULL) return;
 202   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 203   out->print_raw   ("#   ");
 204   out->print_raw_cr(Arguments::java_vendor_url_bug());
 205   // If the crash is in native code, encourage user to submit a bug to the
 206   // provider of that code.
 207   if (thread && thread->is_Java_thread() &&
 208       !thread->is_hidden_from_external_view()) {
 209     JavaThread* jt = (JavaThread*)thread;
 210     if (jt->thread_state() == _thread_in_native) {
 211       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 212     }
 213   }
 214   out->print_raw_cr("#");
 215 }
 216 
 217 bool VMError::coredump_status;
 218 char VMError::coredump_message[O_BUFLEN];
 219 
 220 void VMError::record_coredump_status(const char* message, bool status) {
 221   coredump_status = status;
 222   strncpy(coredump_message, message, sizeof(coredump_message));
 223   coredump_message[sizeof(coredump_message)-1] = 0;
 224 }
 225 
 226 
 227 // Return a string to describe the error
 228 char* VMError::error_string(char* buf, int buflen) {
 229   char signame_buf[64];
 230   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 231 
 232   if (signame) {
 233     jio_snprintf(buf, buflen,
 234                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 235                  signame, _id, _pc,
 236                  os::current_process_id(), os::current_thread_id());
 237   } else if (_filename != NULL && _lineno > 0) {
 238     // skip directory names
 239     char separator = os::file_separator()[0];
 240     const char *p = strrchr(_filename, separator);
 241     int n = jio_snprintf(buf, buflen,
 242                          "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT,
 243                          p ? p + 1 : _filename, _lineno,
 244                          os::current_process_id(), os::current_thread_id());
 245     if (n >= 0 && n < buflen && _message) {
 246       if (_detail_msg) {
 247         jio_snprintf(buf + n, buflen - n, "%s%s: %s",
 248                      os::line_separator(), _message, _detail_msg);
 249       } else {
 250         jio_snprintf(buf + n, buflen - n, "%sError: %s",
 251                      os::line_separator(), _message);
 252       }
 253     }
 254   } else {
 255     jio_snprintf(buf, buflen,
 256                  "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 257                  _id, os::current_process_id(), os::current_thread_id());
 258   }
 259 
 260   return buf;
 261 }
 262 
 263 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 264                                 char* buf, int buflen, bool verbose) {
 265 #ifdef ZERO
 266   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 267     // StackFrameStream uses the frame anchor, which may not have
 268     // been set up.  This can be done at any time in Zero, however,
 269     // so if it hasn't been set up then we just set it up now and
 270     // clear it again when we're done.
 271     bool has_last_Java_frame = jt->has_last_Java_frame();
 272     if (!has_last_Java_frame)
 273       jt->set_last_Java_frame();
 274     st->print("Java frames:");
 275 
 276     // If the top frame is a Shark frame and the frame anchor isn't
 277     // set up then it's possible that the information in the frame
 278     // is garbage: it could be from a previous decache, or it could
 279     // simply have never been written.  So we print a warning...
 280     StackFrameStream sfs(jt);
 281     if (!has_last_Java_frame && !sfs.is_done()) {
 282       if (sfs.current()->zeroframe()->is_shark_frame()) {
 283         st->print(" (TOP FRAME MAY BE JUNK)");
 284       }
 285     }
 286     st->cr();
 287 
 288     // Print the frames
 289     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 290       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 291       st->cr();
 292     }
 293 
 294     // Reset the frame anchor if necessary
 295     if (!has_last_Java_frame)
 296       jt->reset_last_Java_frame();
 297   }
 298 #else
 299   if (jt->has_last_Java_frame()) {
 300     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 301     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 302       sfs.current()->print_on_error(st, buf, buflen, verbose);
 303       st->cr();
 304     }
 305   }
 306 #endif // ZERO
 307 }
 308 
 309 void VMError::print_oom_reasons(outputStream* st) {
 310   st->print_cr("# Possible reasons:");
 311   st->print_cr("#   The system is out of physical RAM or swap space");
 312   st->print_cr("#   In 32 bit mode, the process size limit was hit");
 313   st->print_cr("# Possible solutions:");
 314   st->print_cr("#   Reduce memory load on the system");
 315   st->print_cr("#   Increase physical memory or swap space");
 316   st->print_cr("#   Check if swap backing store is full");
 317   st->print_cr("#   Use 64 bit Java on a 64 bit OS");
 318   st->print_cr("#   Decrease Java heap size (-Xmx/-Xms)");
 319   st->print_cr("#   Decrease number of Java threads");
 320   st->print_cr("#   Decrease Java thread stack sizes (-Xss)");
 321   st->print_cr("#   Set larger code cache with -XX:ReservedCodeCacheSize=");
 322   st->print_cr("# This output file may be truncated or incomplete.");
 323 }
 324 
 325 const char* VMError::gc_mode() {
 326   if (UseG1GC)            return "g1 gc";
 327   if (UseParallelGC)      return "parallel gc";
 328   if (UseConcMarkSweepGC) return "concurrent mark sweep gc";
 329   if (UseSerialGC)        return "serial gc";
 330   return "ERROR in GC mode";
 331 }
 332 
 333 // This is the main function to report a fatal error. Only one thread can
 334 // call this function, so we don't need to worry about MT-safety. But it's
 335 // possible that the error handler itself may crash or die on an internal
 336 // error, for example, when the stack/heap is badly damaged. We must be
 337 // able to handle recursive errors that happen inside error handler.
 338 //
 339 // Error reporting is done in several steps. If a crash or internal error
 340 // occurred when reporting an error, the nested signal/exception handler
 341 // can skip steps that are already (or partially) done. Error reporting will
 342 // continue from the next step. This allows us to retrieve and print
 343 // information that may be unsafe to get after a fatal error. If it happens,
 344 // you may find nested report_and_die() frames when you look at the stack
 345 // in a debugger.
 346 //
 347 // In general, a hang in error handler is much worse than a crash or internal
 348 // error, as it's harder to recover from a hang. Deadlock can happen if we
 349 // try to grab a lock that is already owned by current thread, or if the
 350 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 351 // error handler and all the functions it called should avoid grabbing any
 352 // lock. An important thing to notice is that memory allocation needs a lock.
 353 //
 354 // We should avoid using large stack allocated buffers. Many errors happen
 355 // when stack space is already low. Making things even worse is that there
 356 // could be nested report_and_die() calls on stack (see above). Only one
 357 // thread can report error, so large buffers are statically allocated in data
 358 // segment.
 359 
 360 void VMError::report(outputStream* st) {
 361 # define BEGIN if (_current_step == 0) { _current_step = 1;
 362 # define STEP(n, s) } if (_current_step < n) { _current_step = n; _current_step_info = s;
 363 # define END }
 364 
 365   // don't allocate large buffer on stack
 366   static char buf[O_BUFLEN];
 367 
 368   BEGIN
 369 
 370   STEP(10, "(printing fatal error message)")
 371 
 372     st->print_cr("#");
 373     if (should_report_bug(_id)) {
 374       st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 375     } else {
 376       st->print_cr("# There is insufficient memory for the Java "
 377                    "Runtime Environment to continue.");
 378     }
 379 
 380 #ifndef PRODUCT
 381   // Error handler self tests
 382 
 383   // test secondary error handling. Test it twice, to test that resetting
 384   // error handler after a secondary crash works.
 385   STEP(20, "(test secondary crash 1)")
 386     if (_verbose && TestCrashInErrorHandler != 0) {
 387       st->print_cr("Will crash now (TestCrashInErrorHandler=%d)...",
 388         TestCrashInErrorHandler);
 389       controlled_crash(TestCrashInErrorHandler);
 390     }
 391 
 392   STEP(30, "(test secondary crash 2)")
 393     if (_verbose && TestCrashInErrorHandler != 0) {
 394       st->print_cr("Will crash now (TestCrashInErrorHandler=%d)...",
 395         TestCrashInErrorHandler);
 396       controlled_crash(TestCrashInErrorHandler);
 397     }
 398 
 399   STEP(40, "(test safefetch in error handler)")
 400     // test whether it is safe to use SafeFetch32 in Crash Handler. Test twice
 401     // to test that resetting the signal handler works correctly.
 402     if (_verbose && TestSafeFetchInErrorHandler) {
 403       st->print_cr("Will test SafeFetch...");
 404       if (CanUseSafeFetch32()) {
 405         int* const invalid_pointer = (int*) get_segfault_address();
 406         const int x = 0x76543210;
 407         int i1 = SafeFetch32(invalid_pointer, x);
 408         int i2 = SafeFetch32(invalid_pointer, x);
 409         if (i1 == x && i2 == x) {
 410           st->print_cr("SafeFetch OK."); // Correctly deflected and returned default pattern
 411         } else {
 412           st->print_cr("??");
 413         }
 414       } else {
 415         st->print_cr("not possible; skipped.");
 416       }
 417     }
 418 #endif // PRODUCT
 419 
 420   STEP(50, "(printing type of error)")
 421 
 422      switch(_id) {
 423        case OOM_MALLOC_ERROR:
 424        case OOM_MMAP_ERROR:
 425          if (_size) {
 426            st->print("# Native memory allocation ");
 427            st->print((_id == (int)OOM_MALLOC_ERROR) ? "(malloc) failed to allocate " :
 428                                                  "(mmap) failed to map ");
 429            jio_snprintf(buf, sizeof(buf), SIZE_FORMAT, _size);
 430            st->print("%s", buf);
 431            st->print(" bytes");
 432            if (_message != NULL) {
 433              st->print(" for ");
 434              st->print("%s", _message);
 435            }
 436            st->cr();
 437          } else {
 438            if (_message != NULL) {
 439              st->print("# ");
 440              st->print_cr("%s", _message);
 441            }
 442          }
 443          // In error file give some solutions
 444          if (_verbose) {
 445            print_oom_reasons(st);
 446          } else {
 447            return;  // that's enough for the screen
 448          }
 449          break;
 450        case INTERNAL_ERROR:
 451        default:
 452          break;
 453      }
 454 
 455   STEP(60, "(printing exception/signal name)")
 456 
 457      st->print_cr("#");
 458      st->print("#  ");
 459      // Is it an OS exception/signal?
 460      if (os::exception_name(_id, buf, sizeof(buf))) {
 461        st->print("%s", buf);
 462        st->print(" (0x%x)", _id);                // signal number
 463        st->print(" at pc=" PTR_FORMAT, _pc);
 464      } else {
 465        if (should_report_bug(_id)) {
 466          st->print("Internal Error");
 467        } else {
 468          st->print("Out of Memory Error");
 469        }
 470        if (_filename != NULL && _lineno > 0) {
 471 #ifdef PRODUCT
 472          // In product mode chop off pathname?
 473          char separator = os::file_separator()[0];
 474          const char *p = strrchr(_filename, separator);
 475          const char *file = p ? p+1 : _filename;
 476 #else
 477          const char *file = _filename;
 478 #endif
 479          st->print(" (%s:%d)", file, _lineno);
 480        } else {
 481          st->print(" (0x%x)", _id);
 482        }
 483      }
 484 
 485   STEP(70, "(printing current thread and pid)")
 486 
 487      // process id, thread id
 488      st->print(", pid=%d", os::current_process_id());
 489      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 490      st->cr();
 491 
 492   STEP(80, "(printing error message)")
 493 
 494      if (should_report_bug(_id)) {  // already printed the message.
 495        // error message
 496        if (_detail_msg) {
 497          st->print_cr("#  %s: %s", _message ? _message : "Error", _detail_msg);
 498        } else if (_message) {
 499          st->print_cr("#  Error: %s", _message);
 500        }
 501     }
 502 
 503   STEP(90, "(printing Java version string)")
 504 
 505      // VM version
 506      st->print_cr("#");
 507      JDK_Version::current().to_string(buf, sizeof(buf));
 508      const char* runtime_name = JDK_Version::runtime_name() != NULL ?
 509                                   JDK_Version::runtime_name() : "";
 510      const char* runtime_version = JDK_Version::runtime_version() != NULL ?
 511                                   JDK_Version::runtime_version() : "";
 512      st->print_cr("# JRE version: %s (%s) (build %s)", runtime_name, buf, runtime_version);
 513      // This is the long version with some default settings added
 514      st->print_cr("# Java VM: %s (%s, %s%s%s, %s, %s)",
 515                    Abstract_VM_Version::vm_name(),
 516                    Abstract_VM_Version::vm_release(),
 517                    Abstract_VM_Version::vm_info_string(),
 518                    TieredCompilation ? ", tiered" : "",
 519                    UseCompressedOops ? ", compressed oops" : "",
 520                    gc_mode(),
 521                    Abstract_VM_Version::vm_platform_string()
 522                  );
 523 
 524   STEP(100, "(printing problematic frame)")
 525 
 526      // Print current frame if we have a context (i.e. it's a crash)
 527      if (_context) {
 528        st->print_cr("# Problematic frame:");
 529        st->print("# ");
 530        frame fr = os::fetch_frame_from_context(_context);
 531        fr.print_on_error(st, buf, sizeof(buf));
 532        st->cr();
 533        st->print_cr("#");
 534      }
 535 
 536   STEP(110, "(printing core file information)")
 537     st->print("# ");
 538     if (CreateCoredumpOnCrash) {
 539       if (coredump_status) {
 540         st->print("Core dump will be written. Default location: %s", coredump_message);
 541       } else {
 542         st->print("No core dump will be written. %s", coredump_message);
 543       }
 544     } else {
 545       st->print("CreateCoredumpOnCrash turned off, no core file dumped");
 546     }
 547     st->cr();
 548     st->print_cr("#");
 549 
 550   STEP(120, "(printing bug submit message)")
 551 
 552      if (should_report_bug(_id) && _verbose) {
 553        print_bug_submit_message(st, _thread);
 554      }
 555 
 556   STEP(130, "(printing summary)" )
 557 
 558      if (_verbose) {
 559        st->cr();
 560        st->print_cr("---------------  S U M M A R Y ------------");
 561        st->cr();
 562      }
 563 
 564   STEP(140, "(printing VM option summary)" )
 565 
 566      if (_verbose) {
 567        // VM options
 568        Arguments::print_summary_on(st);
 569        st->cr();
 570      }
 571 
 572   STEP(150, "(printing summary machine and OS info)")
 573 
 574      if (_verbose) {
 575        os::print_summary_info(st, buf, sizeof(buf));
 576      }
 577 
 578 
 579   STEP(160, "(printing date and time)" )
 580 
 581      if (_verbose) {
 582        os::print_date_and_time(st, buf, sizeof(buf));
 583      }
 584 
 585   STEP(170, "(printing thread)" )
 586 
 587      if (_verbose) {
 588        st->cr();
 589        st->print_cr("---------------  T H R E A D  ---------------");
 590        st->cr();
 591      }
 592 
 593   STEP(180, "(printing current thread)" )
 594 
 595      // current thread
 596      if (_verbose) {
 597        if (_thread) {
 598          st->print("Current thread (" PTR_FORMAT "):  ", _thread);
 599          _thread->print_on_error(st, buf, sizeof(buf));
 600          st->cr();
 601        } else {
 602          st->print_cr("Current thread is native thread");
 603        }
 604        st->cr();
 605      }
 606 
 607   STEP(190, "(printing current compile task)" )
 608 
 609      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 610         CompilerThread* t = (CompilerThread*)_thread;
 611         if (t->task()) {
 612            st->cr();
 613            st->print_cr("Current CompileTask:");
 614            t->task()->print_line_on_error(st, buf, sizeof(buf));
 615            st->cr();
 616         }
 617      }
 618 
 619 
 620   STEP(200, "(printing stack bounds)" )
 621 
 622      if (_verbose) {
 623        st->print("Stack: ");
 624 
 625        address stack_top;
 626        size_t stack_size;
 627 
 628        if (_thread) {
 629           stack_top = _thread->stack_base();
 630           stack_size = _thread->stack_size();
 631        } else {
 632           stack_top = os::current_stack_base();
 633           stack_size = os::current_stack_size();
 634        }
 635 
 636        address stack_bottom = stack_top - stack_size;
 637        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", stack_bottom, stack_top);
 638 
 639        frame fr = _context ? os::fetch_frame_from_context(_context)
 640                            : os::current_frame();
 641 
 642        if (fr.sp()) {
 643          st->print(",  sp=" PTR_FORMAT, fr.sp());
 644          size_t free_stack_size = pointer_delta(fr.sp(), stack_bottom, 1024);
 645          st->print(",  free space=" SIZE_FORMAT "k", free_stack_size);
 646        }
 647 
 648        st->cr();
 649      }
 650 
 651   STEP(210, "(printing native stack)" )
 652 
 653    if (_verbose) {
 654      if (os::platform_print_native_stack(st, _context, buf, sizeof(buf))) {
 655        // We have printed the native stack in platform-specific code
 656        // Windows/x64 needs special handling.
 657      } else {
 658        frame fr = _context ? os::fetch_frame_from_context(_context)
 659                            : os::current_frame();
 660 
 661        print_native_stack(st, fr, _thread, buf, sizeof(buf));
 662      }
 663    }
 664 
 665   STEP(220, "(printing Java stack)" )
 666 
 667      if (_verbose && _thread && _thread->is_Java_thread()) {
 668        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 669      }
 670 
 671   STEP(230, "(printing target Java thread stack)" )
 672 
 673      // printing Java thread stack trace if it is involved in GC crash
 674      if (_verbose && _thread && (_thread->is_Named_thread())) {
 675        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 676        if (jt != NULL) {
 677          st->print_cr("JavaThread " PTR_FORMAT " (nid = " UINTX_FORMAT ") was being processed", jt, jt->osthread()->thread_id());
 678          print_stack_trace(st, jt, buf, sizeof(buf), true);
 679        }
 680      }
 681 
 682   STEP(240, "(printing siginfo)" )
 683 
 684      // signal no, signal code, address that caused the fault
 685      if (_verbose && _siginfo) {
 686        st->cr();
 687        os::print_siginfo(st, _siginfo);
 688        st->cr();
 689      }
 690 
 691   STEP(250, "(printing register info)")
 692 
 693      // decode register contents if possible
 694      if (_verbose && _context && Universe::is_fully_initialized()) {
 695        os::print_register_info(st, _context);
 696        st->cr();
 697      }
 698 
 699   STEP(260, "(printing registers, top of stack, instructions near pc)")
 700 
 701      // registers, top of stack, instructions near pc
 702      if (_verbose && _context) {
 703        os::print_context(st, _context);
 704        st->cr();
 705      }
 706 
 707   STEP(270, "(printing VM operation)" )
 708 
 709      if (_verbose && _thread && _thread->is_VM_thread()) {
 710         VMThread* t = (VMThread*)_thread;
 711         VM_Operation* op = t->vm_operation();
 712         if (op) {
 713           op->print_on_error(st);
 714           st->cr();
 715           st->cr();
 716         }
 717      }
 718 
 719   STEP(280, "(printing process)" )
 720 
 721      if (_verbose) {
 722        st->cr();
 723        st->print_cr("---------------  P R O C E S S  ---------------");
 724        st->cr();
 725      }
 726 
 727   STEP(290, "(printing all threads)" )
 728 
 729      // all threads
 730      if (_verbose && _thread) {
 731        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 732        st->cr();
 733      }
 734 
 735   STEP(300, "(printing VM state)" )
 736 
 737      if (_verbose) {
 738        // Safepoint state
 739        st->print("VM state:");
 740 
 741        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 742        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 743        else st->print("not at safepoint");
 744 
 745        // Also see if error occurred during initialization or shutdown
 746        if (!Universe::is_fully_initialized()) {
 747          st->print(" (not fully initialized)");
 748        } else if (VM_Exit::vm_exited()) {
 749          st->print(" (shutting down)");
 750        } else {
 751          st->print(" (normal execution)");
 752        }
 753        st->cr();
 754        st->cr();
 755      }
 756 
 757   STEP(310, "(printing owned locks on error)" )
 758 
 759      // mutexes/monitors that currently have an owner
 760      if (_verbose) {
 761        print_owned_locks_on_error(st);
 762        st->cr();
 763      }
 764 
 765   STEP(320, "(printing number of OutOfMemoryError and StackOverflow exceptions)")
 766 
 767      if (_verbose && Exceptions::has_exception_counts()) {
 768        st->print_cr("OutOfMemory and StackOverflow Exception counts:");
 769        Exceptions::print_exception_counts_on_error(st);
 770        st->cr();
 771      }
 772 
 773   STEP(330, "(printing compressed oops mode")
 774 
 775      if (_verbose && UseCompressedOops) {
 776        Universe::print_compressed_oops_mode(st);
 777        if (UseCompressedClassPointers) {
 778          Metaspace::print_compressed_class_space(st);
 779        }
 780        st->cr();
 781      }
 782 
 783   STEP(340, "(printing heap information)" )
 784 
 785      if (_verbose && Universe::is_fully_initialized()) {
 786        Universe::heap()->print_on_error(st);
 787        st->cr();
 788 
 789        st->print_cr("Polling page: " INTPTR_FORMAT, os::get_polling_page());
 790        st->cr();
 791      }
 792 
 793   STEP(350, "(printing code cache information)" )
 794 
 795      if (_verbose && Universe::is_fully_initialized()) {
 796        // print code cache information before vm abort
 797        CodeCache::print_summary(st);
 798        st->cr();
 799      }
 800 
 801   STEP(360, "(printing ring buffers)" )
 802 
 803      if (_verbose) {
 804        Events::print_all(st);
 805        st->cr();
 806      }
 807 
 808   STEP(370, "(printing dynamic libraries)" )
 809 
 810      if (_verbose) {
 811        // dynamic libraries, or memory map
 812        os::print_dll_info(st);
 813        st->cr();
 814      }
 815 
 816   STEP(380, "(printing VM options)" )
 817 
 818      if (_verbose) {
 819        // VM options
 820        Arguments::print_on(st);
 821        st->cr();
 822      }
 823 
 824   STEP(390, "(printing warning if internal testing API used)" )
 825 
 826      if (WhiteBox::used()) {
 827        st->print_cr("Unsupported internal testing APIs have been used.");
 828        st->cr();
 829      }
 830 
 831   STEP(400, "(printing all environment variables)" )
 832 
 833      if (_verbose) {
 834        os::print_environment_variables(st, env_list);
 835        st->cr();
 836      }
 837 
 838   STEP(410, "(printing signal handlers)" )
 839 
 840      if (_verbose) {
 841        os::print_signal_handlers(st, buf, sizeof(buf));
 842        st->cr();
 843      }
 844 
 845   STEP(420, "(Native Memory Tracking)" )
 846      if (_verbose) {
 847        MemTracker::error_report(st);
 848      }
 849 
 850   STEP(430, "(printing system)" )
 851 
 852      if (_verbose) {
 853        st->cr();
 854        st->print_cr("---------------  S Y S T E M  ---------------");
 855        st->cr();
 856      }
 857 
 858   STEP(440, "(printing OS information)" )
 859 
 860      if (_verbose) {
 861        os::print_os_info(st);
 862        st->cr();
 863      }
 864 
 865   STEP(450, "(printing CPU info)" )
 866      if (_verbose) {
 867        os::print_cpu_info(st, buf, sizeof(buf));
 868        st->cr();
 869      }
 870 
 871   STEP(460, "(printing memory info)" )
 872 
 873      if (_verbose) {
 874        os::print_memory_info(st);
 875        st->cr();
 876      }
 877 
 878   STEP(470, "(printing internal vm info)" )
 879 
 880      if (_verbose) {
 881        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 882        st->cr();
 883      }
 884 
 885   // print a defined marker to show that error handling finished correctly.
 886   STEP(480, "(printing end marker)" )
 887 
 888      if (_verbose) {
 889        st->print_cr("END.");
 890      }
 891 
 892   END
 893 
 894 # undef BEGIN
 895 # undef STEP
 896 # undef END
 897 }
 898 
 899 VMError* volatile VMError::first_error = NULL;
 900 volatile jlong VMError::first_error_tid = -1;
 901 
 902 // An error could happen before tty is initialized or after it has been
 903 // destroyed. Here we use a very simple unbuffered fdStream for printing.
 904 // Only out.print_raw() and out.print_raw_cr() should be used, as other
 905 // printing methods need to allocate large buffer on stack. To format a
 906 // string, use jio_snprintf() with a static buffer or use staticBufferStream.
 907 fdStream VMError::out(defaultStream::output_fd());
 908 fdStream VMError::log; // error log used by VMError::report_and_die()
 909 
 910 /** Expand a pattern into a buffer starting at pos and open a file using constructed path */
 911 static int expand_and_open(const char* pattern, char* buf, size_t buflen, size_t pos) {
 912   int fd = -1;
 913   if (Arguments::copy_expand_pid(pattern, strlen(pattern), &buf[pos], buflen - pos)) {
 914     // the O_EXCL flag will cause the open to fail if the file exists
 915     fd = open(buf, O_RDWR | O_CREAT | O_EXCL, 0666);
 916   }
 917   return fd;
 918 }
 919 
 920 /**
 921  * Construct file name for a log file and return it's file descriptor.
 922  * Name and location depends on pattern, default_pattern params and access
 923  * permissions.
 924  */
 925 static int prepare_log_file(const char* pattern, const char* default_pattern, char* buf, size_t buflen) {
 926   int fd = -1;
 927 
 928   // If possible, use specified pattern to construct log file name
 929   if (pattern != NULL) {
 930     fd = expand_and_open(pattern, buf, buflen, 0);
 931   }
 932 
 933   // Either user didn't specify, or the user's location failed,
 934   // so use the default name in the current directory
 935   if (fd == -1) {
 936     const char* cwd = os::get_current_directory(buf, buflen);
 937     if (cwd != NULL) {
 938       size_t pos = strlen(cwd);
 939       int fsep_len = jio_snprintf(&buf[pos], buflen-pos, "%s", os::file_separator());
 940       pos += fsep_len;
 941       if (fsep_len > 0) {
 942         fd = expand_and_open(default_pattern, buf, buflen, pos);
 943       }
 944     }
 945   }
 946 
 947    // try temp directory if it exists.
 948    if (fd == -1) {
 949      const char* tmpdir = os::get_temp_directory();
 950      if (tmpdir != NULL && strlen(tmpdir) > 0) {
 951        int pos = jio_snprintf(buf, buflen, "%s%s", tmpdir, os::file_separator());
 952        if (pos > 0) {
 953          fd = expand_and_open(default_pattern, buf, buflen, pos);
 954        }
 955      }
 956    }
 957 
 958   return fd;
 959 }
 960 
 961 void VMError::report_and_die() {
 962   // Don't allocate large buffer on stack
 963   static char buffer[O_BUFLEN];
 964 
 965   // How many errors occurred in error handler when reporting first_error.
 966   static int recursive_error_count;
 967 
 968   // We will first print a brief message to standard out (verbose = false),
 969   // then save detailed information in log file (verbose = true).
 970   static bool out_done = false;         // done printing to standard out
 971   static bool log_done = false;         // done saving error log
 972   static bool transmit_report_done = false; // done error reporting
 973 
 974   if (SuppressFatalErrorMessage) {
 975       os::abort(CreateCoredumpOnCrash);
 976   }
 977   jlong mytid = os::current_thread_id();
 978   if (first_error == NULL &&
 979       Atomic::cmpxchg_ptr(this, &first_error, NULL) == NULL) {
 980 
 981     // first time
 982     first_error_tid = mytid;
 983     set_error_reported();
 984 
 985     if (ShowMessageBoxOnError || PauseAtExit) {
 986       show_message_box(buffer, sizeof(buffer));
 987 
 988       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
 989       // WatcherThread can kill JVM if the error handler hangs.
 990       ShowMessageBoxOnError = false;
 991     }
 992 
 993     os::check_dump_limit(buffer, sizeof(buffer));
 994 
 995     // reset signal handlers or exception filter; make sure recursive crashes
 996     // are handled properly.
 997     reset_signal_handlers();
 998 
 999   } else {
1000     // If UseOsErrorReporting we call this for each level of the call stack
1001     // while searching for the exception handler.  Only the first level needs
1002     // to be reported.
1003     if (UseOSErrorReporting && log_done) return;
1004 
1005     // This is not the first error, see if it happened in a different thread
1006     // or in the same thread during error reporting.
1007     if (first_error_tid != mytid) {
1008       char msgbuf[64];
1009       jio_snprintf(msgbuf, sizeof(msgbuf),
1010                    "[thread " INT64_FORMAT " also had an error]",
1011                    mytid);
1012       out.print_raw_cr(msgbuf);
1013 
1014       // error reporting is not MT-safe, block current thread
1015       os::infinite_sleep();
1016 
1017     } else {
1018       if (recursive_error_count++ > 30) {
1019         out.print_raw_cr("[Too many errors, abort]");
1020         os::die();
1021       }
1022 
1023       jio_snprintf(buffer, sizeof(buffer),
1024                    "[error occurred during error reporting %s, id 0x%x]",
1025                    first_error ? first_error->_current_step_info : "",
1026                    _id);
1027       if (log.is_open()) {
1028         log.cr();
1029         log.print_raw_cr(buffer);
1030         log.cr();
1031       } else {
1032         out.cr();
1033         out.print_raw_cr(buffer);
1034         out.cr();
1035       }
1036     }
1037   }
1038 
1039   // print to screen
1040   if (!out_done) {
1041     first_error->_verbose = false;
1042 
1043     staticBufferStream sbs(buffer, sizeof(buffer), &out);
1044     first_error->report(&sbs);
1045 
1046     out_done = true;
1047 
1048     first_error->_current_step = 0;         // reset current_step
1049     first_error->_current_step_info = "";   // reset current_step string
1050   }
1051 
1052   // print to error log file
1053   if (!log_done) {
1054     first_error->_verbose = true;
1055 
1056     // see if log file is already open
1057     if (!log.is_open()) {
1058       // open log file
1059       int fd = prepare_log_file(ErrorFile, "hs_err_pid%p.log", buffer, sizeof(buffer));
1060       if (fd != -1) {
1061         out.print_raw("# An error report file with more information is saved as:\n# ");
1062         out.print_raw_cr(buffer);
1063 
1064         log.set_fd(fd);
1065       } else {
1066         out.print_raw_cr("# Can not save log file, dump to screen..");
1067         log.set_fd(defaultStream::output_fd());
1068         /* Error reporting currently needs dumpfile.
1069          * Maybe implement direct streaming in the future.*/
1070         transmit_report_done = true;
1071       }
1072     }
1073 
1074     staticBufferStream sbs(buffer, O_BUFLEN, &log);
1075     first_error->report(&sbs);
1076     first_error->_current_step = 0;         // reset current_step
1077     first_error->_current_step_info = "";   // reset current_step string
1078 
1079     // Run error reporting to determine whether or not to report the crash.
1080     if (!transmit_report_done && should_report_bug(first_error->_id)) {
1081       transmit_report_done = true;
1082       const int fd2 = ::dup(log.fd());
1083       FILE* const hs_err = ::fdopen(fd2, "r");
1084       if (NULL != hs_err) {
1085         ErrorReporter er;
1086         er.call(hs_err, buffer, O_BUFLEN);
1087       }
1088       ::fclose(hs_err);
1089     }
1090 
1091     if (log.fd() != defaultStream::output_fd()) {
1092       close(log.fd());
1093     }
1094 
1095     log.set_fd(-1);
1096     log_done = true;
1097   }
1098 
1099 
1100   static bool skip_OnError = false;
1101   if (!skip_OnError && OnError && OnError[0]) {
1102     skip_OnError = true;
1103 
1104     out.print_raw_cr("#");
1105     out.print_raw   ("# -XX:OnError=\"");
1106     out.print_raw   (OnError);
1107     out.print_raw_cr("\"");
1108 
1109     char* cmd;
1110     const char* ptr = OnError;
1111     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1112       out.print_raw   ("#   Executing ");
1113 #if defined(LINUX) || defined(_ALLBSD_SOURCE)
1114       out.print_raw   ("/bin/sh -c ");
1115 #elif defined(SOLARIS)
1116       out.print_raw   ("/usr/bin/sh -c ");
1117 #endif
1118       out.print_raw   ("\"");
1119       out.print_raw   (cmd);
1120       out.print_raw_cr("\" ...");
1121 
1122       if (os::fork_and_exec(cmd) < 0) {
1123         out.print_cr("os::fork_and_exec failed: %s (%d)", strerror(errno), errno);
1124       }
1125     }
1126 
1127     // done with OnError
1128     OnError = NULL;
1129   }
1130 
1131   static bool skip_replay = ReplayCompiles; // Do not overwrite file during replay
1132   if (DumpReplayDataOnError && _thread && _thread->is_Compiler_thread() && !skip_replay) {
1133     skip_replay = true;
1134     ciEnv* env = ciEnv::current();
1135     if (env != NULL) {
1136       int fd = prepare_log_file(ReplayDataFile, "replay_pid%p.log", buffer, sizeof(buffer));
1137       if (fd != -1) {
1138         FILE* replay_data_file = os::open(fd, "w");
1139         if (replay_data_file != NULL) {
1140           fileStream replay_data_stream(replay_data_file, /*need_close=*/true);
1141           env->dump_replay_data_unsafe(&replay_data_stream);
1142           out.print_raw("#\n# Compiler replay data is saved as:\n# ");
1143           out.print_raw_cr(buffer);
1144         } else {
1145           out.print_raw("#\n# Can't open file to dump replay data. Error: ");
1146           out.print_raw_cr(strerror(os::get_last_error()));
1147         }
1148       }
1149     }
1150   }
1151 
1152   static bool skip_bug_url = !should_report_bug(first_error->_id);
1153   if (!skip_bug_url) {
1154     skip_bug_url = true;
1155 
1156     out.print_raw_cr("#");
1157     print_bug_submit_message(&out, _thread);
1158   }
1159 
1160   if (!UseOSErrorReporting) {
1161     // os::abort() will call abort hooks, try it first.
1162     static bool skip_os_abort = false;
1163     if (!skip_os_abort) {
1164       skip_os_abort = true;
1165       bool dump_core = should_report_bug(first_error->_id);
1166       os::abort(dump_core && CreateCoredumpOnCrash, _siginfo, _context);
1167     }
1168 
1169     // if os::abort() doesn't abort, try os::die();
1170     os::die();
1171   }
1172 }
1173 
1174 /*
1175  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
1176  * ensures utilities such as jmap can observe the process is a consistent state.
1177  */
1178 class VM_ReportJavaOutOfMemory : public VM_Operation {
1179  private:
1180   VMError *_err;
1181  public:
1182   VM_ReportJavaOutOfMemory(VMError *err) { _err = err; }
1183   VMOp_Type type() const                 { return VMOp_ReportJavaOutOfMemory; }
1184   void doit();
1185 };
1186 
1187 void VM_ReportJavaOutOfMemory::doit() {
1188   // Don't allocate large buffer on stack
1189   static char buffer[O_BUFLEN];
1190 
1191   tty->print_cr("#");
1192   tty->print_cr("# java.lang.OutOfMemoryError: %s", _err->message());
1193   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
1194 
1195   // make heap parsability
1196   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
1197 
1198   char* cmd;
1199   const char* ptr = OnOutOfMemoryError;
1200   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
1201     tty->print("#   Executing ");
1202 #if defined(LINUX)
1203     tty->print  ("/bin/sh -c ");
1204 #elif defined(SOLARIS)
1205     tty->print  ("/usr/bin/sh -c ");
1206 #endif
1207     tty->print_cr("\"%s\"...", cmd);
1208 
1209     if (os::fork_and_exec(cmd) < 0) {
1210       tty->print_cr("os::fork_and_exec failed: %s (%d)", strerror(errno), errno);
1211     }
1212   }
1213 }
1214 
1215 void VMError::report_java_out_of_memory() {
1216   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
1217     MutexLocker ml(Heap_lock);
1218     VM_ReportJavaOutOfMemory op(this);
1219     VMThread::execute(&op);
1220   }
1221 }