1 /*
   2  * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  20  * CA 95054 USA or visit www.sun.com if you need additional information or
  21  * have any questions.
  22  *
  23  */
  24 
  25 # include "incls/_precompiled.incl"
  26 # include "incls/_vmError.cpp.incl"
  27 
  28 // List of environment variables that should be reported in error log file.
  29 const char *env_list[] = {
  30   // All platforms
  31   "JAVA_HOME", "JRE_HOME", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "CLASSPATH",
  32   "JAVA_COMPILER", "PATH", "USERNAME",
  33 
  34   // Env variables that are defined on Solaris/Linux
  35   "LD_LIBRARY_PATH", "LD_PRELOAD", "SHELL", "DISPLAY",
  36   "HOSTTYPE", "OSTYPE", "ARCH", "MACHTYPE",
  37 
  38   // defined on Linux
  39   "LD_ASSUME_KERNEL", "_JAVA_SR_SIGNUM",
  40 
  41   // defined on Windows
  42   "OS", "PROCESSOR_IDENTIFIER", "_ALT_JAVA_HOME_DIR",
  43 
  44   (const char *)0
  45 };
  46 
  47 // Fatal error handler for internal errors and crashes.
  48 //
  49 // The default behavior of fatal error handler is to print a brief message
  50 // to standard out (defaultStream::output_fd()), then save detailed information
  51 // into an error report file (hs_err_pid<pid>.log) and abort VM. If multiple
  52 // threads are having troubles at the same time, only one error is reported.
  53 // The thread that is reporting error will abort VM when it is done, all other
  54 // threads are blocked forever inside report_and_die().
  55 
  56 // Constructor for crashes
  57 VMError::VMError(Thread* thread, int sig, address pc, void* siginfo, void* context) {
  58     _thread = thread;
  59     _id = sig;
  60     _pc   = pc;
  61     _siginfo = siginfo;
  62     _context = context;
  63 
  64     _verbose = false;
  65     _current_step = 0;
  66     _current_step_info = NULL;
  67 
  68     _message = "";
  69     _filename = NULL;
  70     _lineno = 0;
  71 
  72     _size = 0;
  73 }
  74 
  75 // Constructor for internal errors
  76 VMError::VMError(Thread* thread, const char* message, const char* filename, int lineno) {
  77     _thread = thread;
  78     _id = internal_error;     // set it to a value that's not an OS exception/signal
  79     _filename = filename;
  80     _lineno = lineno;
  81     _message = message;
  82 
  83     _verbose = false;
  84     _current_step = 0;
  85     _current_step_info = NULL;
  86 
  87     _pc = NULL;
  88     _siginfo = NULL;
  89     _context = NULL;
  90 
  91     _size = 0;
  92 }
  93 
  94 // Constructor for OOM errors
  95 VMError::VMError(Thread* thread, size_t size, const char* message, const char* filename, int lineno) {
  96     _thread = thread;
  97     _id = oom_error;     // set it to a value that's not an OS exception/signal
  98     _filename = filename;
  99     _lineno = lineno;
 100     _message = message;
 101 
 102     _verbose = false;
 103     _current_step = 0;
 104     _current_step_info = NULL;
 105 
 106     _pc = NULL;
 107     _siginfo = NULL;
 108     _context = NULL;
 109 
 110     _size = size;
 111 }
 112 
 113 
 114 // Constructor for non-fatal errors
 115 VMError::VMError(const char* message) {
 116     _thread = NULL;
 117     _id = internal_error;     // set it to a value that's not an OS exception/signal
 118     _filename = NULL;
 119     _lineno = 0;
 120     _message = message;
 121 
 122     _verbose = false;
 123     _current_step = 0;
 124     _current_step_info = NULL;
 125 
 126     _pc = NULL;
 127     _siginfo = NULL;
 128     _context = NULL;
 129 
 130     _size = 0;
 131 }
 132 
 133 // -XX:OnError=<string>, where <string> can be a list of commands, separated
 134 // by ';'. "%p" is replaced by current process id (pid); "%%" is replaced by
 135 // a single "%". Some examples:
 136 //
 137 // -XX:OnError="pmap %p"                // show memory map
 138 // -XX:OnError="gcore %p; dbx - %p"     // dump core and launch debugger
 139 // -XX:OnError="cat hs_err_pid%p.log | mail my_email@sun.com"
 140 // -XX:OnError="kill -9 %p"             // ?#!@#
 141 
 142 // A simple parser for -XX:OnError, usage:
 143 //  ptr = OnError;
 144 //  while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr) != NULL)
 145 //     ... ...
 146 static char* next_OnError_command(char* buf, int buflen, const char** ptr) {
 147   if (ptr == NULL || *ptr == NULL) return NULL;
 148 
 149   const char* cmd = *ptr;
 150 
 151   // skip leading blanks or ';'
 152   while (*cmd == ' ' || *cmd == ';') cmd++;
 153 
 154   if (*cmd == '\0') return NULL;
 155 
 156   const char * cmdend = cmd;
 157   while (*cmdend != '\0' && *cmdend != ';') cmdend++;
 158 
 159   Arguments::copy_expand_pid(cmd, cmdend - cmd, buf, buflen);
 160 
 161   *ptr = (*cmdend == '\0' ? cmdend : cmdend + 1);
 162   return buf;
 163 }
 164 
 165 
 166 static void print_bug_submit_message(outputStream *out, Thread *thread) {
 167   if (out == NULL) return;
 168   out->print_raw_cr("# If you would like to submit a bug report, please visit:");
 169   out->print_raw   ("#   ");
 170   out->print_raw_cr(Arguments::java_vendor_url_bug());
 171   // If the crash is in native code, encourage user to submit a bug to the
 172   // provider of that code.
 173   if (thread && thread->is_Java_thread() &&
 174       !thread->is_hidden_from_external_view()) {
 175     JavaThread* jt = (JavaThread*)thread;
 176     if (jt->thread_state() == _thread_in_native) {
 177       out->print_cr("# The crash happened outside the Java Virtual Machine in native code.\n# See problematic frame for where to report the bug.");
 178     }
 179   }
 180   out->print_raw_cr("#");
 181 }
 182 
 183 
 184 // Return a string to describe the error
 185 char* VMError::error_string(char* buf, int buflen) {
 186   char signame_buf[64];
 187   const char *signame = os::exception_name(_id, signame_buf, sizeof(signame_buf));
 188 
 189   if (signame) {
 190     jio_snprintf(buf, buflen,
 191                  "%s (0x%x) at pc=" PTR_FORMAT ", pid=%d, tid=" UINTX_FORMAT,
 192                  signame, _id, _pc,
 193                  os::current_process_id(), os::current_thread_id());
 194   } else {
 195     if (_filename != NULL && _lineno > 0) {
 196       // skip directory names
 197       char separator = os::file_separator()[0];
 198       const char *p = strrchr(_filename, separator);
 199 
 200       jio_snprintf(buf, buflen,
 201         "Internal Error at %s:%d, pid=%d, tid=" UINTX_FORMAT " \nError: %s",
 202         p ? p + 1 : _filename, _lineno,
 203         os::current_process_id(), os::current_thread_id(),
 204         _message ? _message : "");
 205     } else {
 206       jio_snprintf(buf, buflen,
 207         "Internal Error (0x%x), pid=%d, tid=" UINTX_FORMAT,
 208         _id, os::current_process_id(), os::current_thread_id());
 209     }
 210   }
 211 
 212   return buf;
 213 }
 214 
 215 void VMError::print_stack_trace(outputStream* st, JavaThread* jt,
 216                                 char* buf, int buflen, bool verbose) {
 217 #ifdef ZERO
 218   if (jt->zero_stack()->sp() && jt->top_zero_frame()) {
 219     // StackFrameStream uses the frame anchor, which may not have
 220     // been set up.  This can be done at any time in Zero, however,
 221     // so if it hasn't been set up then we just set it up now and
 222     // clear it again when we're done.
 223     bool has_last_Java_frame = jt->has_last_Java_frame();
 224     if (!has_last_Java_frame)
 225       jt->set_last_Java_frame();
 226     st->print("Java frames:");
 227 
 228     // If the top frame is a Shark frame and the frame anchor isn't
 229     // set up then it's possible that the information in the frame
 230     // is garbage: it could be from a previous decache, or it could
 231     // simply have never been written.  So we print a warning...
 232     StackFrameStream sfs(jt);
 233     if (!has_last_Java_frame && !sfs.is_done()) {
 234       if (sfs.current()->zeroframe()->is_shark_frame()) {
 235         st->print(" (TOP FRAME MAY BE JUNK)");
 236       }
 237     }
 238     st->cr();
 239 
 240     // Print the frames
 241     for(int i = 0; !sfs.is_done(); sfs.next(), i++) {
 242       sfs.current()->zero_print_on_error(i, st, buf, buflen);
 243       st->cr();
 244     }
 245 
 246     // Reset the frame anchor if necessary
 247     if (!has_last_Java_frame)
 248       jt->reset_last_Java_frame();
 249   }
 250 #else
 251   if (jt->has_last_Java_frame()) {
 252     st->print_cr("Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)");
 253     for(StackFrameStream sfs(jt); !sfs.is_done(); sfs.next()) {
 254       sfs.current()->print_on_error(st, buf, buflen, verbose);
 255       st->cr();
 256     }
 257   }
 258 #endif // ZERO
 259 }
 260 
 261 // This is the main function to report a fatal error. Only one thread can
 262 // call this function, so we don't need to worry about MT-safety. But it's
 263 // possible that the error handler itself may crash or die on an internal
 264 // error, for example, when the stack/heap is badly damaged. We must be
 265 // able to handle recursive errors that happen inside error handler.
 266 //
 267 // Error reporting is done in several steps. If a crash or internal error
 268 // occurred when reporting an error, the nested signal/exception handler
 269 // can skip steps that are already (or partially) done. Error reporting will
 270 // continue from the next step. This allows us to retrieve and print
 271 // information that may be unsafe to get after a fatal error. If it happens,
 272 // you may find nested report_and_die() frames when you look at the stack
 273 // in a debugger.
 274 //
 275 // In general, a hang in error handler is much worse than a crash or internal
 276 // error, as it's harder to recover from a hang. Deadlock can happen if we
 277 // try to grab a lock that is already owned by current thread, or if the
 278 // owner is blocked forever (e.g. in os::infinite_sleep()). If possible, the
 279 // error handler and all the functions it called should avoid grabbing any
 280 // lock. An important thing to notice is that memory allocation needs a lock.
 281 //
 282 // We should avoid using large stack allocated buffers. Many errors happen
 283 // when stack space is already low. Making things even worse is that there
 284 // could be nested report_and_die() calls on stack (see above). Only one
 285 // thread can report error, so large buffers are statically allocated in data
 286 // segment.
 287 
 288 void VMError::report(outputStream* st) {
 289 # define BEGIN if (_current_step == 0) { _current_step = 1;
 290 # define STEP(n, s) } if (_current_step < n) { _current_step = n; _current_step_info = s;
 291 # define END }
 292 
 293   // don't allocate large buffer on stack
 294   static char buf[O_BUFLEN];
 295 
 296   BEGIN
 297 
 298   STEP(10, "(printing fatal error message)")
 299 
 300      st->print_cr("#");
 301      st->print_cr("# A fatal error has been detected by the Java Runtime Environment:");
 302 
 303   STEP(15, "(printing type of error)")
 304 
 305      switch(_id) {
 306        case oom_error:
 307          st->print_cr("#");
 308          st->print("# java.lang.OutOfMemoryError: ");
 309          if (_size) {
 310            st->print("requested ");
 311            sprintf(buf,SIZE_FORMAT,_size);
 312            st->print(buf);
 313            st->print(" bytes");
 314            if (_message != NULL) {
 315              st->print(" for ");
 316              st->print(_message);
 317            }
 318            st->print_cr(". Out of swap space?");
 319          } else {
 320            if (_message != NULL)
 321              st->print_cr(_message);
 322          }
 323          break;
 324        case internal_error:
 325        default:
 326          break;
 327      }
 328 
 329   STEP(20, "(printing exception/signal name)")
 330 
 331      st->print_cr("#");
 332      st->print("#  ");
 333      // Is it an OS exception/signal?
 334      if (os::exception_name(_id, buf, sizeof(buf))) {
 335        st->print("%s", buf);
 336        st->print(" (0x%x)", _id);                // signal number
 337        st->print(" at pc=" PTR_FORMAT, _pc);
 338      } else {
 339        st->print("Internal Error");
 340        if (_filename != NULL && _lineno > 0) {
 341 #ifdef PRODUCT
 342          // In product mode chop off pathname?
 343          char separator = os::file_separator()[0];
 344          const char *p = strrchr(_filename, separator);
 345          const char *file = p ? p+1 : _filename;
 346 #else
 347          const char *file = _filename;
 348 #endif
 349          size_t len = strlen(file);
 350          size_t buflen = sizeof(buf);
 351 
 352          strncpy(buf, file, buflen);
 353          if (len + 10 < buflen) {
 354            sprintf(buf + len, ":%d", _lineno);
 355          }
 356          st->print(" (%s)", buf);
 357        } else {
 358          st->print(" (0x%x)", _id);
 359        }
 360      }
 361 
 362   STEP(30, "(printing current thread and pid)")
 363 
 364      // process id, thread id
 365      st->print(", pid=%d", os::current_process_id());
 366      st->print(", tid=" UINTX_FORMAT, os::current_thread_id());
 367      st->cr();
 368 
 369   STEP(40, "(printing error message)")
 370 
 371      // error message
 372      if (_message && _message[0] != '\0') {
 373        st->print_cr("#  Error: %s", _message);
 374      }
 375 
 376   STEP(50, "(printing Java version string)")
 377 
 378      // VM version
 379      st->print_cr("#");
 380      JDK_Version::current().to_string(buf, sizeof(buf));
 381      st->print_cr("# JRE version: %s", buf);
 382      st->print_cr("# Java VM: %s (%s %s %s %s)",
 383                    Abstract_VM_Version::vm_name(),
 384                    Abstract_VM_Version::vm_release(),
 385                    Abstract_VM_Version::vm_info_string(),
 386                    Abstract_VM_Version::vm_platform_string(),
 387                    UseCompressedOops ? "compressed oops" : ""
 388                  );
 389 
 390   STEP(60, "(printing problematic frame)")
 391 
 392      // Print current frame if we have a context (i.e. it's a crash)
 393      if (_context) {
 394        st->print_cr("# Problematic frame:");
 395        st->print("# ");
 396        frame fr = os::fetch_frame_from_context(_context);
 397        fr.print_on_error(st, buf, sizeof(buf));
 398        st->cr();
 399        st->print_cr("#");
 400      }
 401 
 402   STEP(65, "(printing bug submit message)")
 403 
 404      if (_verbose) print_bug_submit_message(st, _thread);
 405 
 406   STEP(70, "(printing thread)" )
 407 
 408      if (_verbose) {
 409        st->cr();
 410        st->print_cr("---------------  T H R E A D  ---------------");
 411        st->cr();
 412      }
 413 
 414   STEP(80, "(printing current thread)" )
 415 
 416      // current thread
 417      if (_verbose) {
 418        if (_thread) {
 419          st->print("Current thread (" PTR_FORMAT "):  ", _thread);
 420          _thread->print_on_error(st, buf, sizeof(buf));
 421          st->cr();
 422        } else {
 423          st->print_cr("Current thread is native thread");
 424        }
 425        st->cr();
 426      }
 427 
 428   STEP(90, "(printing siginfo)" )
 429 
 430      // signal no, signal code, address that caused the fault
 431      if (_verbose && _siginfo) {
 432        os::print_siginfo(st, _siginfo);
 433        st->cr();
 434      }
 435 
 436   STEP(100, "(printing registers, top of stack, instructions near pc)")
 437 
 438      // registers, top of stack, instructions near pc
 439      if (_verbose && _context) {
 440        os::print_context(st, _context);
 441        st->cr();
 442      }
 443 
 444   STEP(110, "(printing stack bounds)" )
 445 
 446      if (_verbose) {
 447        st->print("Stack: ");
 448 
 449        address stack_top;
 450        size_t stack_size;
 451 
 452        if (_thread) {
 453           stack_top = _thread->stack_base();
 454           stack_size = _thread->stack_size();
 455        } else {
 456           stack_top = os::current_stack_base();
 457           stack_size = os::current_stack_size();
 458        }
 459 
 460        address stack_bottom = stack_top - stack_size;
 461        st->print("[" PTR_FORMAT "," PTR_FORMAT "]", stack_bottom, stack_top);
 462 
 463        frame fr = _context ? os::fetch_frame_from_context(_context)
 464                            : os::current_frame();
 465 
 466        if (fr.sp()) {
 467          st->print(",  sp=" PTR_FORMAT, fr.sp());
 468          st->print(",  free space=%" INTPTR_FORMAT "k",
 469                      ((intptr_t)fr.sp() - (intptr_t)stack_bottom) >> 10);
 470        }
 471 
 472        st->cr();
 473      }
 474 
 475   STEP(120, "(printing native stack)" )
 476 
 477      if (_verbose) {
 478        frame fr = _context ? os::fetch_frame_from_context(_context)
 479                            : os::current_frame();
 480 
 481        // see if it's a valid frame
 482        if (fr.pc()) {
 483           st->print_cr("Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)");
 484 
 485           int count = 0;
 486 
 487           while (count++ < StackPrintLimit) {
 488              fr.print_on_error(st, buf, sizeof(buf));
 489              st->cr();
 490              if (os::is_first_C_frame(&fr)) break;
 491              fr = os::get_sender_for_C_frame(&fr);
 492           }
 493 
 494           if (count > StackPrintLimit) {
 495              st->print_cr("...<more frames>...");
 496           }
 497 
 498           st->cr();
 499        }
 500      }
 501 
 502   STEP(130, "(printing Java stack)" )
 503 
 504      if (_verbose && _thread && _thread->is_Java_thread()) {
 505        print_stack_trace(st, (JavaThread*)_thread, buf, sizeof(buf));
 506      }
 507 
 508   STEP(135, "(printing target Java thread stack)" )
 509 
 510      // printing Java thread stack trace if it is involved in GC crash
 511      if (_verbose && (_thread->is_Named_thread())) {
 512        JavaThread*  jt = ((NamedThread *)_thread)->processed_thread();
 513        if (jt != NULL) {
 514          st->print_cr("JavaThread " PTR_FORMAT " (nid = " UINTX_FORMAT ") was being processed", jt, jt->osthread()->thread_id());
 515          print_stack_trace(st, jt, buf, sizeof(buf), true);
 516        }
 517      }
 518 
 519   STEP(140, "(printing VM operation)" )
 520 
 521      if (_verbose && _thread && _thread->is_VM_thread()) {
 522         VMThread* t = (VMThread*)_thread;
 523         VM_Operation* op = t->vm_operation();
 524         if (op) {
 525           op->print_on_error(st);
 526           st->cr();
 527           st->cr();
 528         }
 529      }
 530 
 531   STEP(150, "(printing current compile task)" )
 532 
 533      if (_verbose && _thread && _thread->is_Compiler_thread()) {
 534         CompilerThread* t = (CompilerThread*)_thread;
 535         if (t->task()) {
 536            st->cr();
 537            st->print_cr("Current CompileTask:");
 538            t->task()->print_line_on_error(st, buf, sizeof(buf));
 539            st->cr();
 540         }
 541      }
 542 
 543   STEP(160, "(printing process)" )
 544 
 545      if (_verbose) {
 546        st->cr();
 547        st->print_cr("---------------  P R O C E S S  ---------------");
 548        st->cr();
 549      }
 550 
 551   STEP(170, "(printing all threads)" )
 552 
 553      // all threads
 554      if (_verbose && _thread) {
 555        Threads::print_on_error(st, _thread, buf, sizeof(buf));
 556        st->cr();
 557      }
 558 
 559   STEP(175, "(printing VM state)" )
 560 
 561      if (_verbose) {
 562        // Safepoint state
 563        st->print("VM state:");
 564 
 565        if (SafepointSynchronize::is_synchronizing()) st->print("synchronizing");
 566        else if (SafepointSynchronize::is_at_safepoint()) st->print("at safepoint");
 567        else st->print("not at safepoint");
 568 
 569        // Also see if error occurred during initialization or shutdown
 570        if (!Universe::is_fully_initialized()) {
 571          st->print(" (not fully initialized)");
 572        } else if (VM_Exit::vm_exited()) {
 573          st->print(" (shutting down)");
 574        } else {
 575          st->print(" (normal execution)");
 576        }
 577        st->cr();
 578        st->cr();
 579      }
 580 
 581   STEP(180, "(printing owned locks on error)" )
 582 
 583      // mutexes/monitors that currently have an owner
 584      if (_verbose) {
 585        print_owned_locks_on_error(st);
 586        st->cr();
 587      }
 588 
 589   STEP(190, "(printing heap information)" )
 590 
 591      if (_verbose && Universe::is_fully_initialized()) {
 592        // print heap information before vm abort
 593        Universe::print_on(st);
 594        st->cr();
 595      }
 596 
 597   STEP(200, "(printing dynamic libraries)" )
 598 
 599      if (_verbose) {
 600        // dynamic libraries, or memory map
 601        os::print_dll_info(st);
 602        st->cr();
 603      }
 604 
 605   STEP(210, "(printing VM options)" )
 606 
 607      if (_verbose) {
 608        // VM options
 609        Arguments::print_on(st);
 610        st->cr();
 611      }
 612 
 613   STEP(220, "(printing environment variables)" )
 614 
 615      if (_verbose) {
 616        os::print_environment_variables(st, env_list, buf, sizeof(buf));
 617        st->cr();
 618      }
 619 
 620   STEP(225, "(printing signal handlers)" )
 621 
 622      if (_verbose) {
 623        os::print_signal_handlers(st, buf, sizeof(buf));
 624        st->cr();
 625      }
 626 
 627   STEP(230, "" )
 628 
 629      if (_verbose) {
 630        st->cr();
 631        st->print_cr("---------------  S Y S T E M  ---------------");
 632        st->cr();
 633      }
 634 
 635   STEP(240, "(printing OS information)" )
 636 
 637      if (_verbose) {
 638        os::print_os_info(st);
 639        st->cr();
 640      }
 641 
 642   STEP(250, "(printing CPU info)" )
 643      if (_verbose) {
 644        os::print_cpu_info(st);
 645        st->cr();
 646      }
 647 
 648   STEP(260, "(printing memory info)" )
 649 
 650      if (_verbose) {
 651        os::print_memory_info(st);
 652        st->cr();
 653      }
 654 
 655   STEP(270, "(printing internal vm info)" )
 656 
 657      if (_verbose) {
 658        st->print_cr("vm_info: %s", Abstract_VM_Version::internal_vm_info_string());
 659        st->cr();
 660      }
 661 
 662   STEP(280, "(printing date and time)" )
 663 
 664      if (_verbose) {
 665        os::print_date_and_time(st);
 666        st->cr();
 667      }
 668 
 669   END
 670 
 671 # undef BEGIN
 672 # undef STEP
 673 # undef END
 674 }
 675 
 676 
 677 void VMError::report_and_die() {
 678   // Don't allocate large buffer on stack
 679   static char buffer[O_BUFLEN];
 680 
 681   // First error, and its thread id. We must be able to handle native thread,
 682   // so use thread id instead of Thread* to identify thread.
 683   static VMError* first_error;
 684   static jlong    first_error_tid;
 685 
 686   // An error could happen before tty is initialized or after it has been
 687   // destroyed. Here we use a very simple unbuffered fdStream for printing.
 688   // Only out.print_raw() and out.print_raw_cr() should be used, as other
 689   // printing methods need to allocate large buffer on stack. To format a
 690   // string, use jio_snprintf() with a static buffer or use staticBufferStream.
 691   static fdStream out(defaultStream::output_fd());
 692 
 693   // How many errors occurred in error handler when reporting first_error.
 694   static int recursive_error_count;
 695 
 696   // We will first print a brief message to standard out (verbose = false),
 697   // then save detailed information in log file (verbose = true).
 698   static bool out_done = false;         // done printing to standard out
 699   static bool log_done = false;         // done saving error log
 700   static fdStream log;                  // error log
 701 
 702   if (SuppressFatalErrorMessage) {
 703       os::abort();
 704   }
 705   jlong mytid = os::current_thread_id();
 706   if (first_error == NULL &&
 707       Atomic::cmpxchg_ptr(this, &first_error, NULL) == NULL) {
 708 
 709     // first time
 710     first_error_tid = mytid;
 711     set_error_reported();
 712 
 713     if (ShowMessageBoxOnError) {
 714       show_message_box(buffer, sizeof(buffer));
 715 
 716       // User has asked JVM to abort. Reset ShowMessageBoxOnError so the
 717       // WatcherThread can kill JVM if the error handler hangs.
 718       ShowMessageBoxOnError = false;
 719     }
 720 
 721     // reset signal handlers or exception filter; make sure recursive crashes
 722     // are handled properly.
 723     reset_signal_handlers();
 724 
 725   } else {
 726     // If UseOsErrorReporting we call this for each level of the call stack
 727     // while searching for the exception handler.  Only the first level needs
 728     // to be reported.
 729     if (UseOSErrorReporting && log_done) return;
 730 
 731     // This is not the first error, see if it happened in a different thread
 732     // or in the same thread during error reporting.
 733     if (first_error_tid != mytid) {
 734       jio_snprintf(buffer, sizeof(buffer),
 735                    "[thread " INT64_FORMAT " also had an error]",
 736                    mytid);
 737       out.print_raw_cr(buffer);
 738 
 739       // error reporting is not MT-safe, block current thread
 740       os::infinite_sleep();
 741 
 742     } else {
 743       if (recursive_error_count++ > 30) {
 744         out.print_raw_cr("[Too many errors, abort]");
 745         os::die();
 746       }
 747 
 748       jio_snprintf(buffer, sizeof(buffer),
 749                    "[error occurred during error reporting %s, id 0x%x]",
 750                    first_error ? first_error->_current_step_info : "",
 751                    _id);
 752       if (log.is_open()) {
 753         log.cr();
 754         log.print_raw_cr(buffer);
 755         log.cr();
 756       } else {
 757         out.cr();
 758         out.print_raw_cr(buffer);
 759         out.cr();
 760       }
 761     }
 762   }
 763 
 764   // print to screen
 765   if (!out_done) {
 766     first_error->_verbose = false;
 767 
 768     staticBufferStream sbs(buffer, sizeof(buffer), &out);
 769     first_error->report(&sbs);
 770 
 771     out_done = true;
 772 
 773     first_error->_current_step = 0;         // reset current_step
 774     first_error->_current_step_info = "";   // reset current_step string
 775   }
 776 
 777   // print to error log file
 778   if (!log_done) {
 779     first_error->_verbose = true;
 780 
 781     // see if log file is already open
 782     if (!log.is_open()) {
 783       // open log file
 784       int fd = -1;
 785 
 786       if (ErrorFile != NULL) {
 787         bool copy_ok =
 788           Arguments::copy_expand_pid(ErrorFile, strlen(ErrorFile), buffer, sizeof(buffer));
 789         if (copy_ok) {
 790           fd = open(buffer, O_WRONLY | O_CREAT | O_TRUNC, 0666);
 791         }
 792       }
 793 
 794       if (fd == -1) {
 795         const char *cwd = os::get_current_directory(buffer, sizeof(buffer));
 796         size_t len = strlen(cwd);
 797         // either user didn't specify, or the user's location failed,
 798         // so use the default name in the current directory
 799         jio_snprintf(&buffer[len], sizeof(buffer)-len, "%shs_err_pid%u.log",
 800                      os::file_separator(), os::current_process_id());
 801         fd = open(buffer, O_WRONLY | O_CREAT | O_TRUNC, 0666);
 802       }
 803 
 804       if (fd == -1) {
 805         // try temp directory
 806         const char * tmpdir = os::get_temp_directory();
 807         jio_snprintf(buffer, sizeof(buffer), "%s%shs_err_pid%u.log",
 808                      tmpdir, os::file_separator(), os::current_process_id());
 809         fd = open(buffer, O_WRONLY | O_CREAT | O_TRUNC, 0666);
 810       }
 811 
 812       if (fd != -1) {
 813         out.print_raw("# An error report file with more information is saved as:\n# ");
 814         out.print_raw_cr(buffer);
 815         os::set_error_file(buffer);
 816 
 817         log.set_fd(fd);
 818       } else {
 819         out.print_raw_cr("# Can not save log file, dump to screen..");
 820         log.set_fd(defaultStream::output_fd());
 821       }
 822     }
 823 
 824     staticBufferStream sbs(buffer, O_BUFLEN, &log);
 825     first_error->report(&sbs);
 826     first_error->_current_step = 0;         // reset current_step
 827     first_error->_current_step_info = "";   // reset current_step string
 828 
 829     if (log.fd() != defaultStream::output_fd()) {
 830       close(log.fd());
 831     }
 832 
 833     log.set_fd(-1);
 834     log_done = true;
 835   }
 836 
 837 
 838   static bool skip_OnError = false;
 839   if (!skip_OnError && OnError && OnError[0]) {
 840     skip_OnError = true;
 841 
 842     out.print_raw_cr("#");
 843     out.print_raw   ("# -XX:OnError=\"");
 844     out.print_raw   (OnError);
 845     out.print_raw_cr("\"");
 846 
 847     char* cmd;
 848     const char* ptr = OnError;
 849     while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
 850       out.print_raw   ("#   Executing ");
 851 #if defined(LINUX)
 852       out.print_raw   ("/bin/sh -c ");
 853 #elif defined(SOLARIS)
 854       out.print_raw   ("/usr/bin/sh -c ");
 855 #endif
 856       out.print_raw   ("\"");
 857       out.print_raw   (cmd);
 858       out.print_raw_cr("\" ...");
 859 
 860       os::fork_and_exec(cmd);
 861     }
 862 
 863     // done with OnError
 864     OnError = NULL;
 865   }
 866 
 867   static bool skip_bug_url = false;
 868   if (!skip_bug_url) {
 869     skip_bug_url = true;
 870 
 871     out.print_raw_cr("#");
 872     print_bug_submit_message(&out, _thread);
 873   }
 874 
 875   if (!UseOSErrorReporting) {
 876     // os::abort() will call abort hooks, try it first.
 877     static bool skip_os_abort = false;
 878     if (!skip_os_abort) {
 879       skip_os_abort = true;
 880       os::abort();
 881     }
 882 
 883     // if os::abort() doesn't abort, try os::die();
 884     os::die();
 885   }
 886 }
 887 
 888 /*
 889  * OnOutOfMemoryError scripts/commands executed while VM is a safepoint - this
 890  * ensures utilities such as jmap can observe the process is a consistent state.
 891  */
 892 class VM_ReportJavaOutOfMemory : public VM_Operation {
 893  private:
 894   VMError *_err;
 895  public:
 896   VM_ReportJavaOutOfMemory(VMError *err) { _err = err; }
 897   VMOp_Type type() const                 { return VMOp_ReportJavaOutOfMemory; }
 898   void doit();
 899 };
 900 
 901 void VM_ReportJavaOutOfMemory::doit() {
 902   // Don't allocate large buffer on stack
 903   static char buffer[O_BUFLEN];
 904 
 905   tty->print_cr("#");
 906   tty->print_cr("# java.lang.OutOfMemoryError: %s", _err->message());
 907   tty->print_cr("# -XX:OnOutOfMemoryError=\"%s\"", OnOutOfMemoryError);
 908 
 909   // make heap parsability
 910   Universe::heap()->ensure_parsability(false);  // no need to retire TLABs
 911 
 912   char* cmd;
 913   const char* ptr = OnOutOfMemoryError;
 914   while ((cmd = next_OnError_command(buffer, sizeof(buffer), &ptr)) != NULL){
 915     tty->print("#   Executing ");
 916 #if defined(LINUX)
 917     tty->print  ("/bin/sh -c ");
 918 #elif defined(SOLARIS)
 919     tty->print  ("/usr/bin/sh -c ");
 920 #endif
 921     tty->print_cr("\"%s\"...", cmd);
 922 
 923     os::fork_and_exec(cmd);
 924   }
 925 }
 926 
 927 void VMError::report_java_out_of_memory() {
 928   if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
 929     MutexLocker ml(Heap_lock);
 930     VM_ReportJavaOutOfMemory op(this);
 931     VMThread::execute(&op);
 932   }
 933 }