1 /*
   2  * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/systemDictionary.hpp"
  27 #include "code/codeCache.hpp"
  28 #include "code/icBuffer.hpp"
  29 #include "code/nmethod.hpp"
  30 #include "code/vtableStubs.hpp"
  31 #include "compiler/compileBroker.hpp"
  32 #include "compiler/disassembler.hpp"
  33 #include "gc_implementation/shared/markSweep.hpp"
  34 #include "gc_interface/collectedHeap.hpp"
  35 #include "interpreter/bytecodeHistogram.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "memory/resourceArea.hpp"
  38 #include "memory/universe.hpp"
  39 #include "oops/oop.inline.hpp"
  40 #include "prims/privilegedStack.hpp"
  41 #include "runtime/arguments.hpp"
  42 #include "runtime/frame.hpp"
  43 #include "runtime/java.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubCodeGenerator.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "runtime/thread.inline.hpp"
  48 #include "runtime/vframe.hpp"
  49 #include "services/heapDumper.hpp"
  50 #include "utilities/defaultStream.hpp"
  51 #include "utilities/events.hpp"
  52 #include "utilities/top.hpp"
  53 #include "utilities/vmError.hpp"
  54 #ifdef TARGET_OS_FAMILY_linux
  55 # include "os_linux.inline.hpp"
  56 #endif
  57 #ifdef TARGET_OS_FAMILY_solaris
  58 # include "os_solaris.inline.hpp"
  59 #endif
  60 #ifdef TARGET_OS_FAMILY_windows
  61 # include "os_windows.inline.hpp"
  62 #endif
  63 #ifdef TARGET_OS_FAMILY_bsd
  64 # include "os_bsd.inline.hpp"
  65 #endif
  66 
  67 #ifndef ASSERT
  68 #  ifdef _DEBUG
  69    // NOTE: don't turn the lines below into a comment -- if you're getting
  70    // a compile error here, change the settings to define ASSERT
  71    ASSERT should be defined when _DEBUG is defined.  It is not intended to be used for debugging
  72    functions that do not slow down the system too much and thus can be left in optimized code.
  73    On the other hand, the code should not be included in a production version.
  74 #  endif // _DEBUG
  75 #endif // ASSERT
  76 
  77 
  78 #ifdef _DEBUG
  79 #  ifndef ASSERT
  80      configuration error: ASSERT must be defined in debug version
  81 #  endif // ASSERT
  82 #endif // _DEBUG
  83 
  84 
  85 #ifdef PRODUCT
  86 #  if -defined _DEBUG || -defined ASSERT
  87      configuration error: ASSERT et al. must not be defined in PRODUCT version
  88 #  endif
  89 #endif // PRODUCT
  90 
  91 FormatBufferResource::FormatBufferResource(const char * format, ...)
  92   : FormatBufferBase((char*)resource_allocate_bytes(RES_BUFSZ)) {
  93   va_list argp;
  94   va_start(argp, format);
  95   jio_vsnprintf(_buf, RES_BUFSZ, format, argp);
  96   va_end(argp);
  97 }
  98 
  99 void warning(const char* format, ...) {
 100   if (PrintWarnings) {
 101     FILE* const err = defaultStream::error_stream();
 102     jio_fprintf(err, "%s warning: ", VM_Version::vm_name());
 103     va_list ap;
 104     va_start(ap, format);
 105     vfprintf(err, format, ap);
 106     va_end(ap);
 107     fputc('\n', err);
 108   }
 109   if (BreakAtWarning) BREAKPOINT;
 110 }
 111 
 112 #ifndef PRODUCT
 113 
 114 #define is_token_break(ch) (isspace(ch) || (ch) == ',')
 115 
 116 static const char* last_file_name = NULL;
 117 static int         last_line_no   = -1;
 118 
 119 // assert/guarantee/... may happen very early during VM initialization.
 120 // Don't rely on anything that is initialized by Threads::create_vm(). For
 121 // example, don't use tty.
 122 bool error_is_suppressed(const char* file_name, int line_no) {
 123   // The following 1-element cache requires that passed-in
 124   // file names are always only constant literals.
 125   if (file_name == last_file_name && line_no == last_line_no)  return true;
 126 
 127   int file_name_len = (int)strlen(file_name);
 128   char separator = os::file_separator()[0];
 129   const char* base_name = strrchr(file_name, separator);
 130   if (base_name == NULL)
 131     base_name = file_name;
 132 
 133   // scan the SuppressErrorAt option
 134   const char* cp = SuppressErrorAt;
 135   for (;;) {
 136     const char* sfile;
 137     int sfile_len;
 138     int sline;
 139     bool noisy;
 140     while ((*cp) != '\0' && is_token_break(*cp))  cp++;
 141     if ((*cp) == '\0')  break;
 142     sfile = cp;
 143     while ((*cp) != '\0' && !is_token_break(*cp) && (*cp) != ':')  cp++;
 144     sfile_len = cp - sfile;
 145     if ((*cp) == ':')  cp++;
 146     sline = 0;
 147     while ((*cp) != '\0' && isdigit(*cp)) {
 148       sline *= 10;
 149       sline += (*cp) - '0';
 150       cp++;
 151     }
 152     // "file:line!" means the assert suppression is not silent
 153     noisy = ((*cp) == '!');
 154     while ((*cp) != '\0' && !is_token_break(*cp))  cp++;
 155     // match the line
 156     if (sline != 0) {
 157       if (sline != line_no)  continue;
 158     }
 159     // match the file
 160     if (sfile_len > 0) {
 161       const char* look = file_name;
 162       const char* look_max = file_name + file_name_len - sfile_len;
 163       const char* foundp;
 164       bool match = false;
 165       while (!match
 166              && (foundp = strchr(look, sfile[0])) != NULL
 167              && foundp <= look_max) {
 168         match = true;
 169         for (int i = 1; i < sfile_len; i++) {
 170           if (sfile[i] != foundp[i]) {
 171             match = false;
 172             break;
 173           }
 174         }
 175         look = foundp + 1;
 176       }
 177       if (!match)  continue;
 178     }
 179     // got a match!
 180     if (noisy) {
 181       fdStream out(defaultStream::output_fd());
 182       out.print_raw("[error suppressed at ");
 183       out.print_raw(base_name);
 184       char buf[16];
 185       jio_snprintf(buf, sizeof(buf), ":%d]", line_no);
 186       out.print_raw_cr(buf);
 187     } else {
 188       // update 1-element cache for fast silent matches
 189       last_file_name = file_name;
 190       last_line_no   = line_no;
 191     }
 192     return true;
 193   }
 194 
 195   if (!is_error_reported()) {
 196     // print a friendly hint:
 197     fdStream out(defaultStream::output_fd());
 198     out.print_raw_cr("# To suppress the following error report, specify this argument");
 199     out.print_raw   ("# after -XX: or in .hotspotrc:  SuppressErrorAt=");
 200     out.print_raw   (base_name);
 201     char buf[16];
 202     jio_snprintf(buf, sizeof(buf), ":%d", line_no);
 203     out.print_raw_cr(buf);
 204   }
 205   return false;
 206 }
 207 
 208 #undef is_token_break
 209 
 210 #else
 211 
 212 // Place-holder for non-existent suppression check:
 213 #define error_is_suppressed(file_name, line_no) (false)
 214 
 215 #endif // !PRODUCT
 216 
 217 void report_vm_error(const char* file, int line, const char* error_msg,
 218                      const char* detail_msg)
 219 {
 220   if (Debugging || error_is_suppressed(file, line)) return;
 221   Thread* const thread = ThreadLocalStorage::get_thread_slow();
 222   VMError err(thread, file, line, error_msg, detail_msg);
 223   err.report_and_die();
 224 }
 225 
 226 void report_fatal(const char* file, int line, const char* message)
 227 {
 228   report_vm_error(file, line, "fatal error", message);
 229 }
 230 
 231 void report_vm_out_of_memory(const char* file, int line, size_t size,
 232                              VMErrorType vm_err_type, const char* message) {
 233   if (Debugging) return;
 234 
 235   Thread* thread = ThreadLocalStorage::get_thread_slow();
 236   VMError(thread, file, line, size, vm_err_type, message).report_and_die();
 237 
 238   // The UseOSErrorReporting option in report_and_die() may allow a return
 239   // to here. If so then we'll have to figure out how to handle it.
 240   guarantee(false, "report_and_die() should not return here");
 241 }
 242 
 243 void report_should_not_call(const char* file, int line) {
 244   report_vm_error(file, line, "ShouldNotCall()");
 245 }
 246 
 247 void report_should_not_reach_here(const char* file, int line) {
 248   report_vm_error(file, line, "ShouldNotReachHere()");
 249 }
 250 
 251 void report_unimplemented(const char* file, int line) {
 252   report_vm_error(file, line, "Unimplemented()");
 253 }
 254 
 255 void report_untested(const char* file, int line, const char* message) {
 256 #ifndef PRODUCT
 257   warning("Untested: %s in %s: %d\n", message, file, line);
 258 #endif // !PRODUCT
 259 }
 260 
 261 void report_out_of_shared_space(SharedSpaceType shared_space) {
 262   static const char* name[] = {
 263     "native memory for metadata",
 264     "shared read only space",
 265     "shared read write space",
 266     "shared miscellaneous data space"
 267   };
 268   static const char* flag[] = {
 269     "Metaspace",
 270     "SharedReadOnlySize",
 271     "SharedReadWriteSize",
 272     "SharedMiscDataSize"
 273   };
 274 
 275    warning("\nThe %s is not large enough\n"
 276            "to preload requested classes. Use -XX:%s=\n"
 277            "to increase the initial size of %s.\n",
 278            name[shared_space], flag[shared_space], name[shared_space]);
 279    exit(2);
 280 }
 281 
 282 void report_java_out_of_memory(const char* message) {
 283   static jint out_of_memory_reported = 0;
 284 
 285   // A number of threads may attempt to report OutOfMemoryError at around the
 286   // same time. To avoid dumping the heap or executing the data collection
 287   // commands multiple times we just do it once when the first threads reports
 288   // the error.
 289   if (Atomic::cmpxchg(1, &out_of_memory_reported, 0) == 0) {
 290     // create heap dump before OnOutOfMemoryError commands are executed
 291     if (HeapDumpOnOutOfMemoryError) {
 292       tty->print_cr("java.lang.OutOfMemoryError: %s", message);
 293       HeapDumper::dump_heap_from_oome();
 294     }
 295 
 296     if (OnOutOfMemoryError && OnOutOfMemoryError[0]) {
 297       VMError err(message);
 298       err.report_java_out_of_memory();
 299     }
 300   }
 301 }
 302 
 303 static bool error_reported = false;
 304 
 305 // call this when the VM is dying--it might loosen some asserts
 306 void set_error_reported() {
 307   error_reported = true;
 308 }
 309 
 310 bool is_error_reported() {
 311     return error_reported;
 312 }
 313 
 314 #ifndef PRODUCT
 315 #include <signal.h>
 316 
 317 void test_error_handler(size_t test_num)
 318 {
 319   if (test_num == 0) return;
 320 
 321   // If asserts are disabled, use the corresponding guarantee instead.
 322   size_t n = test_num;
 323   NOT_DEBUG(if (n <= 2) n += 2);
 324 
 325   const char* const str = "hello";
 326   const size_t      num = (size_t)os::vm_page_size();
 327 
 328   const char* const eol = os::line_separator();
 329   const char* const msg = "this message should be truncated during formatting";
 330   const void (*funcPtr)(void) = (const void(*)()) 0xF;  // bad function pointer
 331 
 332   // Keep this in sync with test/runtime/6888954/vmerrors.sh.
 333   switch (n) {
 334     case  1: assert(str == NULL, "expected null");
 335     case  2: assert(num == 1023 && *str == 'X',
 336                     err_msg("num=" SIZE_FORMAT " str=\"%s\"", num, str));
 337     case  3: guarantee(str == NULL, "expected null");
 338     case  4: guarantee(num == 1023 && *str == 'X',
 339                        err_msg("num=" SIZE_FORMAT " str=\"%s\"", num, str));
 340     case  5: fatal("expected null");
 341     case  6: fatal(err_msg("num=" SIZE_FORMAT " str=\"%s\"", num, str));
 342     case  7: fatal(err_msg("%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
 343                            "%s%s#    %s%s#    %s%s#    %s%s#    %s%s#    "
 344                            "%s%s#    %s%s#    %s%s#    %s%s#    %s",
 345                            msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
 346                            msg, eol, msg, eol, msg, eol, msg, eol, msg, eol,
 347                            msg, eol, msg, eol, msg, eol, msg, eol, msg));
 348     case  8: vm_exit_out_of_memory(num, OOM_MALLOC_ERROR, "ChunkPool::allocate");
 349     case  9: ShouldNotCallThis();
 350     case 10: ShouldNotReachHere();
 351     case 11: Unimplemented();
 352     // This does not generate an hs_err* file on Windows so "break"
 353     // out to the ShouldNotReachHere().
 354     case 12: os::signal_raise(SIGSEGV); break;
 355     // There's no guarantee the bad function pointer will crash us
 356     // so "break" out to the ShouldNotReachHere().
 357     case 13: (*funcPtr)(); ShouldNotReachHere(); break;
 358 
 359     default: tty->print_cr("ERROR: %d: unexpected test_num value.", n);
 360   }
 361   ShouldNotReachHere();
 362 }
 363 #endif // !PRODUCT
 364 
 365 // ------ helper functions for debugging go here ------------
 366 
 367 // All debug entries should be wrapped with a stack allocated
 368 // Command object. It makes sure a resource mark is set and
 369 // flushes the logfile to prevent file sharing problems.
 370 
 371 class Command : public StackObj {
 372  private:
 373   ResourceMark rm;
 374   ResetNoHandleMark rnhm;
 375   HandleMark   hm;
 376   bool debug_save;
 377  public:
 378   static int level;
 379   Command(const char* str) {
 380     debug_save = Debugging;
 381     Debugging = true;
 382     if (level++ > 0)  return;
 383     tty->cr();
 384     tty->print_cr("\"Executing %s\"", str);
 385   }
 386 
 387   ~Command() {
 388         tty->flush();
 389         Debugging = debug_save;
 390         level--;
 391   }
 392 };
 393 
 394 int Command::level = 0;
 395 
 396 #ifndef PRODUCT
 397 
 398 extern "C" void blob(CodeBlob* cb) {
 399   Command c("blob");
 400   cb->print();
 401 }
 402 
 403 
 404 extern "C" void dump_vtable(address p) {
 405   Command c("dump_vtable");
 406   Klass* k = (Klass*)p;
 407   InstanceKlass::cast(k)->vtable()->print();
 408 }
 409 
 410 
 411 extern "C" void nm(intptr_t p) {
 412   // Actually we look through all CodeBlobs (the nm name has been kept for backwards compatability)
 413   Command c("nm");
 414   CodeBlob* cb = CodeCache::find_blob((address)p);
 415   if (cb == NULL) {
 416     tty->print_cr("NULL");
 417   } else {
 418     cb->print();
 419   }
 420 }
 421 
 422 
 423 extern "C" void disnm(intptr_t p) {
 424   Command c("disnm");
 425   CodeBlob* cb = CodeCache::find_blob((address) p);
 426   nmethod* nm = cb->as_nmethod_or_null();
 427   if (nm) {
 428     nm->print();
 429     Disassembler::decode(nm);
 430   } else {
 431     cb->print();
 432     Disassembler::decode(cb);
 433   }
 434 }
 435 
 436 
 437 extern "C" void printnm(intptr_t p) {
 438   char buffer[256];
 439   sprintf(buffer, "printnm: " INTPTR_FORMAT, p);
 440   Command c(buffer);
 441   CodeBlob* cb = CodeCache::find_blob((address) p);
 442   if (cb->is_nmethod()) {
 443     nmethod* nm = (nmethod*)cb;
 444     nm->print_nmethod(true);
 445   }
 446 }
 447 
 448 
 449 extern "C" void universe() {
 450   Command c("universe");
 451   Universe::print();
 452 }
 453 
 454 
 455 extern "C" void verify() {
 456   // try to run a verify on the entire system
 457   // note: this may not be safe if we're not at a safepoint; for debugging,
 458   // this manipulates the safepoint settings to avoid assertion failures
 459   Command c("universe verify");
 460   bool safe = SafepointSynchronize::is_at_safepoint();
 461   if (!safe) {
 462     tty->print_cr("warning: not at safepoint -- verify may fail");
 463     SafepointSynchronize::set_is_at_safepoint();
 464   }
 465   // Ensure Eden top is correct before verification
 466   Universe::heap()->prepare_for_verify();
 467   Universe::verify();
 468   if (!safe) SafepointSynchronize::set_is_not_at_safepoint();
 469 }
 470 
 471 
 472 extern "C" void pp(void* p) {
 473   Command c("pp");
 474   FlagSetting fl(PrintVMMessages, true);
 475   FlagSetting f2(DisplayVMOutput, true);
 476   if (Universe::heap()->is_in(p)) {
 477     oop obj = oop(p);
 478     obj->print();
 479   } else {
 480     tty->print(PTR_FORMAT, p);
 481   }
 482 }
 483 
 484 
 485 // pv: print vm-printable object
 486 extern "C" void pa(intptr_t p)   { ((AllocatedObj*) p)->print(); }
 487 extern "C" void findpc(intptr_t x);
 488 
 489 #endif // !PRODUCT
 490 
 491 extern "C" void ps() { // print stack
 492   if (Thread::current() == NULL) return;
 493   Command c("ps");
 494 
 495 
 496   // Prints the stack of the current Java thread
 497   JavaThread* p = JavaThread::active();
 498   tty->print(" for thread: ");
 499   p->print();
 500   tty->cr();
 501 
 502   if (p->has_last_Java_frame()) {
 503     // If the last_Java_fp is set we are in C land and
 504     // can call the standard stack_trace function.
 505 #ifdef PRODUCT
 506     p->print_stack();
 507   } else {
 508     tty->print_cr("Cannot find the last Java frame, printing stack disabled.");
 509 #else // !PRODUCT
 510     p->trace_stack();
 511   } else {
 512     frame f = os::current_frame();
 513     RegisterMap reg_map(p);
 514     f = f.sender(&reg_map);
 515     tty->print("(guessing starting frame id=%#p based on current fp)\n", f.id());
 516     p->trace_stack_from(vframe::new_vframe(&f, &reg_map, p));
 517   pd_ps(f);
 518 #endif // PRODUCT
 519   }
 520 
 521 }
 522 
 523 extern "C" void pfl() {
 524   // print frame layout
 525   Command c("pfl");
 526   JavaThread* p = JavaThread::active();
 527   tty->print(" for thread: ");
 528   p->print();
 529   tty->cr();
 530   if (p->has_last_Java_frame()) {
 531     p->print_frame_layout();
 532   }
 533 }
 534 
 535 #ifndef PRODUCT
 536 
 537 extern "C" void psf() { // print stack frames
 538   {
 539     Command c("psf");
 540     JavaThread* p = JavaThread::active();
 541     tty->print(" for thread: ");
 542     p->print();
 543     tty->cr();
 544     if (p->has_last_Java_frame()) {
 545       p->trace_frames();
 546     }
 547   }
 548 }
 549 
 550 
 551 extern "C" void threads() {
 552   Command c("threads");
 553   Threads::print(false, true);
 554 }
 555 
 556 
 557 extern "C" void psd() {
 558   Command c("psd");
 559   SystemDictionary::print();
 560 }
 561 
 562 
 563 extern "C" void safepoints() {
 564   Command c("safepoints");
 565   SafepointSynchronize::print_state();
 566 }
 567 
 568 #endif // !PRODUCT
 569 
 570 extern "C" void pss() { // print all stacks
 571   if (Thread::current() == NULL) return;
 572   Command c("pss");
 573   Threads::print(true, PRODUCT_ONLY(false) NOT_PRODUCT(true));
 574 }
 575 
 576 #ifndef PRODUCT
 577 
 578 extern "C" void debug() {               // to set things up for compiler debugging
 579   Command c("debug");
 580   WizardMode = true;
 581   PrintVMMessages = PrintCompilation = true;
 582   PrintInlining = PrintAssembly = true;
 583   tty->flush();
 584 }
 585 
 586 
 587 extern "C" void ndebug() {              // undo debug()
 588   Command c("ndebug");
 589   PrintCompilation = false;
 590   PrintInlining = PrintAssembly = false;
 591   tty->flush();
 592 }
 593 
 594 
 595 extern "C" void flush()  {
 596   Command c("flush");
 597   tty->flush();
 598 }
 599 
 600 extern "C" void events() {
 601   Command c("events");
 602   Events::print();
 603 }
 604 
 605 extern "C" Method* findm(intptr_t pc) {
 606   Command c("findm");
 607   nmethod* nm = CodeCache::find_nmethod((address)pc);
 608   return (nm == NULL) ? (Method*)NULL : nm->method();
 609 }
 610 
 611 
 612 extern "C" nmethod* findnm(intptr_t addr) {
 613   Command c("findnm");
 614   return  CodeCache::find_nmethod((address)addr);
 615 }
 616 
 617 // Another interface that isn't ambiguous in dbx.
 618 // Can we someday rename the other find to hsfind?
 619 extern "C" void hsfind(intptr_t x) {
 620   Command c("hsfind");
 621   os::print_location(tty, x, false);
 622 }
 623 
 624 
 625 extern "C" void find(intptr_t x) {
 626   Command c("find");
 627   os::print_location(tty, x, false);
 628 }
 629 
 630 
 631 extern "C" void findpc(intptr_t x) {
 632   Command c("findpc");
 633   os::print_location(tty, x, true);
 634 }
 635 
 636 
 637 // Need method pointer to find bcp, when not in permgen.
 638 extern "C" void findbcp(intptr_t method, intptr_t bcp) {
 639   Command c("findbcp");
 640   Method* mh = (Method*)method;
 641   if (!mh->is_native()) {
 642     tty->print_cr("bci_from(%p) = %d; print_codes():",
 643                         mh, mh->bci_from(address(bcp)));
 644     mh->print_codes_on(tty);
 645   }
 646 }
 647 
 648 // int versions of all methods to avoid having to type type casts in the debugger
 649 
 650 void pp(intptr_t p)          { pp((void*)p); }
 651 void pp(oop p)               { pp((void*)p); }
 652 
 653 void help() {
 654   Command c("help");
 655   tty->print_cr("basic");
 656   tty->print_cr("  pp(void* p)   - try to make sense of p");
 657   tty->print_cr("  pv(intptr_t p)- ((PrintableResourceObj*) p)->print()");
 658   tty->print_cr("  ps()          - print current thread stack");
 659   tty->print_cr("  pss()         - print all thread stacks");
 660   tty->print_cr("  pm(int pc)    - print Method* given compiled PC");
 661   tty->print_cr("  findm(intptr_t pc) - finds Method*");
 662   tty->print_cr("  find(intptr_t x)   - finds & prints nmethod/stub/bytecode/oop based on pointer into it");
 663 
 664   tty->print_cr("misc.");
 665   tty->print_cr("  flush()       - flushes the log file");
 666   tty->print_cr("  events()      - dump events from ring buffers");
 667 
 668 
 669   tty->print_cr("compiler debugging");
 670   tty->print_cr("  debug()       - to set things up for compiler debugging");
 671   tty->print_cr("  ndebug()      - undo debug");
 672 }
 673 
 674 #endif // !PRODUCT