1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "classfile/stringTable.hpp"
  27 #include "classfile/symbolTable.hpp"
  28 #include "classfile/systemDictionary.hpp"
  29 #include "code/codeCache.hpp"
  30 #include "code/icBuffer.hpp"
  31 #include "code/nmethod.hpp"
  32 #include "code/pcDesc.hpp"
  33 #include "code/scopeDesc.hpp"
  34 #include "gc/shared/collectedHeap.hpp"
  35 #include "gc/shared/gcLocker.inline.hpp"
  36 #include "interpreter/interpreter.hpp"
  37 #include "logging/log.hpp"
  38 #include "memory/resourceArea.hpp"
  39 #include "memory/universe.inline.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "oops/symbol.hpp"
  42 #include "oops/valueKlass.hpp"
  43 #include "runtime/atomic.hpp"
  44 #include "runtime/compilationPolicy.hpp"
  45 #include "runtime/deoptimization.hpp"
  46 #include "runtime/frame.inline.hpp"
  47 #include "runtime/interfaceSupport.hpp"
  48 #include "runtime/mutexLocker.hpp"
  49 #include "runtime/orderAccess.inline.hpp"
  50 #include "runtime/osThread.hpp"
  51 #include "runtime/safepoint.hpp"
  52 #include "runtime/signature.hpp"
  53 #include "runtime/stubCodeGenerator.hpp"
  54 #include "runtime/stubRoutines.hpp"
  55 #include "runtime/sweeper.hpp"
  56 #include "runtime/synchronizer.hpp"
  57 #include "runtime/thread.inline.hpp"
  58 #include "runtime/timerTrace.hpp"
  59 #include "services/runtimeService.hpp"
  60 #include "trace/tracing.hpp"
  61 #include "trace/traceMacros.hpp"
  62 #include "utilities/events.hpp"
  63 #include "utilities/macros.hpp"
  64 #if INCLUDE_ALL_GCS
  65 #include "gc/cms/concurrentMarkSweepThread.hpp"
  66 #include "gc/g1/suspendibleThreadSet.hpp"
  67 #endif // INCLUDE_ALL_GCS
  68 #ifdef COMPILER1
  69 #include "c1/c1_globals.hpp"
  70 #endif
  71 
  72 // --------------------------------------------------------------------------------------------------
  73 // Implementation of Safepoint begin/end
  74 
  75 SafepointSynchronize::SynchronizeState volatile SafepointSynchronize::_state = SafepointSynchronize::_not_synchronized;
  76 volatile int  SafepointSynchronize::_waiting_to_block = 0;
  77 volatile int SafepointSynchronize::_safepoint_counter = 0;
  78 int SafepointSynchronize::_current_jni_active_count = 0;
  79 long  SafepointSynchronize::_end_of_last_safepoint = 0;
  80 static volatile int PageArmed = 0 ;        // safepoint polling page is RO|RW vs PROT_NONE
  81 static volatile int TryingToBlock = 0 ;    // proximate value -- for advisory use only
  82 static bool timeout_error_printed = false;
  83 
  84 // Roll all threads forward to a safepoint and suspend them all
  85 void SafepointSynchronize::begin() {
  86   EventSafepointBegin begin_event;
  87   Thread* myThread = Thread::current();
  88   assert(myThread->is_VM_thread(), "Only VM thread may execute a safepoint");
  89 
  90   if (PrintSafepointStatistics || PrintSafepointStatisticsTimeout > 0) {
  91     _safepoint_begin_time = os::javaTimeNanos();
  92     _ts_of_current_safepoint = tty->time_stamp().seconds();
  93   }
  94 
  95 #if INCLUDE_ALL_GCS
  96   if (UseConcMarkSweepGC) {
  97     // In the future we should investigate whether CMS can use the
  98     // more-general mechanism below.  DLD (01/05).
  99     ConcurrentMarkSweepThread::synchronize(false);
 100   } else if (UseG1GC) {
 101     SuspendibleThreadSet::synchronize();
 102   }
 103 #endif // INCLUDE_ALL_GCS
 104 
 105   // By getting the Threads_lock, we assure that no threads are about to start or
 106   // exit. It is released again in SafepointSynchronize::end().
 107   Threads_lock->lock();
 108 
 109   assert( _state == _not_synchronized, "trying to safepoint synchronize with wrong state");
 110 
 111   int nof_threads = Threads::number_of_threads();
 112 
 113   log_debug(safepoint)("Safepoint synchronization initiated. (%d)", nof_threads);
 114 
 115   RuntimeService::record_safepoint_begin();
 116 
 117   MutexLocker mu(Safepoint_lock);
 118 
 119   // Reset the count of active JNI critical threads
 120   _current_jni_active_count = 0;
 121 
 122   // Set number of threads to wait for, before we initiate the callbacks
 123   _waiting_to_block = nof_threads;
 124   TryingToBlock     = 0 ;
 125   int still_running = nof_threads;
 126 
 127   // Save the starting time, so that it can be compared to see if this has taken
 128   // too long to complete.
 129   jlong safepoint_limit_time = 0;
 130   timeout_error_printed = false;
 131 
 132   // PrintSafepointStatisticsTimeout can be specified separately. When
 133   // specified, PrintSafepointStatistics will be set to true in
 134   // deferred_initialize_stat method. The initialization has to be done
 135   // early enough to avoid any races. See bug 6880029 for details.
 136   if (PrintSafepointStatistics || PrintSafepointStatisticsTimeout > 0) {
 137     deferred_initialize_stat();
 138   }
 139 
 140   // Begin the process of bringing the system to a safepoint.
 141   // Java threads can be in several different states and are
 142   // stopped by different mechanisms:
 143   //
 144   //  1. Running interpreted
 145   //     The interpreter dispatch table is changed to force it to
 146   //     check for a safepoint condition between bytecodes.
 147   //  2. Running in native code
 148   //     When returning from the native code, a Java thread must check
 149   //     the safepoint _state to see if we must block.  If the
 150   //     VM thread sees a Java thread in native, it does
 151   //     not wait for this thread to block.  The order of the memory
 152   //     writes and reads of both the safepoint state and the Java
 153   //     threads state is critical.  In order to guarantee that the
 154   //     memory writes are serialized with respect to each other,
 155   //     the VM thread issues a memory barrier instruction
 156   //     (on MP systems).  In order to avoid the overhead of issuing
 157   //     a memory barrier for each Java thread making native calls, each Java
 158   //     thread performs a write to a single memory page after changing
 159   //     the thread state.  The VM thread performs a sequence of
 160   //     mprotect OS calls which forces all previous writes from all
 161   //     Java threads to be serialized.  This is done in the
 162   //     os::serialize_thread_states() call.  This has proven to be
 163   //     much more efficient than executing a membar instruction
 164   //     on every call to native code.
 165   //  3. Running compiled Code
 166   //     Compiled code reads a global (Safepoint Polling) page that
 167   //     is set to fault if we are trying to get to a safepoint.
 168   //  4. Blocked
 169   //     A thread which is blocked will not be allowed to return from the
 170   //     block condition until the safepoint operation is complete.
 171   //  5. In VM or Transitioning between states
 172   //     If a Java thread is currently running in the VM or transitioning
 173   //     between states, the safepointing code will wait for the thread to
 174   //     block itself when it attempts transitions to a new state.
 175   //
 176   {
 177     EventSafepointStateSynchronization sync_event;
 178     int initial_running = 0;
 179 
 180     _state            = _synchronizing;
 181     OrderAccess::fence();
 182 
 183     // Flush all thread states to memory
 184     if (!UseMembar) {
 185       os::serialize_thread_states();
 186     }
 187 
 188     // Make interpreter safepoint aware
 189     Interpreter::notice_safepoints();
 190 
 191     if (DeferPollingPageLoopCount < 0) {
 192       // Make polling safepoint aware
 193       guarantee (PageArmed == 0, "invariant") ;
 194       PageArmed = 1 ;
 195       os::make_polling_page_unreadable();
 196     }
 197 
 198     // Consider using active_processor_count() ... but that call is expensive.
 199     int ncpus = os::processor_count() ;
 200 
 201 #ifdef ASSERT
 202     for (JavaThread *cur = Threads::first(); cur != NULL; cur = cur->next()) {
 203       assert(cur->safepoint_state()->is_running(), "Illegal initial state");
 204       // Clear the visited flag to ensure that the critical counts are collected properly.
 205       cur->set_visited_for_critical_count(false);
 206     }
 207 #endif // ASSERT
 208 
 209     if (SafepointTimeout)
 210       safepoint_limit_time = os::javaTimeNanos() + (jlong)SafepointTimeoutDelay * MICROUNITS;
 211 
 212     // Iterate through all threads until it have been determined how to stop them all at a safepoint
 213     unsigned int iterations = 0;
 214     int steps = 0 ;
 215     while(still_running > 0) {
 216       for (JavaThread *cur = Threads::first(); cur != NULL; cur = cur->next()) {
 217         assert(!cur->is_ConcurrentGC_thread(), "A concurrent GC thread is unexpectly being suspended");
 218         ThreadSafepointState *cur_state = cur->safepoint_state();
 219         if (cur_state->is_running()) {
 220           cur_state->examine_state_of_thread();
 221           if (!cur_state->is_running()) {
 222             still_running--;
 223             // consider adjusting steps downward:
 224             //   steps = 0
 225             //   steps -= NNN
 226             //   steps >>= 1
 227             //   steps = MIN(steps, 2000-100)
 228             //   if (iterations != 0) steps -= NNN
 229           }
 230           if (log_is_enabled(Trace, safepoint)) {
 231             ResourceMark rm;
 232             cur_state->print_on(Log(safepoint)::trace_stream());
 233           }
 234         }
 235       }
 236 
 237       if (iterations == 0) {
 238         initial_running = still_running;
 239         if (PrintSafepointStatistics) {
 240           begin_statistics(nof_threads, still_running);
 241         }
 242       }
 243 
 244       if (still_running > 0) {
 245         // Check for if it takes to long
 246         if (SafepointTimeout && safepoint_limit_time < os::javaTimeNanos()) {
 247           print_safepoint_timeout(_spinning_timeout);
 248         }
 249 
 250         // Spin to avoid context switching.
 251         // There's a tension between allowing the mutators to run (and rendezvous)
 252         // vs spinning.  As the VM thread spins, wasting cycles, it consumes CPU that
 253         // a mutator might otherwise use profitably to reach a safepoint.  Excessive
 254         // spinning by the VM thread on a saturated system can increase rendezvous latency.
 255         // Blocking or yielding incur their own penalties in the form of context switching
 256         // and the resultant loss of $ residency.
 257         //
 258         // Further complicating matters is that yield() does not work as naively expected
 259         // on many platforms -- yield() does not guarantee that any other ready threads
 260         // will run.   As such we revert to naked_short_sleep() after some number of iterations.
 261         // nakes_short_sleep() is implemented as a short unconditional sleep.
 262         // Typical operating systems round a "short" sleep period up to 10 msecs, so sleeping
 263         // can actually increase the time it takes the VM thread to detect that a system-wide
 264         // stop-the-world safepoint has been reached.  In a pathological scenario such as that
 265         // described in CR6415670 the VMthread may sleep just before the mutator(s) become safe.
 266         // In that case the mutators will be stalled waiting for the safepoint to complete and the
 267         // the VMthread will be sleeping, waiting for the mutators to rendezvous.  The VMthread
 268         // will eventually wake up and detect that all mutators are safe, at which point
 269         // we'll again make progress.
 270         //
 271         // Beware too that that the VMThread typically runs at elevated priority.
 272         // Its default priority is higher than the default mutator priority.
 273         // Obviously, this complicates spinning.
 274         //
 275         // Note too that on Windows XP SwitchThreadTo() has quite different behavior than Sleep(0).
 276         // Sleep(0) will _not yield to lower priority threads, while SwitchThreadTo() will.
 277         //
 278         // See the comments in synchronizer.cpp for additional remarks on spinning.
 279         //
 280         // In the future we might:
 281         // 1. Modify the safepoint scheme to avoid potentially unbounded spinning.
 282         //    This is tricky as the path used by a thread exiting the JVM (say on
 283         //    on JNI call-out) simply stores into its state field.  The burden
 284         //    is placed on the VM thread, which must poll (spin).
 285         // 2. Find something useful to do while spinning.  If the safepoint is GC-related
 286         //    we might aggressively scan the stacks of threads that are already safe.
 287         // 3. Use Solaris schedctl to examine the state of the still-running mutators.
 288         //    If all the mutators are ONPROC there's no reason to sleep or yield.
 289         // 4. YieldTo() any still-running mutators that are ready but OFFPROC.
 290         // 5. Check system saturation.  If the system is not fully saturated then
 291         //    simply spin and avoid sleep/yield.
 292         // 6. As still-running mutators rendezvous they could unpark the sleeping
 293         //    VMthread.  This works well for still-running mutators that become
 294         //    safe.  The VMthread must still poll for mutators that call-out.
 295         // 7. Drive the policy on time-since-begin instead of iterations.
 296         // 8. Consider making the spin duration a function of the # of CPUs:
 297         //    Spin = (((ncpus-1) * M) + K) + F(still_running)
 298         //    Alternately, instead of counting iterations of the outer loop
 299         //    we could count the # of threads visited in the inner loop, above.
 300         // 9. On windows consider using the return value from SwitchThreadTo()
 301         //    to drive subsequent spin/SwitchThreadTo()/Sleep(N) decisions.
 302 
 303         if (int(iterations) == DeferPollingPageLoopCount) {
 304           guarantee (PageArmed == 0, "invariant") ;
 305           PageArmed = 1 ;
 306           os::make_polling_page_unreadable();
 307         }
 308 
 309         // Instead of (ncpus > 1) consider either (still_running < (ncpus + EPSILON)) or
 310         // ((still_running + _waiting_to_block - TryingToBlock)) < ncpus)
 311         ++steps ;
 312         if (ncpus > 1 && steps < SafepointSpinBeforeYield) {
 313           SpinPause() ;     // MP-Polite spin
 314         } else
 315           if (steps < DeferThrSuspendLoopCount) {
 316             os::naked_yield() ;
 317           } else {
 318             os::naked_short_sleep(1);
 319           }
 320 
 321         iterations ++ ;
 322       }
 323       assert(iterations < (uint)max_jint, "We have been iterating in the safepoint loop too long");
 324     }
 325     assert(still_running == 0, "sanity check");
 326 
 327     if (PrintSafepointStatistics) {
 328       update_statistics_on_spin_end();
 329     }
 330 
 331     if (sync_event.should_commit()) {
 332       sync_event.set_safepointId(safepoint_counter());
 333       sync_event.set_initialThreadCount(initial_running);
 334       sync_event.set_runningThreadCount(_waiting_to_block);
 335       sync_event.set_iterations(iterations);
 336       sync_event.commit();
 337     }
 338   } //EventSafepointStateSync
 339 
 340   // wait until all threads are stopped
 341   {
 342     EventSafepointWaitBlocked wait_blocked_event;
 343     int initial_waiting_to_block = _waiting_to_block;
 344 
 345     while (_waiting_to_block > 0) {
 346       log_debug(safepoint)("Waiting for %d thread(s) to block", _waiting_to_block);
 347       if (!SafepointTimeout || timeout_error_printed) {
 348         Safepoint_lock->wait(true);  // true, means with no safepoint checks
 349       } else {
 350         // Compute remaining time
 351         jlong remaining_time = safepoint_limit_time - os::javaTimeNanos();
 352 
 353         // If there is no remaining time, then there is an error
 354         if (remaining_time < 0 || Safepoint_lock->wait(true, remaining_time / MICROUNITS)) {
 355           print_safepoint_timeout(_blocking_timeout);
 356         }
 357       }
 358     }
 359     assert(_waiting_to_block == 0, "sanity check");
 360 
 361 #ifndef PRODUCT
 362     if (SafepointTimeout) {
 363       jlong current_time = os::javaTimeNanos();
 364       if (safepoint_limit_time < current_time) {
 365         tty->print_cr("# SafepointSynchronize: Finished after "
 366                       INT64_FORMAT_W(6) " ms",
 367                       ((current_time - safepoint_limit_time) / MICROUNITS +
 368                        (jlong)SafepointTimeoutDelay));
 369       }
 370     }
 371 #endif
 372 
 373     assert((_safepoint_counter & 0x1) == 0, "must be even");
 374     assert(Threads_lock->owned_by_self(), "must hold Threads_lock");
 375     _safepoint_counter ++;
 376 
 377     // Record state
 378     _state = _synchronized;
 379 
 380     OrderAccess::fence();
 381 
 382     if (wait_blocked_event.should_commit()) {
 383       wait_blocked_event.set_safepointId(safepoint_counter());
 384       wait_blocked_event.set_runningThreadCount(initial_waiting_to_block);
 385       wait_blocked_event.commit();
 386     }
 387   } // EventSafepointWaitBlocked
 388 
 389 #ifdef ASSERT
 390   for (JavaThread *cur = Threads::first(); cur != NULL; cur = cur->next()) {
 391     // make sure all the threads were visited
 392     assert(cur->was_visited_for_critical_count(), "missed a thread");
 393   }
 394 #endif // ASSERT
 395 
 396   // Update the count of active JNI critical regions
 397   GCLocker::set_jni_lock_count(_current_jni_active_count);
 398 
 399   if (log_is_enabled(Debug, safepoint)) {
 400     VM_Operation *op = VMThread::vm_operation();
 401     log_debug(safepoint)("Entering safepoint region: %s",
 402                          (op != NULL) ? op->name() : "no vm operation");
 403   }
 404 
 405   RuntimeService::record_safepoint_synchronized();
 406   if (PrintSafepointStatistics) {
 407     update_statistics_on_sync_end(os::javaTimeNanos());
 408   }
 409 
 410   // Call stuff that needs to be run when a safepoint is just about to be completed
 411   {
 412     EventSafepointCleanup cleanup_event;
 413     do_cleanup_tasks();
 414     if (cleanup_event.should_commit()) {
 415       cleanup_event.set_safepointId(safepoint_counter());
 416       cleanup_event.commit();
 417     }
 418   }
 419 
 420   if (PrintSafepointStatistics) {
 421     // Record how much time spend on the above cleanup tasks
 422     update_statistics_on_cleanup_end(os::javaTimeNanos());
 423   }
 424   if (begin_event.should_commit()) {
 425     begin_event.set_safepointId(safepoint_counter());
 426     begin_event.set_totalThreadCount(nof_threads);
 427     begin_event.set_jniCriticalThreadCount(_current_jni_active_count);
 428     begin_event.commit();
 429   }
 430 }
 431 
 432 // Wake up all threads, so they are ready to resume execution after the safepoint
 433 // operation has been carried out
 434 void SafepointSynchronize::end() {
 435   EventSafepointEnd event;
 436   int safepoint_id = safepoint_counter(); // Keep the odd counter as "id"
 437 
 438   assert(Threads_lock->owned_by_self(), "must hold Threads_lock");
 439   assert((_safepoint_counter & 0x1) == 1, "must be odd");
 440   _safepoint_counter ++;
 441   // memory fence isn't required here since an odd _safepoint_counter
 442   // value can do no harm and a fence is issued below anyway.
 443 
 444   DEBUG_ONLY(Thread* myThread = Thread::current();)
 445   assert(myThread->is_VM_thread(), "Only VM thread can execute a safepoint");
 446 
 447   if (PrintSafepointStatistics) {
 448     end_statistics(os::javaTimeNanos());
 449   }
 450 
 451 #ifdef ASSERT
 452   // A pending_exception cannot be installed during a safepoint.  The threads
 453   // may install an async exception after they come back from a safepoint into
 454   // pending_exception after they unblock.  But that should happen later.
 455   for(JavaThread *cur = Threads::first(); cur; cur = cur->next()) {
 456     assert (!(cur->has_pending_exception() &&
 457               cur->safepoint_state()->is_at_poll_safepoint()),
 458             "safepoint installed a pending exception");
 459   }
 460 #endif // ASSERT
 461 
 462   if (PageArmed) {
 463     // Make polling safepoint aware
 464     os::make_polling_page_readable();
 465     PageArmed = 0 ;
 466   }
 467 
 468   // Remove safepoint check from interpreter
 469   Interpreter::ignore_safepoints();
 470 
 471   {
 472     MutexLocker mu(Safepoint_lock);
 473 
 474     assert(_state == _synchronized, "must be synchronized before ending safepoint synchronization");
 475 
 476     // Set to not synchronized, so the threads will not go into the signal_thread_blocked method
 477     // when they get restarted.
 478     _state = _not_synchronized;
 479     OrderAccess::fence();
 480 
 481     log_debug(safepoint)("Leaving safepoint region");
 482 
 483     // Start suspended threads
 484     for(JavaThread *current = Threads::first(); current; current = current->next()) {
 485       // A problem occurring on Solaris is when attempting to restart threads
 486       // the first #cpus - 1 go well, but then the VMThread is preempted when we get
 487       // to the next one (since it has been running the longest).  We then have
 488       // to wait for a cpu to become available before we can continue restarting
 489       // threads.
 490       // FIXME: This causes the performance of the VM to degrade when active and with
 491       // large numbers of threads.  Apparently this is due to the synchronous nature
 492       // of suspending threads.
 493       //
 494       // TODO-FIXME: the comments above are vestigial and no longer apply.
 495       // Furthermore, using solaris' schedctl in this particular context confers no benefit
 496       if (VMThreadHintNoPreempt) {
 497         os::hint_no_preempt();
 498       }
 499       ThreadSafepointState* cur_state = current->safepoint_state();
 500       assert(cur_state->type() != ThreadSafepointState::_running, "Thread not suspended at safepoint");
 501       cur_state->restart();
 502       assert(cur_state->is_running(), "safepoint state has not been reset");
 503     }
 504 
 505     RuntimeService::record_safepoint_end();
 506 
 507     // Release threads lock, so threads can be created/destroyed again. It will also starts all threads
 508     // blocked in signal_thread_blocked
 509     Threads_lock->unlock();
 510 
 511   }
 512 #if INCLUDE_ALL_GCS
 513   // If there are any concurrent GC threads resume them.
 514   if (UseConcMarkSweepGC) {
 515     ConcurrentMarkSweepThread::desynchronize(false);
 516   } else if (UseG1GC) {
 517     SuspendibleThreadSet::desynchronize();
 518   }
 519 #endif // INCLUDE_ALL_GCS
 520   // record this time so VMThread can keep track how much time has elapsed
 521   // since last safepoint.
 522   _end_of_last_safepoint = os::javaTimeMillis();
 523 
 524   if (event.should_commit()) {
 525     event.set_safepointId(safepoint_id);
 526     event.commit();
 527   }
 528 }
 529 
 530 bool SafepointSynchronize::is_cleanup_needed() {
 531   // Need a safepoint if some inline cache buffers is non-empty
 532   if (!InlineCacheBuffer::is_empty()) return true;
 533   return false;
 534 }
 535 
 536 static void event_safepoint_cleanup_task_commit(EventSafepointCleanupTask& event, const char* name) {
 537   if (event.should_commit()) {
 538     event.set_safepointId(SafepointSynchronize::safepoint_counter());
 539     event.set_name(name);
 540     event.commit();
 541   }
 542 }
 543 
 544 // Various cleaning tasks that should be done periodically at safepoints
 545 void SafepointSynchronize::do_cleanup_tasks() {
 546   {
 547     const char* name = "deflating idle monitors";
 548     EventSafepointCleanupTask event;
 549     TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup));
 550     ObjectSynchronizer::deflate_idle_monitors();
 551     event_safepoint_cleanup_task_commit(event, name);
 552   }
 553 
 554   {
 555     const char* name = "updating inline caches";
 556     EventSafepointCleanupTask event;
 557     TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup));
 558     InlineCacheBuffer::update_inline_caches();
 559     event_safepoint_cleanup_task_commit(event, name);
 560   }
 561   {
 562     const char* name = "compilation policy safepoint handler";
 563     EventSafepointCleanupTask event;
 564     TraceTime timer("compilation policy safepoint handler", TRACETIME_LOG(Info, safepoint, cleanup));
 565     CompilationPolicy::policy()->do_safepoint_work();
 566     event_safepoint_cleanup_task_commit(event, name);
 567   }
 568 
 569   {
 570     const char* name = "mark nmethods";
 571     EventSafepointCleanupTask event;
 572     TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup));
 573     NMethodSweeper::mark_active_nmethods();
 574     event_safepoint_cleanup_task_commit(event, name);
 575   }
 576 
 577   if (SymbolTable::needs_rehashing()) {
 578     const char* name = "rehashing symbol table";
 579     EventSafepointCleanupTask event;
 580     TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup));
 581     SymbolTable::rehash_table();
 582     event_safepoint_cleanup_task_commit(event, name);
 583   }
 584 
 585   if (StringTable::needs_rehashing()) {
 586     const char* name = "rehashing string table";
 587     EventSafepointCleanupTask event;
 588     TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup));
 589     StringTable::rehash_table();
 590     event_safepoint_cleanup_task_commit(event, name);
 591   }
 592 
 593   {
 594     // CMS delays purging the CLDG until the beginning of the next safepoint and to
 595     // make sure concurrent sweep is done
 596     const char* name = "purging class loader data graph";
 597     EventSafepointCleanupTask event;
 598     TraceTime timer(name, TRACETIME_LOG(Info, safepoint, cleanup));
 599     ClassLoaderDataGraph::purge_if_needed();
 600     event_safepoint_cleanup_task_commit(event, name);
 601   }
 602 }
 603 
 604 
 605 bool SafepointSynchronize::safepoint_safe(JavaThread *thread, JavaThreadState state) {
 606   switch(state) {
 607   case _thread_in_native:
 608     // native threads are safe if they have no java stack or have walkable stack
 609     return !thread->has_last_Java_frame() || thread->frame_anchor()->walkable();
 610 
 611    // blocked threads should have already have walkable stack
 612   case _thread_blocked:
 613     assert(!thread->has_last_Java_frame() || thread->frame_anchor()->walkable(), "blocked and not walkable");
 614     return true;
 615 
 616   default:
 617     return false;
 618   }
 619 }
 620 
 621 
 622 // See if the thread is running inside a lazy critical native and
 623 // update the thread critical count if so.  Also set a suspend flag to
 624 // cause the native wrapper to return into the JVM to do the unlock
 625 // once the native finishes.
 626 void SafepointSynchronize::check_for_lazy_critical_native(JavaThread *thread, JavaThreadState state) {
 627   if (state == _thread_in_native &&
 628       thread->has_last_Java_frame() &&
 629       thread->frame_anchor()->walkable()) {
 630     // This thread might be in a critical native nmethod so look at
 631     // the top of the stack and increment the critical count if it
 632     // is.
 633     frame wrapper_frame = thread->last_frame();
 634     CodeBlob* stub_cb = wrapper_frame.cb();
 635     if (stub_cb != NULL &&
 636         stub_cb->is_nmethod() &&
 637         stub_cb->as_nmethod_or_null()->is_lazy_critical_native()) {
 638       // A thread could potentially be in a critical native across
 639       // more than one safepoint, so only update the critical state on
 640       // the first one.  When it returns it will perform the unlock.
 641       if (!thread->do_critical_native_unlock()) {
 642 #ifdef ASSERT
 643         if (!thread->in_critical()) {
 644           GCLocker::increment_debug_jni_lock_count();
 645         }
 646 #endif
 647         thread->enter_critical();
 648         // Make sure the native wrapper calls back on return to
 649         // perform the needed critical unlock.
 650         thread->set_critical_native_unlock();
 651       }
 652     }
 653   }
 654 }
 655 
 656 
 657 
 658 // -------------------------------------------------------------------------------------------------------
 659 // Implementation of Safepoint callback point
 660 
 661 void SafepointSynchronize::block(JavaThread *thread) {
 662   assert(thread != NULL, "thread must be set");
 663   assert(thread->is_Java_thread(), "not a Java thread");
 664 
 665   // Threads shouldn't block if they are in the middle of printing, but...
 666   ttyLocker::break_tty_lock_for_safepoint(os::current_thread_id());
 667 
 668   // Only bail from the block() call if the thread is gone from the
 669   // thread list; starting to exit should still block.
 670   if (thread->is_terminated()) {
 671      // block current thread if we come here from native code when VM is gone
 672      thread->block_if_vm_exited();
 673 
 674      // otherwise do nothing
 675      return;
 676   }
 677 
 678   JavaThreadState state = thread->thread_state();
 679   thread->frame_anchor()->make_walkable(thread);
 680 
 681   // Check that we have a valid thread_state at this point
 682   switch(state) {
 683     case _thread_in_vm_trans:
 684     case _thread_in_Java:        // From compiled code
 685 
 686       // We are highly likely to block on the Safepoint_lock. In order to avoid blocking in this case,
 687       // we pretend we are still in the VM.
 688       thread->set_thread_state(_thread_in_vm);
 689 
 690       if (is_synchronizing()) {
 691          Atomic::inc (&TryingToBlock) ;
 692       }
 693 
 694       // We will always be holding the Safepoint_lock when we are examine the state
 695       // of a thread. Hence, the instructions between the Safepoint_lock->lock() and
 696       // Safepoint_lock->unlock() are happening atomic with regards to the safepoint code
 697       Safepoint_lock->lock_without_safepoint_check();
 698       if (is_synchronizing()) {
 699         // Decrement the number of threads to wait for and signal vm thread
 700         assert(_waiting_to_block > 0, "sanity check");
 701         _waiting_to_block--;
 702         thread->safepoint_state()->set_has_called_back(true);
 703 
 704         DEBUG_ONLY(thread->set_visited_for_critical_count(true));
 705         if (thread->in_critical()) {
 706           // Notice that this thread is in a critical section
 707           increment_jni_active_count();
 708         }
 709 
 710         // Consider (_waiting_to_block < 2) to pipeline the wakeup of the VM thread
 711         if (_waiting_to_block == 0) {
 712           Safepoint_lock->notify_all();
 713         }
 714       }
 715 
 716       // We transition the thread to state _thread_blocked here, but
 717       // we can't do our usual check for external suspension and then
 718       // self-suspend after the lock_without_safepoint_check() call
 719       // below because we are often called during transitions while
 720       // we hold different locks. That would leave us suspended while
 721       // holding a resource which results in deadlocks.
 722       thread->set_thread_state(_thread_blocked);
 723       Safepoint_lock->unlock();
 724 
 725       // We now try to acquire the threads lock. Since this lock is hold by the VM thread during
 726       // the entire safepoint, the threads will all line up here during the safepoint.
 727       Threads_lock->lock_without_safepoint_check();
 728       // restore original state. This is important if the thread comes from compiled code, so it
 729       // will continue to execute with the _thread_in_Java state.
 730       thread->set_thread_state(state);
 731       Threads_lock->unlock();
 732       break;
 733 
 734     case _thread_in_native_trans:
 735     case _thread_blocked_trans:
 736     case _thread_new_trans:
 737       if (thread->safepoint_state()->type() == ThreadSafepointState::_call_back) {
 738         thread->print_thread_state();
 739         fatal("Deadlock in safepoint code.  "
 740               "Should have called back to the VM before blocking.");
 741       }
 742 
 743       // We transition the thread to state _thread_blocked here, but
 744       // we can't do our usual check for external suspension and then
 745       // self-suspend after the lock_without_safepoint_check() call
 746       // below because we are often called during transitions while
 747       // we hold different locks. That would leave us suspended while
 748       // holding a resource which results in deadlocks.
 749       thread->set_thread_state(_thread_blocked);
 750 
 751       // It is not safe to suspend a thread if we discover it is in _thread_in_native_trans. Hence,
 752       // the safepoint code might still be waiting for it to block. We need to change the state here,
 753       // so it can see that it is at a safepoint.
 754 
 755       // Block until the safepoint operation is completed.
 756       Threads_lock->lock_without_safepoint_check();
 757 
 758       // Restore state
 759       thread->set_thread_state(state);
 760 
 761       Threads_lock->unlock();
 762       break;
 763 
 764     default:
 765      fatal("Illegal threadstate encountered: %d", state);
 766   }
 767 
 768   // Check for pending. async. exceptions or suspends - except if the
 769   // thread was blocked inside the VM. has_special_runtime_exit_condition()
 770   // is called last since it grabs a lock and we only want to do that when
 771   // we must.
 772   //
 773   // Note: we never deliver an async exception at a polling point as the
 774   // compiler may not have an exception handler for it. The polling
 775   // code will notice the async and deoptimize and the exception will
 776   // be delivered. (Polling at a return point is ok though). Sure is
 777   // a lot of bother for a deprecated feature...
 778   //
 779   // We don't deliver an async exception if the thread state is
 780   // _thread_in_native_trans so JNI functions won't be called with
 781   // a surprising pending exception. If the thread state is going back to java,
 782   // async exception is checked in check_special_condition_for_native_trans().
 783 
 784   if (state != _thread_blocked_trans &&
 785       state != _thread_in_vm_trans &&
 786       thread->has_special_runtime_exit_condition()) {
 787     thread->handle_special_runtime_exit_condition(
 788       !thread->is_at_poll_safepoint() && (state != _thread_in_native_trans));
 789   }
 790 }
 791 
 792 // ------------------------------------------------------------------------------------------------------
 793 // Exception handlers
 794 
 795 
 796 void SafepointSynchronize::handle_polling_page_exception(JavaThread *thread) {
 797   assert(thread->is_Java_thread(), "polling reference encountered by VM thread");
 798   assert(thread->thread_state() == _thread_in_Java, "should come from Java code");
 799   assert(SafepointSynchronize::is_synchronizing(), "polling encountered outside safepoint synchronization");
 800 
 801   if (ShowSafepointMsgs) {
 802     tty->print("handle_polling_page_exception: ");
 803   }
 804 
 805   if (PrintSafepointStatistics) {
 806     inc_page_trap_count();
 807   }
 808 
 809   ThreadSafepointState* state = thread->safepoint_state();
 810 
 811   state->handle_polling_page_exception();
 812 }
 813 
 814 
 815 void SafepointSynchronize::print_safepoint_timeout(SafepointTimeoutReason reason) {
 816   if (!timeout_error_printed) {
 817     timeout_error_printed = true;
 818     // Print out the thread info which didn't reach the safepoint for debugging
 819     // purposes (useful when there are lots of threads in the debugger).
 820     tty->cr();
 821     tty->print_cr("# SafepointSynchronize::begin: Timeout detected:");
 822     if (reason ==  _spinning_timeout) {
 823       tty->print_cr("# SafepointSynchronize::begin: Timed out while spinning to reach a safepoint.");
 824     } else if (reason == _blocking_timeout) {
 825       tty->print_cr("# SafepointSynchronize::begin: Timed out while waiting for threads to stop.");
 826     }
 827 
 828     tty->print_cr("# SafepointSynchronize::begin: Threads which did not reach the safepoint:");
 829     ThreadSafepointState *cur_state;
 830     ResourceMark rm;
 831     for(JavaThread *cur_thread = Threads::first(); cur_thread;
 832         cur_thread = cur_thread->next()) {
 833       cur_state = cur_thread->safepoint_state();
 834 
 835       if (cur_thread->thread_state() != _thread_blocked &&
 836           ((reason == _spinning_timeout && cur_state->is_running()) ||
 837            (reason == _blocking_timeout && !cur_state->has_called_back()))) {
 838         tty->print("# ");
 839         cur_thread->print();
 840         tty->cr();
 841       }
 842     }
 843     tty->print_cr("# SafepointSynchronize::begin: (End of list)");
 844   }
 845 
 846   // To debug the long safepoint, specify both DieOnSafepointTimeout &
 847   // ShowMessageBoxOnError.
 848   if (DieOnSafepointTimeout) {
 849     VM_Operation *op = VMThread::vm_operation();
 850     fatal("Safepoint sync time longer than " INTX_FORMAT "ms detected when executing %s.",
 851           SafepointTimeoutDelay,
 852           op != NULL ? op->name() : "no vm operation");
 853   }
 854 }
 855 
 856 
 857 // -------------------------------------------------------------------------------------------------------
 858 // Implementation of ThreadSafepointState
 859 
 860 ThreadSafepointState::ThreadSafepointState(JavaThread *thread) {
 861   _thread = thread;
 862   _type   = _running;
 863   _has_called_back = false;
 864   _at_poll_safepoint = false;
 865 }
 866 
 867 void ThreadSafepointState::create(JavaThread *thread) {
 868   ThreadSafepointState *state = new ThreadSafepointState(thread);
 869   thread->set_safepoint_state(state);
 870 }
 871 
 872 void ThreadSafepointState::destroy(JavaThread *thread) {
 873   if (thread->safepoint_state()) {
 874     delete(thread->safepoint_state());
 875     thread->set_safepoint_state(NULL);
 876   }
 877 }
 878 
 879 void ThreadSafepointState::examine_state_of_thread() {
 880   assert(is_running(), "better be running or just have hit safepoint poll");
 881 
 882   JavaThreadState state = _thread->thread_state();
 883 
 884   // Save the state at the start of safepoint processing.
 885   _orig_thread_state = state;
 886 
 887   // Check for a thread that is suspended. Note that thread resume tries
 888   // to grab the Threads_lock which we own here, so a thread cannot be
 889   // resumed during safepoint synchronization.
 890 
 891   // We check to see if this thread is suspended without locking to
 892   // avoid deadlocking with a third thread that is waiting for this
 893   // thread to be suspended. The third thread can notice the safepoint
 894   // that we're trying to start at the beginning of its SR_lock->wait()
 895   // call. If that happens, then the third thread will block on the
 896   // safepoint while still holding the underlying SR_lock. We won't be
 897   // able to get the SR_lock and we'll deadlock.
 898   //
 899   // We don't need to grab the SR_lock here for two reasons:
 900   // 1) The suspend flags are both volatile and are set with an
 901   //    Atomic::cmpxchg() call so we should see the suspended
 902   //    state right away.
 903   // 2) We're being called from the safepoint polling loop; if
 904   //    we don't see the suspended state on this iteration, then
 905   //    we'll come around again.
 906   //
 907   bool is_suspended = _thread->is_ext_suspended();
 908   if (is_suspended) {
 909     roll_forward(_at_safepoint);
 910     return;
 911   }
 912 
 913   // Some JavaThread states have an initial safepoint state of
 914   // running, but are actually at a safepoint. We will happily
 915   // agree and update the safepoint state here.
 916   if (SafepointSynchronize::safepoint_safe(_thread, state)) {
 917     SafepointSynchronize::check_for_lazy_critical_native(_thread, state);
 918     roll_forward(_at_safepoint);
 919     return;
 920   }
 921 
 922   if (state == _thread_in_vm) {
 923     roll_forward(_call_back);
 924     return;
 925   }
 926 
 927   // All other thread states will continue to run until they
 928   // transition and self-block in state _blocked
 929   // Safepoint polling in compiled code causes the Java threads to do the same.
 930   // Note: new threads may require a malloc so they must be allowed to finish
 931 
 932   assert(is_running(), "examine_state_of_thread on non-running thread");
 933   return;
 934 }
 935 
 936 // Returns true is thread could not be rolled forward at present position.
 937 void ThreadSafepointState::roll_forward(suspend_type type) {
 938   _type = type;
 939 
 940   switch(_type) {
 941     case _at_safepoint:
 942       SafepointSynchronize::signal_thread_at_safepoint();
 943       DEBUG_ONLY(_thread->set_visited_for_critical_count(true));
 944       if (_thread->in_critical()) {
 945         // Notice that this thread is in a critical section
 946         SafepointSynchronize::increment_jni_active_count();
 947       }
 948       break;
 949 
 950     case _call_back:
 951       set_has_called_back(false);
 952       break;
 953 
 954     case _running:
 955     default:
 956       ShouldNotReachHere();
 957   }
 958 }
 959 
 960 void ThreadSafepointState::restart() {
 961   switch(type()) {
 962     case _at_safepoint:
 963     case _call_back:
 964       break;
 965 
 966     case _running:
 967     default:
 968        tty->print_cr("restart thread " INTPTR_FORMAT " with state %d",
 969                      p2i(_thread), _type);
 970        _thread->print();
 971       ShouldNotReachHere();
 972   }
 973   _type = _running;
 974   set_has_called_back(false);
 975 }
 976 
 977 
 978 void ThreadSafepointState::print_on(outputStream *st) const {
 979   const char *s = NULL;
 980 
 981   switch(_type) {
 982     case _running                : s = "_running";              break;
 983     case _at_safepoint           : s = "_at_safepoint";         break;
 984     case _call_back              : s = "_call_back";            break;
 985     default:
 986       ShouldNotReachHere();
 987   }
 988 
 989   st->print_cr("Thread: " INTPTR_FORMAT
 990               "  [0x%2x] State: %s _has_called_back %d _at_poll_safepoint %d",
 991                p2i(_thread), _thread->osthread()->thread_id(), s, _has_called_back,
 992                _at_poll_safepoint);
 993 
 994   _thread->print_thread_state_on(st);
 995 }
 996 
 997 // ---------------------------------------------------------------------------------------------------------------------
 998 
 999 // Block the thread at the safepoint poll or poll return.
1000 void ThreadSafepointState::handle_polling_page_exception() {
1001 
1002   // Check state.  block() will set thread state to thread_in_vm which will
1003   // cause the safepoint state _type to become _call_back.
1004   assert(type() == ThreadSafepointState::_running,
1005          "polling page exception on thread not running state");
1006 
1007   // Step 1: Find the nmethod from the return address
1008   if (ShowSafepointMsgs && Verbose) {
1009     tty->print_cr("Polling page exception at " INTPTR_FORMAT, p2i(thread()->saved_exception_pc()));
1010   }
1011   address real_return_addr = thread()->saved_exception_pc();
1012 
1013   CodeBlob *cb = CodeCache::find_blob(real_return_addr);
1014   assert(cb != NULL && cb->is_compiled(), "return address should be in nmethod");
1015   CompiledMethod* nm = (CompiledMethod*)cb;
1016 
1017   // Find frame of caller
1018   frame stub_fr = thread()->last_frame();
1019   CodeBlob* stub_cb = stub_fr.cb();
1020   assert(stub_cb->is_safepoint_stub(), "must be a safepoint stub");
1021   RegisterMap map(thread(), true);
1022   frame caller_fr = stub_fr.sender(&map);
1023 
1024   // Should only be poll_return or poll
1025   assert( nm->is_at_poll_or_poll_return(real_return_addr), "should not be at call" );
1026 
1027   // This is a poll immediately before a return. The exception handling code
1028   // has already had the effect of causing the return to occur, so the execution
1029   // will continue immediately after the call. In addition, the oopmap at the
1030   // return point does not mark the return value as an oop (if it is), so
1031   // it needs a handle here to be updated.
1032   if( nm->is_at_poll_return(real_return_addr) ) {
1033     ResourceMark rm;
1034     // See if return type is an oop.
1035     Method* method = nm->method();
1036     bool return_oop = method->is_returning_oop();
1037     
1038     GrowableArray<Handle> return_values;
1039     ValueKlass* vk = NULL;
1040     if (!return_oop && method->is_returning_vt()) {
1041       // We're at a safepoint at the return of a method that returns
1042       // multiple values. We must make sure we preserve the oop values
1043       // across the safepoint.
1044       vk = ValueKlass::returned_value_type(map);
1045       assert(vk == NULL || vk == method->returned_value_type(thread()) ||
1046              method->returned_value_type(thread()) == SystemDictionary::___Value_klass(), "Bad value klass");
1047       if (vk != NULL && !vk->save_oop_results(map, return_values)) {
1048         return_oop = true;
1049         vk = NULL;
1050       }
1051     }
1052 
1053     if (return_oop) {
1054       // The oop result has been saved on the stack together with all
1055       // the other registers. In order to preserve it over GCs we need
1056       // to keep it in a handle.
1057       oop result = caller_fr.saved_oop_result(&map);
1058       assert(result == NULL || result->is_oop(), "must be oop");
1059       return_values.push(Handle(thread(), result));
1060       assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
1061     }
1062 
1063     // Block the thread
1064     SafepointSynchronize::block(thread());
1065 
1066     // restore oop result, if any
1067     if (return_oop) {
1068       assert(return_values.length() == 1, "only one return value");
1069       caller_fr.set_saved_oop_result(&map, return_values.pop()());
1070     } else if (vk != NULL) {
1071       vk->restore_oop_results(map, return_values);
1072     }
1073   }
1074 
1075   // This is a safepoint poll. Verify the return address and block.
1076   else {
1077     set_at_poll_safepoint(true);
1078 
1079     // verify the blob built the "return address" correctly
1080     assert(real_return_addr == caller_fr.pc(), "must match");
1081 
1082     // Block the thread
1083     SafepointSynchronize::block(thread());
1084     set_at_poll_safepoint(false);
1085 
1086     // If we have a pending async exception deoptimize the frame
1087     // as otherwise we may never deliver it.
1088     if (thread()->has_async_condition()) {
1089       ThreadInVMfromJavaNoAsyncException __tiv(thread());
1090       Deoptimization::deoptimize_frame(thread(), caller_fr.id());
1091     }
1092 
1093     // If an exception has been installed we must check for a pending deoptimization
1094     // Deoptimize frame if exception has been thrown.
1095 
1096     if (thread()->has_pending_exception() ) {
1097       RegisterMap map(thread(), true);
1098       frame caller_fr = stub_fr.sender(&map);
1099       if (caller_fr.is_deoptimized_frame()) {
1100         // The exception patch will destroy registers that are still
1101         // live and will be needed during deoptimization. Defer the
1102         // Async exception should have deferred the exception until the
1103         // next safepoint which will be detected when we get into
1104         // the interpreter so if we have an exception now things
1105         // are messed up.
1106 
1107         fatal("Exception installed and deoptimization is pending");
1108       }
1109     }
1110   }
1111 }
1112 
1113 
1114 //
1115 //                     Statistics & Instrumentations
1116 //
1117 SafepointSynchronize::SafepointStats*  SafepointSynchronize::_safepoint_stats = NULL;
1118 jlong  SafepointSynchronize::_safepoint_begin_time = 0;
1119 int    SafepointSynchronize::_cur_stat_index = 0;
1120 julong SafepointSynchronize::_safepoint_reasons[VM_Operation::VMOp_Terminating];
1121 julong SafepointSynchronize::_coalesced_vmop_count = 0;
1122 jlong  SafepointSynchronize::_max_sync_time = 0;
1123 jlong  SafepointSynchronize::_max_vmop_time = 0;
1124 float  SafepointSynchronize::_ts_of_current_safepoint = 0.0f;
1125 
1126 static jlong  cleanup_end_time = 0;
1127 static bool   need_to_track_page_armed_status = false;
1128 static bool   init_done = false;
1129 
1130 // Helper method to print the header.
1131 static void print_header() {
1132   tty->print("         vmop                    "
1133              "[threads: total initially_running wait_to_block]    ");
1134   tty->print("[time: spin block sync cleanup vmop] ");
1135 
1136   // no page armed status printed out if it is always armed.
1137   if (need_to_track_page_armed_status) {
1138     tty->print("page_armed ");
1139   }
1140 
1141   tty->print_cr("page_trap_count");
1142 }
1143 
1144 void SafepointSynchronize::deferred_initialize_stat() {
1145   if (init_done) return;
1146 
1147   // If PrintSafepointStatisticsTimeout is specified, the statistics data will
1148   // be printed right away, in which case, _safepoint_stats will regress to
1149   // a single element array. Otherwise, it is a circular ring buffer with default
1150   // size of PrintSafepointStatisticsCount.
1151   int stats_array_size;
1152   if (PrintSafepointStatisticsTimeout > 0) {
1153     stats_array_size = 1;
1154     PrintSafepointStatistics = true;
1155   } else {
1156     stats_array_size = PrintSafepointStatisticsCount;
1157   }
1158   _safepoint_stats = (SafepointStats*)os::malloc(stats_array_size
1159                                                  * sizeof(SafepointStats), mtInternal);
1160   guarantee(_safepoint_stats != NULL,
1161             "not enough memory for safepoint instrumentation data");
1162 
1163   if (DeferPollingPageLoopCount >= 0) {
1164     need_to_track_page_armed_status = true;
1165   }
1166   init_done = true;
1167 }
1168 
1169 void SafepointSynchronize::begin_statistics(int nof_threads, int nof_running) {
1170   assert(init_done, "safepoint statistics array hasn't been initialized");
1171   SafepointStats *spstat = &_safepoint_stats[_cur_stat_index];
1172 
1173   spstat->_time_stamp = _ts_of_current_safepoint;
1174 
1175   VM_Operation *op = VMThread::vm_operation();
1176   spstat->_vmop_type = (op != NULL ? op->type() : -1);
1177   if (op != NULL) {
1178     _safepoint_reasons[spstat->_vmop_type]++;
1179   }
1180 
1181   spstat->_nof_total_threads = nof_threads;
1182   spstat->_nof_initial_running_threads = nof_running;
1183   spstat->_nof_threads_hit_page_trap = 0;
1184 
1185   // Records the start time of spinning. The real time spent on spinning
1186   // will be adjusted when spin is done. Same trick is applied for time
1187   // spent on waiting for threads to block.
1188   if (nof_running != 0) {
1189     spstat->_time_to_spin = os::javaTimeNanos();
1190   }  else {
1191     spstat->_time_to_spin = 0;
1192   }
1193 }
1194 
1195 void SafepointSynchronize::update_statistics_on_spin_end() {
1196   SafepointStats *spstat = &_safepoint_stats[_cur_stat_index];
1197 
1198   jlong cur_time = os::javaTimeNanos();
1199 
1200   spstat->_nof_threads_wait_to_block = _waiting_to_block;
1201   if (spstat->_nof_initial_running_threads != 0) {
1202     spstat->_time_to_spin = cur_time - spstat->_time_to_spin;
1203   }
1204 
1205   if (need_to_track_page_armed_status) {
1206     spstat->_page_armed = (PageArmed == 1);
1207   }
1208 
1209   // Records the start time of waiting for to block. Updated when block is done.
1210   if (_waiting_to_block != 0) {
1211     spstat->_time_to_wait_to_block = cur_time;
1212   } else {
1213     spstat->_time_to_wait_to_block = 0;
1214   }
1215 }
1216 
1217 void SafepointSynchronize::update_statistics_on_sync_end(jlong end_time) {
1218   SafepointStats *spstat = &_safepoint_stats[_cur_stat_index];
1219 
1220   if (spstat->_nof_threads_wait_to_block != 0) {
1221     spstat->_time_to_wait_to_block = end_time -
1222       spstat->_time_to_wait_to_block;
1223   }
1224 
1225   // Records the end time of sync which will be used to calculate the total
1226   // vm operation time. Again, the real time spending in syncing will be deducted
1227   // from the start of the sync time later when end_statistics is called.
1228   spstat->_time_to_sync = end_time - _safepoint_begin_time;
1229   if (spstat->_time_to_sync > _max_sync_time) {
1230     _max_sync_time = spstat->_time_to_sync;
1231   }
1232 
1233   spstat->_time_to_do_cleanups = end_time;
1234 }
1235 
1236 void SafepointSynchronize::update_statistics_on_cleanup_end(jlong end_time) {
1237   SafepointStats *spstat = &_safepoint_stats[_cur_stat_index];
1238 
1239   // Record how long spent in cleanup tasks.
1240   spstat->_time_to_do_cleanups = end_time - spstat->_time_to_do_cleanups;
1241 
1242   cleanup_end_time = end_time;
1243 }
1244 
1245 void SafepointSynchronize::end_statistics(jlong vmop_end_time) {
1246   SafepointStats *spstat = &_safepoint_stats[_cur_stat_index];
1247 
1248   // Update the vm operation time.
1249   spstat->_time_to_exec_vmop = vmop_end_time -  cleanup_end_time;
1250   if (spstat->_time_to_exec_vmop > _max_vmop_time) {
1251     _max_vmop_time = spstat->_time_to_exec_vmop;
1252   }
1253   // Only the sync time longer than the specified
1254   // PrintSafepointStatisticsTimeout will be printed out right away.
1255   // By default, it is -1 meaning all samples will be put into the list.
1256   if ( PrintSafepointStatisticsTimeout > 0) {
1257     if (spstat->_time_to_sync > (jlong)PrintSafepointStatisticsTimeout * MICROUNITS) {
1258       print_statistics();
1259     }
1260   } else {
1261     // The safepoint statistics will be printed out when the _safepoin_stats
1262     // array fills up.
1263     if (_cur_stat_index == PrintSafepointStatisticsCount - 1) {
1264       print_statistics();
1265       _cur_stat_index = 0;
1266     } else {
1267       _cur_stat_index++;
1268     }
1269   }
1270 }
1271 
1272 void SafepointSynchronize::print_statistics() {
1273   SafepointStats* sstats = _safepoint_stats;
1274 
1275   for (int index = 0; index <= _cur_stat_index; index++) {
1276     if (index % 30 == 0) {
1277       print_header();
1278     }
1279     sstats = &_safepoint_stats[index];
1280     tty->print("%.3f: ", sstats->_time_stamp);
1281     tty->print("%-26s       ["
1282                INT32_FORMAT_W(8) INT32_FORMAT_W(11) INT32_FORMAT_W(15)
1283                "    ]    ",
1284                sstats->_vmop_type == -1 ? "no vm operation" :
1285                VM_Operation::name(sstats->_vmop_type),
1286                sstats->_nof_total_threads,
1287                sstats->_nof_initial_running_threads,
1288                sstats->_nof_threads_wait_to_block);
1289     // "/ MICROUNITS " is to convert the unit from nanos to millis.
1290     tty->print("  ["
1291                INT64_FORMAT_W(6) INT64_FORMAT_W(6)
1292                INT64_FORMAT_W(6) INT64_FORMAT_W(6)
1293                INT64_FORMAT_W(6) "    ]  ",
1294                sstats->_time_to_spin / MICROUNITS,
1295                sstats->_time_to_wait_to_block / MICROUNITS,
1296                sstats->_time_to_sync / MICROUNITS,
1297                sstats->_time_to_do_cleanups / MICROUNITS,
1298                sstats->_time_to_exec_vmop / MICROUNITS);
1299 
1300     if (need_to_track_page_armed_status) {
1301       tty->print(INT32_FORMAT "         ", sstats->_page_armed);
1302     }
1303     tty->print_cr(INT32_FORMAT "   ", sstats->_nof_threads_hit_page_trap);
1304   }
1305 }
1306 
1307 // This method will be called when VM exits. It will first call
1308 // print_statistics to print out the rest of the sampling.  Then
1309 // it tries to summarize the sampling.
1310 void SafepointSynchronize::print_stat_on_exit() {
1311   if (_safepoint_stats == NULL) return;
1312 
1313   SafepointStats *spstat = &_safepoint_stats[_cur_stat_index];
1314 
1315   // During VM exit, end_statistics may not get called and in that
1316   // case, if the sync time is less than PrintSafepointStatisticsTimeout,
1317   // don't print it out.
1318   // Approximate the vm op time.
1319   _safepoint_stats[_cur_stat_index]._time_to_exec_vmop =
1320     os::javaTimeNanos() - cleanup_end_time;
1321 
1322   if ( PrintSafepointStatisticsTimeout < 0 ||
1323        spstat->_time_to_sync > (jlong)PrintSafepointStatisticsTimeout * MICROUNITS) {
1324     print_statistics();
1325   }
1326   tty->cr();
1327 
1328   // Print out polling page sampling status.
1329   if (!need_to_track_page_armed_status) {
1330     tty->print_cr("Polling page always armed");
1331   } else {
1332     tty->print_cr("Defer polling page loop count = " INTX_FORMAT "\n",
1333                   DeferPollingPageLoopCount);
1334   }
1335 
1336   for (int index = 0; index < VM_Operation::VMOp_Terminating; index++) {
1337     if (_safepoint_reasons[index] != 0) {
1338       tty->print_cr("%-26s" UINT64_FORMAT_W(10), VM_Operation::name(index),
1339                     _safepoint_reasons[index]);
1340     }
1341   }
1342 
1343   tty->print_cr(UINT64_FORMAT_W(5) " VM operations coalesced during safepoint",
1344                 _coalesced_vmop_count);
1345   tty->print_cr("Maximum sync time  " INT64_FORMAT_W(5) " ms",
1346                 _max_sync_time / MICROUNITS);
1347   tty->print_cr("Maximum vm operation time (except for Exit VM operation)  "
1348                 INT64_FORMAT_W(5) " ms",
1349                 _max_vmop_time / MICROUNITS);
1350 }
1351 
1352 // ------------------------------------------------------------------------------------------------
1353 // Non-product code
1354 
1355 #ifndef PRODUCT
1356 
1357 void SafepointSynchronize::print_state() {
1358   if (_state == _not_synchronized) {
1359     tty->print_cr("not synchronized");
1360   } else if (_state == _synchronizing || _state == _synchronized) {
1361     tty->print_cr("State: %s", (_state == _synchronizing) ? "synchronizing" :
1362                   "synchronized");
1363 
1364     for(JavaThread *cur = Threads::first(); cur; cur = cur->next()) {
1365        cur->safepoint_state()->print();
1366     }
1367   }
1368 }
1369 
1370 void SafepointSynchronize::safepoint_msg(const char* format, ...) {
1371   if (ShowSafepointMsgs) {
1372     va_list ap;
1373     va_start(ap, format);
1374     tty->vprint_cr(format, ap);
1375     va_end(ap);
1376   }
1377 }
1378 
1379 #endif // !PRODUCT