1 /*
   2  * Copyright (c) 2002, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/parallel/gcTaskManager.hpp"
  27 #include "gc/parallel/gcTaskThread.hpp"
  28 #include "gc/shared/gcId.hpp"
  29 #include "gc/shared/workerManager.hpp"
  30 #include "logging/log.hpp"
  31 #include "memory/allocation.hpp"
  32 #include "memory/allocation.inline.hpp"
  33 #include "memory/resourceArea.hpp"
  34 #include "runtime/mutex.hpp"
  35 #include "runtime/mutexLocker.hpp"
  36 #include "runtime/orderAccess.inline.hpp"
  37 #include "runtime/os.hpp"
  38 
  39 //
  40 // GCTask
  41 //
  42 
  43 const char* GCTask::Kind::to_string(kind value) {
  44   const char* result = "unknown GCTask kind";
  45   switch (value) {
  46   default:
  47     result = "unknown GCTask kind";
  48     break;
  49   case unknown_task:
  50     result = "unknown task";
  51     break;
  52   case ordinary_task:
  53     result = "ordinary task";
  54     break;
  55   case wait_for_barrier_task:
  56     result = "wait for barrier task";
  57     break;
  58   case noop_task:
  59     result = "noop task";
  60     break;
  61   case idle_task:
  62     result = "idle task";
  63     break;
  64   }
  65   return result;
  66 };
  67 
  68 GCTask::GCTask() {
  69   initialize(Kind::ordinary_task, GCId::current());
  70 }
  71 
  72 GCTask::GCTask(Kind::kind kind) {
  73   initialize(kind, GCId::current());
  74 }
  75 
  76 GCTask::GCTask(Kind::kind kind, uint gc_id) {
  77   initialize(kind, gc_id);
  78 }
  79 
  80 void GCTask::initialize(Kind::kind kind, uint gc_id) {
  81   _kind = kind;
  82   _affinity = GCTaskManager::sentinel_worker();
  83   _older = NULL;
  84   _newer = NULL;
  85   _gc_id = gc_id;
  86 }
  87 
  88 void GCTask::destruct() {
  89   assert(older() == NULL, "shouldn't have an older task");
  90   assert(newer() == NULL, "shouldn't have a newer task");
  91   // Nothing to do.
  92 }
  93 
  94 NOT_PRODUCT(
  95 void GCTask::print(const char* message) const {
  96   tty->print(INTPTR_FORMAT " <- " INTPTR_FORMAT "(%u) -> " INTPTR_FORMAT,
  97              p2i(newer()), p2i(this), affinity(), p2i(older()));
  98 }
  99 )
 100 
 101 //
 102 // GCTaskQueue
 103 //
 104 
 105 GCTaskQueue* GCTaskQueue::create() {
 106   GCTaskQueue* result = new GCTaskQueue(false);
 107   if (TraceGCTaskQueue) {
 108     tty->print_cr("GCTaskQueue::create()"
 109                   " returns " INTPTR_FORMAT, p2i(result));
 110   }
 111   return result;
 112 }
 113 
 114 GCTaskQueue* GCTaskQueue::create_on_c_heap() {
 115   GCTaskQueue* result = new(ResourceObj::C_HEAP, mtGC) GCTaskQueue(true);
 116   if (TraceGCTaskQueue) {
 117     tty->print_cr("GCTaskQueue::create_on_c_heap()"
 118                   " returns " INTPTR_FORMAT,
 119                   p2i(result));
 120   }
 121   return result;
 122 }
 123 
 124 GCTaskQueue::GCTaskQueue(bool on_c_heap) :
 125   _is_c_heap_obj(on_c_heap) {
 126   initialize();
 127   if (TraceGCTaskQueue) {
 128     tty->print_cr("[" INTPTR_FORMAT "]"
 129                   " GCTaskQueue::GCTaskQueue() constructor",
 130                   p2i(this));
 131   }
 132 }
 133 
 134 void GCTaskQueue::destruct() {
 135   // Nothing to do.
 136 }
 137 
 138 void GCTaskQueue::destroy(GCTaskQueue* that) {
 139   if (TraceGCTaskQueue) {
 140     tty->print_cr("[" INTPTR_FORMAT "]"
 141                   " GCTaskQueue::destroy()"
 142                   "  is_c_heap_obj:  %s",
 143                   p2i(that),
 144                   that->is_c_heap_obj() ? "true" : "false");
 145   }
 146   // That instance may have been allocated as a CHeapObj,
 147   // in which case we have to free it explicitly.
 148   if (that != NULL) {
 149     that->destruct();
 150     assert(that->is_empty(), "should be empty");
 151     if (that->is_c_heap_obj()) {
 152       FreeHeap(that);
 153     }
 154   }
 155 }
 156 
 157 void GCTaskQueue::initialize() {
 158   set_insert_end(NULL);
 159   set_remove_end(NULL);
 160   set_length(0);
 161 }
 162 
 163 // Enqueue one task.
 164 void GCTaskQueue::enqueue(GCTask* task) {
 165   if (TraceGCTaskQueue) {
 166     tty->print_cr("[" INTPTR_FORMAT "]"
 167                   " GCTaskQueue::enqueue(task: "
 168                   INTPTR_FORMAT ")",
 169                   p2i(this), p2i(task));
 170     print("before:");
 171   }
 172   assert(task != NULL, "shouldn't have null task");
 173   assert(task->older() == NULL, "shouldn't be on queue");
 174   assert(task->newer() == NULL, "shouldn't be on queue");
 175   task->set_newer(NULL);
 176   task->set_older(insert_end());
 177   if (is_empty()) {
 178     set_remove_end(task);
 179   } else {
 180     insert_end()->set_newer(task);
 181   }
 182   set_insert_end(task);
 183   increment_length();
 184   verify_length();
 185   if (TraceGCTaskQueue) {
 186     print("after:");
 187   }
 188 }
 189 
 190 // Enqueue a whole list of tasks.  Empties the argument list.
 191 void GCTaskQueue::enqueue(GCTaskQueue* list) {
 192   if (TraceGCTaskQueue) {
 193     tty->print_cr("[" INTPTR_FORMAT "]"
 194                   " GCTaskQueue::enqueue(list: "
 195                   INTPTR_FORMAT ")",
 196                   p2i(this), p2i(list));
 197     print("before:");
 198     list->print("list:");
 199   }
 200   if (list->is_empty()) {
 201     // Enqueueing the empty list: nothing to do.
 202     return;
 203   }
 204   uint list_length = list->length();
 205   if (is_empty()) {
 206     // Enqueueing to empty list: just acquire elements.
 207     set_insert_end(list->insert_end());
 208     set_remove_end(list->remove_end());
 209     set_length(list_length);
 210   } else {
 211     // Prepend argument list to our queue.
 212     list->remove_end()->set_older(insert_end());
 213     insert_end()->set_newer(list->remove_end());
 214     set_insert_end(list->insert_end());
 215     set_length(length() + list_length);
 216     // empty the argument list.
 217   }
 218   list->initialize();
 219   if (TraceGCTaskQueue) {
 220     print("after:");
 221     list->print("list:");
 222   }
 223   verify_length();
 224 }
 225 
 226 // Dequeue one task.
 227 GCTask* GCTaskQueue::dequeue() {
 228   if (TraceGCTaskQueue) {
 229     tty->print_cr("[" INTPTR_FORMAT "]"
 230                   " GCTaskQueue::dequeue()", p2i(this));
 231     print("before:");
 232   }
 233   assert(!is_empty(), "shouldn't dequeue from empty list");
 234   GCTask* result = remove();
 235   assert(result != NULL, "shouldn't have NULL task");
 236   if (TraceGCTaskQueue) {
 237     tty->print_cr("    return: " INTPTR_FORMAT, p2i(result));
 238     print("after:");
 239   }
 240   return result;
 241 }
 242 
 243 // Dequeue one task, preferring one with affinity.
 244 GCTask* GCTaskQueue::dequeue(uint affinity) {
 245   if (TraceGCTaskQueue) {
 246     tty->print_cr("[" INTPTR_FORMAT "]"
 247                   " GCTaskQueue::dequeue(%u)", p2i(this), affinity);
 248     print("before:");
 249   }
 250   assert(!is_empty(), "shouldn't dequeue from empty list");
 251   // Look down to the next barrier for a task with this affinity.
 252   GCTask* result = NULL;
 253   for (GCTask* element = remove_end();
 254        element != NULL;
 255        element = element->newer()) {
 256     if (element->is_barrier_task()) {
 257       // Don't consider barrier tasks, nor past them.
 258       result = NULL;
 259       break;
 260     }
 261     if (element->affinity() == affinity) {
 262       result = remove(element);
 263       break;
 264     }
 265   }
 266   // If we didn't find anything with affinity, just take the next task.
 267   if (result == NULL) {
 268     result = remove();
 269   }
 270   if (TraceGCTaskQueue) {
 271     tty->print_cr("    return: " INTPTR_FORMAT, p2i(result));
 272     print("after:");
 273   }
 274   return result;
 275 }
 276 
 277 GCTask* GCTaskQueue::remove() {
 278   // Dequeue from remove end.
 279   GCTask* result = remove_end();
 280   assert(result != NULL, "shouldn't have null task");
 281   assert(result->older() == NULL, "not the remove_end");
 282   set_remove_end(result->newer());
 283   if (remove_end() == NULL) {
 284     assert(insert_end() == result, "not a singleton");
 285     set_insert_end(NULL);
 286   } else {
 287     remove_end()->set_older(NULL);
 288   }
 289   result->set_newer(NULL);
 290   decrement_length();
 291   assert(result->newer() == NULL, "shouldn't be on queue");
 292   assert(result->older() == NULL, "shouldn't be on queue");
 293   verify_length();
 294   return result;
 295 }
 296 
 297 GCTask* GCTaskQueue::remove(GCTask* task) {
 298   // This is slightly more work, and has slightly fewer asserts
 299   // than removing from the remove end.
 300   assert(task != NULL, "shouldn't have null task");
 301   GCTask* result = task;
 302   if (result->newer() != NULL) {
 303     result->newer()->set_older(result->older());
 304   } else {
 305     assert(insert_end() == result, "not youngest");
 306     set_insert_end(result->older());
 307   }
 308   if (result->older() != NULL) {
 309     result->older()->set_newer(result->newer());
 310   } else {
 311     assert(remove_end() == result, "not oldest");
 312     set_remove_end(result->newer());
 313   }
 314   result->set_newer(NULL);
 315   result->set_older(NULL);
 316   decrement_length();
 317   verify_length();
 318   return result;
 319 }
 320 
 321 NOT_PRODUCT(
 322 // Count the elements in the queue and verify the length against
 323 // that count.
 324 void GCTaskQueue::verify_length() const {
 325   uint count = 0;
 326   for (GCTask* element = insert_end();
 327        element != NULL;
 328        element = element->older()) {
 329 
 330     count++;
 331   }
 332   assert(count == length(), "Length does not match queue");
 333 }
 334 
 335 void GCTaskQueue::print(const char* message) const {
 336   tty->print_cr("[" INTPTR_FORMAT "] GCTaskQueue:"
 337                 "  insert_end: " INTPTR_FORMAT
 338                 "  remove_end: " INTPTR_FORMAT
 339                 "  length:       %d"
 340                 "  %s",
 341                 p2i(this), p2i(insert_end()), p2i(remove_end()), length(), message);
 342   uint count = 0;
 343   for (GCTask* element = insert_end();
 344        element != NULL;
 345        element = element->older()) {
 346     element->print("    ");
 347     count++;
 348     tty->cr();
 349   }
 350   tty->print("Total tasks: %d", count);
 351 }
 352 )
 353 
 354 //
 355 // SynchronizedGCTaskQueue
 356 //
 357 
 358 SynchronizedGCTaskQueue::SynchronizedGCTaskQueue(GCTaskQueue* queue_arg,
 359                                                  Monitor *       lock_arg) :
 360   _unsynchronized_queue(queue_arg),
 361   _lock(lock_arg) {
 362   assert(unsynchronized_queue() != NULL, "null queue");
 363   assert(lock() != NULL, "null lock");
 364 }
 365 
 366 SynchronizedGCTaskQueue::~SynchronizedGCTaskQueue() {
 367   // Nothing to do.
 368 }
 369 
 370 //
 371 // GCTaskManager
 372 //
 373 GCTaskManager::GCTaskManager(uint workers) :
 374   _workers(workers),
 375   _active_workers(0),
 376   _idle_workers(0),
 377   _created_workers(0) {
 378   initialize();
 379 }
 380 
 381 GCTaskThread* GCTaskManager::install_worker(uint t) {
 382   GCTaskThread* new_worker = GCTaskThread::create(this, t, _processor_assignment[t]);
 383   set_thread(t, new_worker);
 384   return new_worker;
 385 }
 386 
 387 void GCTaskManager::add_workers(bool initializing) {
 388   os::ThreadType worker_type = os::pgc_thread;
 389   _created_workers = WorkerManager::add_workers(this,
 390                                                 _active_workers,
 391                                                 (uint) _workers,
 392                                                 _created_workers,
 393                                                 worker_type,
 394                                                 initializing);
 395   _active_workers = MIN2(_created_workers, _active_workers);
 396 }
 397 
 398 void GCTaskManager::initialize() {
 399   if (TraceGCTaskManager) {
 400     tty->print_cr("GCTaskManager::initialize: workers: %u", workers());
 401   }
 402   assert(workers() != 0, "no workers");
 403   _monitor = new Monitor(Mutex::barrier,                // rank
 404                          "GCTaskManager monitor",       // name
 405                          Mutex::_allow_vm_block_flag,   // allow_vm_block
 406                          Monitor::_safepoint_check_never);
 407   // The queue for the GCTaskManager must be a CHeapObj.
 408   GCTaskQueue* unsynchronized_queue = GCTaskQueue::create_on_c_heap();
 409   _queue = SynchronizedGCTaskQueue::create(unsynchronized_queue, lock());
 410   _noop_task = NoopGCTask::create_on_c_heap();
 411   _resource_flag = NEW_C_HEAP_ARRAY(bool, workers(), mtGC);
 412   {
 413     // Set up worker threads.
 414     //     Distribute the workers among the available processors,
 415     //     unless we were told not to, or if the os doesn't want to.
 416     _processor_assignment = NEW_C_HEAP_ARRAY(uint, workers(), mtGC);
 417     if (!BindGCTaskThreadsToCPUs ||
 418         !os::distribute_processes(workers(), _processor_assignment)) {
 419       for (uint a = 0; a < workers(); a += 1) {
 420         _processor_assignment[a] = sentinel_worker();
 421       }
 422     }
 423 
 424     _thread = NEW_C_HEAP_ARRAY(GCTaskThread*, workers(), mtGC);
 425     _active_workers = ParallelGCThreads;
 426     if (UseDynamicNumberOfGCThreads && !FLAG_IS_CMDLINE(ParallelGCThreads)) {
 427       _active_workers = 1U;
 428     }
 429 
 430     Log(gc, task, thread) log;
 431     if (log.is_trace()) {
 432       ResourceMark rm;
 433       outputStream* out = log.trace_stream();
 434       out->print("GCTaskManager::initialize: distribution:");
 435       for (uint t = 0; t < workers(); t += 1) {
 436         out->print("  %u", _processor_assignment[t]);
 437       }
 438       out->cr();
 439     }
 440   }
 441   reset_busy_workers();
 442   set_unblocked();
 443   for (uint w = 0; w < workers(); w += 1) {
 444     set_resource_flag(w, false);
 445   }
 446   reset_delivered_tasks();
 447   reset_completed_tasks();
 448   reset_barriers();
 449   reset_emptied_queue();
 450 
 451   add_workers(true);
 452 }
 453 
 454 GCTaskManager::~GCTaskManager() {
 455   assert(busy_workers() == 0, "still have busy workers");
 456   assert(queue()->is_empty(), "still have queued work");
 457   NoopGCTask::destroy(_noop_task);
 458   _noop_task = NULL;
 459   if (_thread != NULL) {
 460     for (uint i = 0; i < created_workers(); i += 1) {
 461       GCTaskThread::destroy(thread(i));
 462       set_thread(i, NULL);
 463     }
 464     FREE_C_HEAP_ARRAY(GCTaskThread*, _thread);
 465     _thread = NULL;
 466   }
 467   if (_processor_assignment != NULL) {
 468     FREE_C_HEAP_ARRAY(uint, _processor_assignment);
 469     _processor_assignment = NULL;
 470   }
 471   if (_resource_flag != NULL) {
 472     FREE_C_HEAP_ARRAY(bool, _resource_flag);
 473     _resource_flag = NULL;
 474   }
 475   if (queue() != NULL) {
 476     GCTaskQueue* unsynchronized_queue = queue()->unsynchronized_queue();
 477     GCTaskQueue::destroy(unsynchronized_queue);
 478     SynchronizedGCTaskQueue::destroy(queue());
 479     _queue = NULL;
 480   }
 481   if (monitor() != NULL) {
 482     delete monitor();
 483     _monitor = NULL;
 484   }
 485 }
 486 
 487 void GCTaskManager::set_active_gang() {
 488   _active_workers =
 489     AdaptiveSizePolicy::calc_active_workers(workers(),
 490                                  active_workers(),
 491                                  Threads::number_of_non_daemon_threads());
 492 
 493   assert(!all_workers_active() || active_workers() == ParallelGCThreads,
 494          "all_workers_active() is  incorrect: "
 495          "active %d  ParallelGCThreads %u", active_workers(),
 496          ParallelGCThreads);
 497   _active_workers = MIN2(_active_workers, _workers);
 498   // "add_workers" does not guarantee any additional workers
 499   add_workers(false);
 500   log_trace(gc, task)("GCTaskManager::set_active_gang(): "
 501                       "all_workers_active()  %d  workers %d  "
 502                       "active  %d  ParallelGCThreads %u",
 503                       all_workers_active(), workers(),  active_workers(),
 504                       ParallelGCThreads);
 505 }
 506 
 507 // Create IdleGCTasks for inactive workers.
 508 // Creates tasks in a ResourceArea and assumes
 509 // an appropriate ResourceMark.
 510 void GCTaskManager::task_idle_workers() {
 511   {
 512     int more_inactive_workers = 0;
 513     {
 514       // Stop any idle tasks from exiting their IdleGCTask's
 515       // and get the count for additional IdleGCTask's under
 516       // the GCTaskManager's monitor so that the "more_inactive_workers"
 517       // count is correct.
 518       MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
 519       _wait_helper.set_should_wait(true);
 520       // active_workers are a number being requested.  idle_workers
 521       // are the number currently idle.  If all the workers are being
 522       // requested to be active but some are already idle, reduce
 523       // the number of active_workers to be consistent with the
 524       // number of idle_workers.  The idle_workers are stuck in
 525       // idle tasks and will no longer be release (since a new GC
 526       // is starting).  Try later to release enough idle_workers
 527       // to allow the desired number of active_workers.
 528       more_inactive_workers =
 529         created_workers() - active_workers() - idle_workers();
 530       if (more_inactive_workers < 0) {
 531         int reduced_active_workers = active_workers() + more_inactive_workers;
 532         set_active_workers(reduced_active_workers);
 533         more_inactive_workers = 0;
 534       }
 535       log_trace(gc, task)("JT: %d  workers %d  active  %d  idle %d  more %d",
 536                           Threads::number_of_non_daemon_threads(),
 537                           created_workers(),
 538                           active_workers(),
 539                           idle_workers(),
 540                           more_inactive_workers);
 541     }
 542     GCTaskQueue* q = GCTaskQueue::create();
 543     for(uint i = 0; i < (uint) more_inactive_workers; i++) {
 544       q->enqueue(IdleGCTask::create_on_c_heap());
 545       increment_idle_workers();
 546     }
 547     assert(created_workers() == active_workers() + idle_workers(),
 548       "total workers should equal active + inactive");
 549     add_list(q);
 550     // GCTaskQueue* q was created in a ResourceArea so a
 551     // destroy() call is not needed.
 552   }
 553 }
 554 
 555 void  GCTaskManager::release_idle_workers() {
 556   {
 557     MutexLockerEx ml(monitor(),
 558       Mutex::_no_safepoint_check_flag);
 559     _wait_helper.set_should_wait(false);
 560     monitor()->notify_all();
 561   // Release monitor
 562   }
 563 }
 564 
 565 void GCTaskManager::print_task_time_stamps() {
 566   if (!log_is_enabled(Debug, gc, task, time)) {
 567     return;
 568   }
 569   uint num_thr = created_workers();
 570   for(uint i=0; i < num_thr; i++) {
 571     GCTaskThread* t = thread(i);
 572     t->print_task_time_stamps();
 573   }
 574 }
 575 
 576 void GCTaskManager::print_threads_on(outputStream* st) {
 577   uint num_thr = created_workers();
 578   for (uint i = 0; i < num_thr; i++) {
 579     thread(i)->print_on(st);
 580     st->cr();
 581   }
 582 }
 583 
 584 void GCTaskManager::threads_do(ThreadClosure* tc) {
 585   assert(tc != NULL, "Null ThreadClosure");
 586   uint num_thr = created_workers();
 587   for (uint i = 0; i < num_thr; i++) {
 588     tc->do_thread(thread(i));
 589   }
 590 }
 591 
 592 GCTaskThread* GCTaskManager::thread(uint which) {
 593   assert(which < created_workers(), "index out of bounds");
 594   assert(_thread[which] != NULL, "shouldn't have null thread");
 595   return _thread[which];
 596 }
 597 
 598 void GCTaskManager::set_thread(uint which, GCTaskThread* value) {
 599   // "_created_workers" may not have been updated yet so use workers()
 600   assert(which < workers(), "index out of bounds");
 601   assert(value != NULL, "shouldn't have null thread");
 602   _thread[which] = value;
 603 }
 604 
 605 void GCTaskManager::add_task(GCTask* task) {
 606   assert(task != NULL, "shouldn't have null task");
 607   MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
 608   if (TraceGCTaskManager) {
 609     tty->print_cr("GCTaskManager::add_task(" INTPTR_FORMAT " [%s])",
 610                   p2i(task), GCTask::Kind::to_string(task->kind()));
 611   }
 612   queue()->enqueue(task);
 613   // Notify with the lock held to avoid missed notifies.
 614   if (TraceGCTaskManager) {
 615     tty->print_cr("    GCTaskManager::add_task (%s)->notify_all",
 616                   monitor()->name());
 617   }
 618   (void) monitor()->notify_all();
 619   // Release monitor().
 620 }
 621 
 622 void GCTaskManager::add_list(GCTaskQueue* list) {
 623   assert(list != NULL, "shouldn't have null task");
 624   MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
 625   if (TraceGCTaskManager) {
 626     tty->print_cr("GCTaskManager::add_list(%u)", list->length());
 627   }
 628   queue()->enqueue(list);
 629   // Notify with the lock held to avoid missed notifies.
 630   if (TraceGCTaskManager) {
 631     tty->print_cr("    GCTaskManager::add_list (%s)->notify_all",
 632                   monitor()->name());
 633   }
 634   (void) monitor()->notify_all();
 635   // Release monitor().
 636 }
 637 
 638 // GC workers wait in get_task() for new work to be added
 639 // to the GCTaskManager's queue.  When new work is added,
 640 // a notify is sent to the waiting GC workers which then
 641 // compete to get tasks.  If a GC worker wakes up and there
 642 // is no work on the queue, it is given a noop_task to execute
 643 // and then loops to find more work.
 644 
 645 GCTask* GCTaskManager::get_task(uint which) {
 646   GCTask* result = NULL;
 647   // Grab the queue lock.
 648   MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
 649   // Wait while the queue is block or
 650   // there is nothing to do, except maybe release resources.
 651   while (is_blocked() ||
 652          (queue()->is_empty() && !should_release_resources(which))) {
 653     if (TraceGCTaskManager) {
 654       tty->print_cr("GCTaskManager::get_task(%u)"
 655                     "  blocked: %s"
 656                     "  empty: %s"
 657                     "  release: %s",
 658                     which,
 659                     is_blocked() ? "true" : "false",
 660                     queue()->is_empty() ? "true" : "false",
 661                     should_release_resources(which) ? "true" : "false");
 662       tty->print_cr("    => (%s)->wait()",
 663                     monitor()->name());
 664     }
 665     monitor()->wait(Mutex::_no_safepoint_check_flag, 0);
 666   }
 667   // We've reacquired the queue lock here.
 668   // Figure out which condition caused us to exit the loop above.
 669   if (!queue()->is_empty()) {
 670     if (UseGCTaskAffinity) {
 671       result = queue()->dequeue(which);
 672     } else {
 673       result = queue()->dequeue();
 674     }
 675     if (result->is_barrier_task()) {
 676       assert(which != sentinel_worker(),
 677              "blocker shouldn't be bogus");
 678       set_blocking_worker(which);
 679     }
 680   } else {
 681     // The queue is empty, but we were woken up.
 682     // Just hand back a Noop task,
 683     // in case someone wanted us to release resources, or whatever.
 684     result = noop_task();
 685   }
 686   assert(result != NULL, "shouldn't have null task");
 687   if (TraceGCTaskManager) {
 688     tty->print_cr("GCTaskManager::get_task(%u) => " INTPTR_FORMAT " [%s]",
 689                   which, p2i(result), GCTask::Kind::to_string(result->kind()));
 690     tty->print_cr("     %s", result->name());
 691   }
 692   if (!result->is_idle_task()) {
 693     increment_busy_workers();
 694     increment_delivered_tasks();
 695   }
 696   return result;
 697   // Release monitor().
 698 }
 699 
 700 void GCTaskManager::note_completion(uint which) {
 701   MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
 702   if (TraceGCTaskManager) {
 703     tty->print_cr("GCTaskManager::note_completion(%u)", which);
 704   }
 705   // If we are blocked, check if the completing thread is the blocker.
 706   if (blocking_worker() == which) {
 707     assert(blocking_worker() != sentinel_worker(),
 708            "blocker shouldn't be bogus");
 709     increment_barriers();
 710     set_unblocked();
 711   }
 712   increment_completed_tasks();
 713   uint active = decrement_busy_workers();
 714   if ((active == 0) && (queue()->is_empty())) {
 715     increment_emptied_queue();
 716     if (TraceGCTaskManager) {
 717       tty->print_cr("    GCTaskManager::note_completion(%u) done", which);
 718     }
 719   }
 720   if (TraceGCTaskManager) {
 721     tty->print_cr("    GCTaskManager::note_completion(%u) (%s)->notify_all",
 722                   which, monitor()->name());
 723     tty->print_cr("  "
 724                   "  blocked: %s"
 725                   "  empty: %s"
 726                   "  release: %s",
 727                   is_blocked() ? "true" : "false",
 728                   queue()->is_empty() ? "true" : "false",
 729                   should_release_resources(which) ? "true" : "false");
 730     tty->print_cr("  "
 731                   "  delivered: %u"
 732                   "  completed: %u"
 733                   "  barriers: %u"
 734                   "  emptied: %u",
 735                   delivered_tasks(),
 736                   completed_tasks(),
 737                   barriers(),
 738                   emptied_queue());
 739   }
 740   // Tell everyone that a task has completed.
 741   (void) monitor()->notify_all();
 742   // Release monitor().
 743 }
 744 
 745 uint GCTaskManager::increment_busy_workers() {
 746   assert(queue()->own_lock(), "don't own the lock");
 747   _busy_workers += 1;
 748   return _busy_workers;
 749 }
 750 
 751 uint GCTaskManager::decrement_busy_workers() {
 752   assert(queue()->own_lock(), "don't own the lock");
 753   assert(_busy_workers > 0, "About to make a mistake");
 754   _busy_workers -= 1;
 755   return _busy_workers;
 756 }
 757 
 758 void GCTaskManager::release_all_resources() {
 759   // If you want this to be done atomically, do it in a WaitForBarrierGCTask.
 760   for (uint i = 0; i < created_workers(); i += 1) {
 761     set_resource_flag(i, true);
 762   }
 763 }
 764 
 765 bool GCTaskManager::should_release_resources(uint which) {
 766   // This can be done without a lock because each thread reads one element.
 767   return resource_flag(which);
 768 }
 769 
 770 void GCTaskManager::note_release(uint which) {
 771   // This can be done without a lock because each thread writes one element.
 772   set_resource_flag(which, false);
 773 }
 774 
 775 // "list" contains tasks that are ready to execute.  Those
 776 // tasks are added to the GCTaskManager's queue of tasks and
 777 // then the GC workers are notified that there is new work to
 778 // do.
 779 //
 780 // Typically different types of tasks can be added to the "list".
 781 // For example in PSScavenge OldToYoungRootsTask, SerialOldToYoungRootsTask,
 782 // ScavengeRootsTask, and StealTask tasks are all added to the list
 783 // and then the GC workers are notified of new work.  The tasks are
 784 // handed out in the order in which they are added to the list
 785 // (although execution is not necessarily in that order).  As long
 786 // as any tasks are running the GCTaskManager will wait for execution
 787 // to complete.  GC workers that execute a stealing task remain in
 788 // the stealing task until all stealing tasks have completed.  The load
 789 // balancing afforded by the stealing tasks work best if the stealing
 790 // tasks are added last to the list.
 791 
 792 void GCTaskManager::execute_and_wait(GCTaskQueue* list) {
 793   WaitForBarrierGCTask* fin = WaitForBarrierGCTask::create();
 794   list->enqueue(fin);
 795   // The barrier task will be read by one of the GC
 796   // workers once it is added to the list of tasks.
 797   // Be sure that is globally visible before the
 798   // GC worker reads it (which is after the task is added
 799   // to the list of tasks below).
 800   OrderAccess::storestore();
 801   add_list(list);
 802   fin->wait_for(true /* reset */);
 803   // We have to release the barrier tasks!
 804   WaitForBarrierGCTask::destroy(fin);
 805 }
 806 
 807 bool GCTaskManager::resource_flag(uint which) {
 808   assert(which < workers(), "index out of bounds");
 809   return _resource_flag[which];
 810 }
 811 
 812 void GCTaskManager::set_resource_flag(uint which, bool value) {
 813   assert(which < workers(), "index out of bounds");
 814   _resource_flag[which] = value;
 815 }
 816 
 817 //
 818 // NoopGCTask
 819 //
 820 
 821 NoopGCTask* NoopGCTask::create_on_c_heap() {
 822   NoopGCTask* result = new(ResourceObj::C_HEAP, mtGC) NoopGCTask();
 823   return result;
 824 }
 825 
 826 void NoopGCTask::destroy(NoopGCTask* that) {
 827   if (that != NULL) {
 828     that->destruct();
 829     FreeHeap(that);
 830   }
 831 }
 832 
 833 // This task should never be performing GC work that require
 834 // a valid GC id.
 835 NoopGCTask::NoopGCTask() : GCTask(GCTask::Kind::noop_task, GCId::undefined()) { }
 836 
 837 void NoopGCTask::destruct() {
 838   // This has to know it's superclass structure, just like the constructor.
 839   this->GCTask::destruct();
 840   // Nothing else to do.
 841 }
 842 
 843 //
 844 // IdleGCTask
 845 //
 846 
 847 IdleGCTask* IdleGCTask::create() {
 848   IdleGCTask* result = new IdleGCTask(false);
 849   assert(UseDynamicNumberOfGCThreads,
 850     "Should only be used with dynamic GC thread");
 851   return result;
 852 }
 853 
 854 IdleGCTask* IdleGCTask::create_on_c_heap() {
 855   IdleGCTask* result = new(ResourceObj::C_HEAP, mtGC) IdleGCTask(true);
 856   assert(UseDynamicNumberOfGCThreads,
 857     "Should only be used with dynamic GC thread");
 858   return result;
 859 }
 860 
 861 void IdleGCTask::do_it(GCTaskManager* manager, uint which) {
 862   WaitHelper* wait_helper = manager->wait_helper();
 863   log_trace(gc, task)("[" INTPTR_FORMAT "] IdleGCTask:::do_it() should_wait: %s",
 864       p2i(this), wait_helper->should_wait() ? "true" : "false");
 865 
 866   MutexLockerEx ml(manager->monitor(), Mutex::_no_safepoint_check_flag);
 867   log_trace(gc, task)("--- idle %d", which);
 868   // Increment has to be done when the idle tasks are created.
 869   // manager->increment_idle_workers();
 870   manager->monitor()->notify_all();
 871   while (wait_helper->should_wait()) {
 872     log_trace(gc, task)("[" INTPTR_FORMAT "] IdleGCTask::do_it()  [" INTPTR_FORMAT "] (%s)->wait()",
 873       p2i(this), p2i(manager->monitor()), manager->monitor()->name());
 874     manager->monitor()->wait(Mutex::_no_safepoint_check_flag, 0);
 875   }
 876   manager->decrement_idle_workers();
 877 
 878   log_trace(gc, task)("--- release %d", which);
 879   log_trace(gc, task)("[" INTPTR_FORMAT "] IdleGCTask::do_it() returns should_wait: %s",
 880     p2i(this), wait_helper->should_wait() ? "true" : "false");
 881   // Release monitor().
 882 }
 883 
 884 void IdleGCTask::destroy(IdleGCTask* that) {
 885   if (that != NULL) {
 886     that->destruct();
 887     if (that->is_c_heap_obj()) {
 888       FreeHeap(that);
 889     }
 890   }
 891 }
 892 
 893 void IdleGCTask::destruct() {
 894   // This has to know it's superclass structure, just like the constructor.
 895   this->GCTask::destruct();
 896   // Nothing else to do.
 897 }
 898 
 899 //
 900 // WaitForBarrierGCTask
 901 //
 902 WaitForBarrierGCTask* WaitForBarrierGCTask::create() {
 903   WaitForBarrierGCTask* result = new WaitForBarrierGCTask();
 904   return result;
 905 }
 906 
 907 WaitForBarrierGCTask::WaitForBarrierGCTask() : GCTask(GCTask::Kind::wait_for_barrier_task) { }
 908 
 909 void WaitForBarrierGCTask::destroy(WaitForBarrierGCTask* that) {
 910   if (that != NULL) {
 911     if (TraceGCTaskManager) {
 912       tty->print_cr("[" INTPTR_FORMAT "] WaitForBarrierGCTask::destroy()", p2i(that));
 913     }
 914     that->destruct();
 915   }
 916 }
 917 
 918 void WaitForBarrierGCTask::destruct() {
 919   if (TraceGCTaskManager) {
 920     tty->print_cr("[" INTPTR_FORMAT "] WaitForBarrierGCTask::destruct()", p2i(this));
 921   }
 922   this->GCTask::destruct();
 923   // Clean up that should be in the destructor,
 924   // except that ResourceMarks don't call destructors.
 925   _wait_helper.release_monitor();
 926 }
 927 
 928 void WaitForBarrierGCTask::do_it_internal(GCTaskManager* manager, uint which) {
 929   // Wait for this to be the only busy worker.
 930   assert(manager->monitor()->owned_by_self(), "don't own the lock");
 931   assert(manager->is_blocked(), "manager isn't blocked");
 932   while (manager->busy_workers() > 1) {
 933     if (TraceGCTaskManager) {
 934       tty->print_cr("WaitForBarrierGCTask::do_it(%u) waiting on %u workers",
 935                     which, manager->busy_workers());
 936     }
 937     manager->monitor()->wait(Mutex::_no_safepoint_check_flag, 0);
 938   }
 939 }
 940 
 941 void WaitForBarrierGCTask::do_it(GCTaskManager* manager, uint which) {
 942   if (TraceGCTaskManager) {
 943     tty->print_cr("[" INTPTR_FORMAT "]"
 944                   " WaitForBarrierGCTask::do_it() waiting for idle",
 945                   p2i(this));
 946   }
 947   {
 948     // First, wait for the barrier to arrive.
 949     MutexLockerEx ml(manager->lock(), Mutex::_no_safepoint_check_flag);
 950     do_it_internal(manager, which);
 951     // Release manager->lock().
 952   }
 953   // Then notify the waiter.
 954   _wait_helper.notify();
 955 }
 956 
 957 WaitHelper::WaitHelper() : _should_wait(true), _monitor(MonitorSupply::reserve()) {
 958   if (TraceGCTaskManager) {
 959     tty->print_cr("[" INTPTR_FORMAT "]"
 960                   " WaitHelper::WaitHelper()"
 961                   "  monitor: " INTPTR_FORMAT,
 962                   p2i(this), p2i(monitor()));
 963   }
 964 }
 965 
 966 void WaitHelper::release_monitor() {
 967   assert(_monitor != NULL, "");
 968   MonitorSupply::release(_monitor);
 969   _monitor = NULL;
 970 }
 971 
 972 WaitHelper::~WaitHelper() {
 973   release_monitor();
 974 }
 975 
 976 void WaitHelper::wait_for(bool reset) {
 977   if (TraceGCTaskManager) {
 978     tty->print_cr("[" INTPTR_FORMAT "]"
 979                   " WaitForBarrierGCTask::wait_for()"
 980       "  should_wait: %s",
 981       p2i(this), should_wait() ? "true" : "false");
 982   }
 983   {
 984     // Grab the lock and check again.
 985     MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
 986     while (should_wait()) {
 987       if (TraceGCTaskManager) {
 988         tty->print_cr("[" INTPTR_FORMAT "]"
 989                       " WaitForBarrierGCTask::wait_for()"
 990           "  [" INTPTR_FORMAT "] (%s)->wait()",
 991           p2i(this), p2i(monitor()), monitor()->name());
 992       }
 993       monitor()->wait(Mutex::_no_safepoint_check_flag, 0);
 994     }
 995     // Reset the flag in case someone reuses this task.
 996     if (reset) {
 997       set_should_wait(true);
 998     }
 999     if (TraceGCTaskManager) {
1000       tty->print_cr("[" INTPTR_FORMAT "]"
1001                     " WaitForBarrierGCTask::wait_for() returns"
1002         "  should_wait: %s",
1003         p2i(this), should_wait() ? "true" : "false");
1004     }
1005     // Release monitor().
1006   }
1007 }
1008 
1009 void WaitHelper::notify() {
1010   MutexLockerEx ml(monitor(), Mutex::_no_safepoint_check_flag);
1011   set_should_wait(false);
1012   // Waiter doesn't miss the notify in the wait_for method
1013   // since it checks the flag after grabbing the monitor.
1014   if (TraceGCTaskManager) {
1015     tty->print_cr("[" INTPTR_FORMAT "]"
1016                   " WaitForBarrierGCTask::do_it()"
1017                   "  [" INTPTR_FORMAT "] (%s)->notify_all()",
1018                   p2i(this), p2i(monitor()), monitor()->name());
1019   }
1020   monitor()->notify_all();
1021 }
1022 
1023 Mutex*                   MonitorSupply::_lock     = NULL;
1024 GrowableArray<Monitor*>* MonitorSupply::_freelist = NULL;
1025 
1026 Monitor* MonitorSupply::reserve() {
1027   Monitor* result = NULL;
1028   // Lazy initialization: possible race.
1029   if (lock() == NULL) {
1030     _lock = new Mutex(Mutex::barrier,                  // rank
1031                       "MonitorSupply mutex",           // name
1032                       Mutex::_allow_vm_block_flag);    // allow_vm_block
1033   }
1034   {
1035     MutexLockerEx ml(lock());
1036     // Lazy initialization.
1037     if (freelist() == NULL) {
1038       _freelist =
1039         new(ResourceObj::C_HEAP, mtGC) GrowableArray<Monitor*>(ParallelGCThreads,
1040                                                          true);
1041     }
1042     if (! freelist()->is_empty()) {
1043       result = freelist()->pop();
1044     } else {
1045       result = new Monitor(Mutex::barrier,                  // rank
1046                            "MonitorSupply monitor",         // name
1047                            Mutex::_allow_vm_block_flag,     // allow_vm_block
1048                            Monitor::_safepoint_check_never);
1049     }
1050     guarantee(result != NULL, "shouldn't return NULL");
1051     assert(!result->is_locked(), "shouldn't be locked");
1052     // release lock().
1053   }
1054   return result;
1055 }
1056 
1057 void MonitorSupply::release(Monitor* instance) {
1058   assert(instance != NULL, "shouldn't release NULL");
1059   assert(!instance->is_locked(), "shouldn't be locked");
1060   {
1061     MutexLockerEx ml(lock());
1062     freelist()->push(instance);
1063     // release lock().
1064   }
1065 }