1 /*
   2  * Copyright (c) 2002, 2012, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc_implementation/parallelScavenge/parallelScavengeHeap.hpp"
  27 #include "gc_implementation/parallelScavenge/psOldGen.hpp"
  28 #include "gc_implementation/parallelScavenge/psPromotionManager.inline.hpp"
  29 #include "gc_implementation/parallelScavenge/psScavenge.inline.hpp"
  30 #include "gc_implementation/shared/mutableSpace.hpp"
  31 #include "memory/memRegion.hpp"
  32 #include "oops/oop.inline.hpp"
  33 #include "oops/oop.psgc.inline.hpp"
  34 
  35 PSPromotionManager**         PSPromotionManager::_manager_array = NULL;
  36 OopStarTaskQueueSet*         PSPromotionManager::_stack_array_depth = NULL;
  37 PSOldGen*                    PSPromotionManager::_old_gen = NULL;
  38 MutableSpace*                PSPromotionManager::_young_space = NULL;
  39 
  40 void PSPromotionManager::initialize() {
  41   ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
  42   assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
  43 
  44   _old_gen = heap->old_gen();
  45   _young_space = heap->young_gen()->to_space();
  46 
  47   assert(_manager_array == NULL, "Attempt to initialize twice");
  48   _manager_array = NEW_C_HEAP_ARRAY(PSPromotionManager*, ParallelGCThreads+1, mtGC);
  49   guarantee(_manager_array != NULL, "Could not initialize promotion manager");
  50 
  51   _stack_array_depth = new OopStarTaskQueueSet(ParallelGCThreads);
  52   guarantee(_stack_array_depth != NULL, "Cound not initialize promotion manager");
  53 
  54   // Create and register the PSPromotionManager(s) for the worker threads.
  55   for(uint i=0; i<ParallelGCThreads; i++) {
  56     _manager_array[i] = new PSPromotionManager();
  57     guarantee(_manager_array[i] != NULL, "Could not create PSPromotionManager");
  58     stack_array_depth()->register_queue(i, _manager_array[i]->claimed_stack_depth());
  59   }
  60 
  61   // The VMThread gets its own PSPromotionManager, which is not available
  62   // for work stealing.
  63   _manager_array[ParallelGCThreads] = new PSPromotionManager();
  64   guarantee(_manager_array[ParallelGCThreads] != NULL, "Could not create PSPromotionManager");
  65 }
  66 
  67 PSPromotionManager* PSPromotionManager::gc_thread_promotion_manager(int index) {
  68   assert(index >= 0 && index < (int)ParallelGCThreads, "index out of range");
  69   assert(_manager_array != NULL, "Sanity");
  70   return _manager_array[index];
  71 }
  72 
  73 PSPromotionManager* PSPromotionManager::vm_thread_promotion_manager() {
  74   assert(_manager_array != NULL, "Sanity");
  75   return _manager_array[ParallelGCThreads];
  76 }
  77 
  78 void PSPromotionManager::pre_scavenge() {
  79   ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
  80   assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
  81 
  82   _young_space = heap->young_gen()->to_space();
  83 
  84   for(uint i=0; i<ParallelGCThreads+1; i++) {
  85     manager_array(i)->reset();
  86   }
  87 }
  88 
  89 void PSPromotionManager::post_scavenge() {
  90   TASKQUEUE_STATS_ONLY(if (PrintGCDetails && ParallelGCVerbose) print_stats());
  91   for (uint i = 0; i < ParallelGCThreads + 1; i++) {
  92     PSPromotionManager* manager = manager_array(i);
  93     assert(manager->claimed_stack_depth()->is_empty(), "should be empty");
  94     manager->flush_labs();
  95   }
  96 }
  97 
  98 #if TASKQUEUE_STATS
  99 void
 100 PSPromotionManager::print_taskqueue_stats(uint i) const {
 101   tty->print("%3u ", i);
 102   _claimed_stack_depth.stats.print();
 103   tty->cr();
 104 }
 105 
 106 void
 107 PSPromotionManager::print_local_stats(uint i) const {
 108   #define FMT " " SIZE_FORMAT_W(10)
 109   tty->print_cr("%3u" FMT FMT FMT FMT, i, _masked_pushes, _masked_steals,
 110                 _arrays_chunked, _array_chunks_processed);
 111   #undef FMT
 112 }
 113 
 114 static const char* const pm_stats_hdr[] = {
 115   "    --------masked-------     arrays      array",
 116   "thr       push      steal    chunked     chunks",
 117   "--- ---------- ---------- ---------- ----------"
 118 };
 119 
 120 void
 121 PSPromotionManager::print_stats() {
 122   tty->print_cr("== GC Tasks Stats, GC %3d",
 123                 Universe::heap()->total_collections());
 124 
 125   tty->print("thr "); TaskQueueStats::print_header(1); tty->cr();
 126   tty->print("--- "); TaskQueueStats::print_header(2); tty->cr();
 127   for (uint i = 0; i < ParallelGCThreads + 1; ++i) {
 128     manager_array(i)->print_taskqueue_stats(i);
 129   }
 130 
 131   const uint hlines = sizeof(pm_stats_hdr) / sizeof(pm_stats_hdr[0]);
 132   for (uint i = 0; i < hlines; ++i) tty->print_cr(pm_stats_hdr[i]);
 133   for (uint i = 0; i < ParallelGCThreads + 1; ++i) {
 134     manager_array(i)->print_local_stats(i);
 135   }
 136 }
 137 
 138 void
 139 PSPromotionManager::reset_stats() {
 140   claimed_stack_depth()->stats.reset();
 141   _masked_pushes = _masked_steals = 0;
 142   _arrays_chunked = _array_chunks_processed = 0;
 143 }
 144 #endif // TASKQUEUE_STATS
 145 
 146 PSPromotionManager::PSPromotionManager() {
 147   ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
 148   assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
 149 
 150   // We set the old lab's start array.
 151   _old_lab.set_start_array(old_gen()->start_array());
 152 
 153   uint queue_size;
 154   claimed_stack_depth()->initialize();
 155   queue_size = claimed_stack_depth()->max_elems();
 156 
 157   _totally_drain = (ParallelGCThreads == 1) || (GCDrainStackTargetSize == 0);
 158   if (_totally_drain) {
 159     _target_stack_size = 0;
 160   } else {
 161     // don't let the target stack size to be more than 1/4 of the entries
 162     _target_stack_size = (uint) MIN2((uint) GCDrainStackTargetSize,
 163                                      (uint) (queue_size / 4));
 164   }
 165 
 166   _array_chunk_size = ParGCArrayScanChunk;
 167   // let's choose 1.5x the chunk size
 168   _min_array_size_for_chunking = 3 * _array_chunk_size / 2;
 169 
 170   reset();
 171 }
 172 
 173 void PSPromotionManager::reset() {
 174   assert(stacks_empty(), "reset of non-empty stack");
 175 
 176   // We need to get an assert in here to make sure the labs are always flushed.
 177 
 178   ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
 179   assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
 180 
 181   // Do not prefill the LAB's, save heap wastage!
 182   HeapWord* lab_base = young_space()->top();
 183   _young_lab.initialize(MemRegion(lab_base, (size_t)0));
 184   _young_gen_is_full = false;
 185 
 186   lab_base = old_gen()->object_space()->top();
 187   _old_lab.initialize(MemRegion(lab_base, (size_t)0));
 188   _old_gen_is_full = false;
 189 
 190   TASKQUEUE_STATS_ONLY(reset_stats());
 191 }
 192 
 193 
 194 void PSPromotionManager::drain_stacks_depth(bool totally_drain) {
 195   totally_drain = totally_drain || _totally_drain;
 196 
 197 #ifdef ASSERT
 198   ParallelScavengeHeap* heap = (ParallelScavengeHeap*)Universe::heap();
 199   assert(heap->kind() == CollectedHeap::ParallelScavengeHeap, "Sanity");
 200   MutableSpace* to_space = heap->young_gen()->to_space();
 201   MutableSpace* old_space = heap->old_gen()->object_space();
 202 #endif /* ASSERT */
 203 
 204   OopStarTaskQueue* const tq = claimed_stack_depth();
 205   do {
 206     StarTask p;
 207 
 208     // Drain overflow stack first, so other threads can steal from
 209     // claimed stack while we work.
 210     while (tq->pop_overflow(p)) {
 211       process_popped_location_depth(p);
 212     }
 213 
 214     if (totally_drain) {
 215       while (tq->pop_local(p)) {
 216         process_popped_location_depth(p);
 217       }
 218     } else {
 219       while (tq->size() > _target_stack_size && tq->pop_local(p)) {
 220         process_popped_location_depth(p);
 221       }
 222     }
 223   } while (totally_drain && !tq->taskqueue_empty() || !tq->overflow_empty());
 224 
 225   assert(!totally_drain || tq->taskqueue_empty(), "Sanity");
 226   assert(totally_drain || tq->size() <= _target_stack_size, "Sanity");
 227   assert(tq->overflow_empty(), "Sanity");
 228 }
 229 
 230 void PSPromotionManager::flush_labs() {
 231   assert(stacks_empty(), "Attempt to flush lab with live stack");
 232 
 233   // If either promotion lab fills up, we can flush the
 234   // lab but not refill it, so check first.
 235   assert(!_young_lab.is_flushed() || _young_gen_is_full, "Sanity");
 236   if (!_young_lab.is_flushed())
 237     _young_lab.flush();
 238 
 239   assert(!_old_lab.is_flushed() || _old_gen_is_full, "Sanity");
 240   if (!_old_lab.is_flushed())
 241     _old_lab.flush();
 242 
 243   // Let PSScavenge know if we overflowed
 244   if (_young_gen_is_full) {
 245     PSScavenge::set_survivor_overflow(true);
 246   }
 247 }
 248 
 249 template <class T> void PSPromotionManager::process_array_chunk_work(
 250                                                  oop obj,
 251                                                  int start, int end) {
 252   assert(start <= end, "invariant");
 253   T* const base      = (T*)objArrayOop(obj)->base();
 254   T* p               = base + start;
 255   T* const chunk_end = base + end;
 256   while (p < chunk_end) {
 257     if (PSScavenge::should_scavenge(p)) {
 258       claim_or_forward_depth(p);
 259     }
 260     ++p;
 261   }
 262 }
 263 
 264 void PSPromotionManager::process_array_chunk(oop old) {
 265   assert(PSChunkLargeArrays, "invariant");
 266   assert(old->is_objArray(), "invariant");
 267   assert(old->is_forwarded(), "invariant");
 268 
 269   TASKQUEUE_STATS_ONLY(++_array_chunks_processed);
 270 
 271   oop const obj = old->forwardee();
 272 
 273   int start;
 274   int const end = arrayOop(old)->length();
 275   if (end > (int) _min_array_size_for_chunking) {
 276     // we'll chunk more
 277     start = end - _array_chunk_size;
 278     assert(start > 0, "invariant");
 279     arrayOop(old)->set_length(start);
 280     push_depth(mask_chunked_array_oop(old));
 281     TASKQUEUE_STATS_ONLY(++_masked_pushes);
 282   } else {
 283     // this is the final chunk for this array
 284     start = 0;
 285     int const actual_length = arrayOop(obj)->length();
 286     arrayOop(old)->set_length(actual_length);
 287   }
 288 
 289   if (UseCompressedOops) {
 290     process_array_chunk_work<narrowOop>(obj, start, end);
 291   } else {
 292     process_array_chunk_work<oop>(obj, start, end);
 293   }
 294 }
 295 
 296 oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) {
 297   assert(_old_gen_is_full || PromotionFailureALot, "Sanity");
 298 
 299   // Attempt to CAS in the header.
 300   // This tests if the header is still the same as when
 301   // this started.  If it is the same (i.e., no forwarding
 302   // pointer has been installed), then this thread owns
 303   // it.
 304   if (obj->cas_forward_to(obj, obj_mark)) {
 305     // We won any races, we "own" this object.
 306     assert(obj == obj->forwardee(), "Sanity");
 307 
 308     obj->push_contents(this);
 309 
 310     // Save the mark if needed
 311     PSScavenge::oop_promotion_failed(obj, obj_mark);
 312   }  else {
 313     // We lost, someone else "owns" this object
 314     guarantee(obj->is_forwarded(), "Object must be forwarded if the cas failed.");
 315 
 316     // No unallocation to worry about.
 317     obj = obj->forwardee();
 318   }
 319 
 320 #ifndef PRODUCT
 321   if (TraceScavenge) {
 322     gclog_or_tty->print_cr("{%s %s 0x%x (%d)}",
 323                            "promotion-failure",
 324                            obj->klass()->internal_name(),
 325                            obj, obj->size());
 326 
 327   }
 328 #endif
 329 
 330   return obj;
 331 }