src/share/vm/gc_implementation/parNew/parNewGeneration.cpp

Print this page




  52 #include "runtime/handles.inline.hpp"
  53 #include "runtime/java.hpp"
  54 #include "runtime/thread.inline.hpp"
  55 #include "utilities/copy.hpp"
  56 #include "utilities/globalDefinitions.hpp"
  57 #include "utilities/workgroup.hpp"
  58 
  59 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
  60 
  61 #ifdef _MSC_VER
  62 #pragma warning( push )
  63 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
  64 #endif
  65 ParScanThreadState::ParScanThreadState(Space* to_space_,
  66                                        ParNewGeneration* gen_,
  67                                        Generation* old_gen_,
  68                                        int thread_num_,
  69                                        ObjToScanQueueSet* work_queue_set_,
  70                                        Stack<oop, mtGC>* overflow_stacks_,
  71                                        size_t desired_plab_sz_,

  72                                        ParallelTaskTerminator& term_) :
  73   _to_space(to_space_), _old_gen(old_gen_), _young_gen(gen_), _thread_num(thread_num_),
  74   _work_queue(work_queue_set_->queue(thread_num_)), _to_space_full(false),
  75   _overflow_stack(overflow_stacks_ ? overflow_stacks_ + thread_num_ : NULL),
  76   _ageTable(false), // false ==> not the global age table, no perf data.
  77   _to_space_alloc_buffer(desired_plab_sz_),
  78   _to_space_closure(gen_, this), _old_gen_closure(gen_, this),
  79   _to_space_root_closure(gen_, this), _old_gen_root_closure(gen_, this),
  80   _older_gen_closure(gen_, this),
  81   _evacuate_followers(this, &_to_space_closure, &_old_gen_closure,
  82                       &_to_space_root_closure, gen_, &_old_gen_root_closure,
  83                       work_queue_set_, &term_),
  84   _is_alive_closure(gen_), _scan_weak_ref_closure(gen_, this),
  85   _keep_alive_closure(&_scan_weak_ref_closure),

  86   _strong_roots_time(0.0), _term_time(0.0)
  87 {
  88   #if TASKQUEUE_STATS
  89   _term_attempts = 0;
  90   _overflow_refills = 0;
  91   _overflow_refill_objs = 0;
  92   #endif // TASKQUEUE_STATS
  93 
  94   _survivor_chunk_array =
  95     (ChunkArray*) old_gen()->get_data_recorder(thread_num());
  96   _hash_seed = 17;  // Might want to take time-based random value.
  97   _start = os::elapsedTime();
  98   _old_gen_closure.set_generation(old_gen_);
  99   _old_gen_root_closure.set_generation(old_gen_);
 100 }
 101 #ifdef _MSC_VER
 102 #pragma warning( pop )
 103 #endif
 104 
 105 void ParScanThreadState::record_survivor_plab(HeapWord* plab_start,


 205     assert(Universe::heap()->is_in_reserved(cur), "Should be in heap");
 206     assert(!old_gen()->is_in_reserved(cur), "Should be in young gen");
 207     assert(Universe::heap()->is_in_reserved(obj_to_push), "Should be in heap");
 208     if (should_be_partially_scanned(obj_to_push, cur)) {
 209       assert(arrayOop(cur)->length() == 0, "entire array remaining to be scanned");
 210       obj_to_push = cur;
 211     }
 212     bool ok = queue->push(obj_to_push);
 213     assert(ok, "Should have succeeded");
 214   }
 215   assert(young_gen()->overflow_list() == NULL, "Error");
 216   return num_take_elems > 0;  // was something transferred?
 217 }
 218 
 219 void ParScanThreadState::push_on_overflow_stack(oop p) {
 220   assert(ParGCUseLocalOverflow, "Else should not call");
 221   overflow_stack()->push(p);
 222   assert(young_gen()->overflow_list() == NULL, "Error");
 223 }
 224 
 225 HeapWord* ParScanThreadState::alloc_in_to_space_slow(size_t word_sz) {


 226 
 227   // Otherwise, if the object is small enough, try to reallocate the
 228   // buffer.
 229   HeapWord* obj = NULL;
 230   if (!_to_space_full) {
 231     ParGCAllocBuffer* const plab = to_space_alloc_buffer();
 232     Space*            const sp   = to_space();
 233     if (word_sz * 100 <
 234         ParallelGCBufferWastePct * plab->word_sz()) {
 235       // Is small enough; abandon this buffer and start a new one.
 236       plab->retire(false, false);
 237       size_t buf_size = plab->word_sz();
 238       HeapWord* buf_space = sp->par_allocate(buf_size);
 239       if (buf_space == NULL) {
 240         const size_t min_bytes =
 241           ParGCAllocBuffer::min_size() << LogHeapWordSize;
 242         size_t free_bytes = sp->free();
 243         while(buf_space == NULL && free_bytes >= min_bytes) {
 244           buf_size = free_bytes >> LogHeapWordSize;
 245           assert(buf_size == (size_t)align_object_size(buf_size),
 246                  "Invariant");
 247           buf_space  = sp->par_allocate(buf_size);
 248           free_bytes = sp->free();
 249         }
 250       }
 251       if (buf_space != NULL) {
 252         plab->set_word_size(buf_size);
 253         plab->set_buf(buf_space);
 254         record_survivor_plab(buf_space, buf_size);


 255         obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes);
 256         // Note that we cannot compare buf_size < word_sz below
 257         // because of AlignmentReserve (see ParGCAllocBuffer::allocate()).
 258         assert(obj != NULL || plab->words_remaining() < word_sz,
 259                "Else should have been able to allocate");
 260         // It's conceivable that we may be able to use the
 261         // buffer we just grabbed for subsequent small requests
 262         // even if not for this one.
 263       } else {
 264         // We're used up.
 265         _to_space_full = true;
 266       }
 267 
 268     } else {
 269       // Too large; allocate the object individually.

 270       obj = sp->par_allocate(word_sz);
 271     }
 272   }
 273   return obj;
 274 }
 275 
 276 
 277 void ParScanThreadState::undo_alloc_in_to_space(HeapWord* obj,
 278                                                 size_t word_sz) {
 279   // Is the alloc in the current alloc buffer?
 280   if (to_space_alloc_buffer()->contains(obj)) {
 281     assert(to_space_alloc_buffer()->contains(obj + word_sz - 1),
 282            "Should contain whole object.");
 283     to_space_alloc_buffer()->undo_allocation(obj, word_sz);
 284   } else {
 285     CollectedHeap::fill_with_object(obj, word_sz);
 286   }
 287 }
 288 
 289 void ParScanThreadState::print_promotion_failure_size() {
 290   if (_promotion_failed_info.has_failed() && PrintPromotionFailure) {
 291     gclog_or_tty->print(" (%d: promotion failure size = " SIZE_FORMAT ") ",
 292                         _thread_num, _promotion_failed_info.first_size());
 293   }
 294 }
 295 
 296 class ParScanThreadStateSet: private ResourceArray {
 297 public:
 298   // Initializes states for the specified number of threads;
 299   ParScanThreadStateSet(int                     num_threads,
 300                         Space&                  to_space,
 301                         ParNewGeneration&       gen,
 302                         Generation&             old_gen,
 303                         ObjToScanQueueSet&      queue_set,
 304                         Stack<oop, mtGC>*       overflow_stacks_,
 305                         size_t                  desired_plab_sz,

 306                         ParallelTaskTerminator& term);
 307 
 308   ~ParScanThreadStateSet() { TASKQUEUE_STATS_ONLY(reset_stats()); }
 309 
 310   inline ParScanThreadState& thread_state(int i);
 311 
 312   void trace_promotion_failed(YoungGCTracer& gc_tracer);
 313   void reset(int active_workers, bool promotion_failed);
 314   void flush();
 315 
 316   #if TASKQUEUE_STATS
 317   static void
 318     print_termination_stats_hdr(outputStream* const st = gclog_or_tty);
 319   void print_termination_stats(outputStream* const st = gclog_or_tty);
 320   static void
 321     print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty);
 322   void print_taskqueue_stats(outputStream* const st = gclog_or_tty);
 323   void reset_stats();
 324   #endif // TASKQUEUE_STATS
 325 
 326 private:
 327   ParallelTaskTerminator& _term;
 328   ParNewGeneration&       _gen;
 329   Generation&             _next_gen;
 330  public:
 331   bool is_valid(int id) const { return id < length(); }
 332   ParallelTaskTerminator* terminator() { return &_term; }
 333 };
 334 
 335 
 336 ParScanThreadStateSet::ParScanThreadStateSet(
 337   int num_threads, Space& to_space, ParNewGeneration& gen,
 338   Generation& old_gen, ObjToScanQueueSet& queue_set,
 339   Stack<oop, mtGC>* overflow_stacks,
 340   size_t desired_plab_sz, ParallelTaskTerminator& term)

 341   : ResourceArray(sizeof(ParScanThreadState), num_threads),
 342     _gen(gen), _next_gen(old_gen), _term(term)
 343 {
 344   assert(num_threads > 0, "sanity check!");
 345   assert(ParGCUseLocalOverflow == (overflow_stacks != NULL),
 346          "overflow_stack allocation mismatch");
 347   // Initialize states.
 348   for (int i = 0; i < num_threads; ++i) {
 349     new ((ParScanThreadState*)_data + i)
 350         ParScanThreadState(&to_space, &gen, &old_gen, i, &queue_set,
 351                            overflow_stacks, desired_plab_sz, term);
 352   }
 353 }
 354 
 355 inline ParScanThreadState& ParScanThreadStateSet::thread_state(int i)
 356 {
 357   assert(i >= 0 && i < length(), "sanity check!");
 358   return ((ParScanThreadState*)_data)[i];
 359 }
 360 
 361 void ParScanThreadStateSet::trace_promotion_failed(YoungGCTracer& gc_tracer) {
 362   for (int i = 0; i < length(); ++i) {
 363     if (thread_state(i).promotion_failed()) {
 364       gc_tracer.report_promotion_failed(thread_state(i).promotion_failed_info());
 365       thread_state(i).promotion_failed_info().reset();
 366     }
 367   }
 368 }
 369 
 370 void ParScanThreadStateSet::reset(int active_threads, bool promotion_failed)
 371 {


 963   // Capture heap used before collection (for printing).
 964   size_t gch_prev_used = gch->used();
 965 
 966   SpecializationStats::clear();
 967 
 968   age_table()->clear();
 969   to()->clear(SpaceDecorator::Mangle);
 970 
 971   gch->save_marks();
 972   assert(workers != NULL, "Need parallel worker threads.");
 973   int n_workers = active_workers;
 974 
 975   // Set the correct parallelism (number of queues) in the reference processor
 976   ref_processor()->set_active_mt_degree(n_workers);
 977 
 978   // Always set the terminator for the active number of workers
 979   // because only those workers go through the termination protocol.
 980   ParallelTaskTerminator _term(n_workers, task_queues());
 981   ParScanThreadStateSet thread_state_set(workers->active_workers(),
 982                                          *to(), *this, *_next_gen, *task_queues(),
 983                                          _overflow_stacks, desired_plab_sz(), _term);

 984 
 985   ParNewGenTask tsk(this, _next_gen, reserved().end(), &thread_state_set);
 986   gch->set_par_threads(n_workers);
 987   gch->rem_set()->prepare_for_younger_refs_iterate(true);
 988   // It turns out that even when we're using 1 thread, doing the work in a
 989   // separate thread causes wide variance in run times.  We can't help this
 990   // in the multi-threaded case, but we special-case n=1 here to get
 991   // repeatable measurements of the 1-thread overhead of the parallel code.
 992   if (n_workers > 1) {
 993     GenCollectedHeap::StrongRootsScope srs(gch);
 994     workers->run_task(&tsk);
 995   } else {
 996     GenCollectedHeap::StrongRootsScope srs(gch);
 997     tsk.work(0);
 998   }
 999   thread_state_set.reset(0 /* Bad value in debug if not reset */,
1000                          promotion_failed());
1001 
1002   // Process (weak) reference objects found during scavenge.
1003   ReferenceProcessor* rp = ref_processor();


1161         ParScanThreadState* par_scan_state, oop old, size_t sz, markOop m) {
1162   // In the sequential version, this assert also says that the object is
1163   // not forwarded.  That might not be the case here.  It is the case that
1164   // the caller observed it to be not forwarded at some time in the past.
1165   assert(is_in_reserved(old), "shouldn't be scavenging this oop");
1166 
1167   // The sequential code read "old->age()" below.  That doesn't work here,
1168   // since the age is in the mark word, and that might be overwritten with
1169   // a forwarding pointer by a parallel thread.  So we must save the mark
1170   // word in a local and then analyze it.
1171   oopDesc dummyOld;
1172   dummyOld.set_mark(m);
1173   assert(!dummyOld.is_forwarded(),
1174          "should not be called with forwarding pointer mark word.");
1175 
1176   oop new_obj = NULL;
1177   oop forward_ptr;
1178 
1179   // Try allocating obj in to-space (unless too old)
1180   if (dummyOld.age() < tenuring_threshold()) {
1181     new_obj = (oop)par_scan_state->alloc_in_to_space(sz);
1182     if (new_obj == NULL) {
1183       set_survivor_overflow(true);
1184     }
1185   }
1186 
1187   if (new_obj == NULL) {
1188     // Either to-space is full or we decided to promote
1189     // try allocating obj tenured
1190 
1191     // Attempt to install a null forwarding pointer (atomically),
1192     // to claim the right to install the real forwarding pointer.
1193     forward_ptr = old->forward_to_atomic(ClaimedForwardPtr);
1194     if (forward_ptr != NULL) {
1195       // someone else beat us to it.
1196         return real_forwardee(old);
1197     }
1198 
1199     new_obj = _next_gen->par_promote(par_scan_state->thread_num(),

1200                                        old, m, sz);
1201 
1202     if (new_obj == NULL) {
1203       // promotion failed, forward to self
1204       _promotion_failed = true;
1205       new_obj = old;
1206 
1207       preserve_mark_if_necessary(old, m);
1208       par_scan_state->register_promotion_failure(sz);
1209     }
1210 
1211     old->forward_to(new_obj);
1212     forward_ptr = NULL;
1213   } else {
1214     // Is in to-space; do copying ourselves.
1215     Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz);
1216     forward_ptr = old->forward_to_atomic(new_obj);
1217     // Restore the mark word copied above.
1218     new_obj->set_mark(m);
1219     // Increment age if obj still in new generation


1288   // In the sequential version, this assert also says that the object is
1289   // not forwarded.  That might not be the case here.  It is the case that
1290   // the caller observed it to be not forwarded at some time in the past.
1291   assert(is_in_reserved(old), "shouldn't be scavenging this oop");
1292 
1293   // The sequential code read "old->age()" below.  That doesn't work here,
1294   // since the age is in the mark word, and that might be overwritten with
1295   // a forwarding pointer by a parallel thread.  So we must save the mark
1296   // word here, install it in a local oopDesc, and then analyze it.
1297   oopDesc dummyOld;
1298   dummyOld.set_mark(m);
1299   assert(!dummyOld.is_forwarded(),
1300          "should not be called with forwarding pointer mark word.");
1301 
1302   bool failed_to_promote = false;
1303   oop new_obj = NULL;
1304   oop forward_ptr;
1305 
1306   // Try allocating obj in to-space (unless too old)
1307   if (dummyOld.age() < tenuring_threshold()) {
1308     new_obj = (oop)par_scan_state->alloc_in_to_space(sz);
1309     if (new_obj == NULL) {
1310       set_survivor_overflow(true);
1311     }
1312   }
1313 
1314   if (new_obj == NULL) {
1315     // Either to-space is full or we decided to promote
1316     // try allocating obj tenured
1317     new_obj = _next_gen->par_promote(par_scan_state->thread_num(),

1318                                        old, m, sz);
1319 
1320     if (new_obj == NULL) {
1321       // promotion failed, forward to self
1322       forward_ptr = old->forward_to_atomic(old);
1323       new_obj = old;
1324 
1325       if (forward_ptr != NULL) {
1326         return forward_ptr;   // someone else succeeded
1327       }
1328 
1329       _promotion_failed = true;
1330       failed_to_promote = true;
1331 
1332       preserve_mark_if_necessary(old, m);
1333       par_scan_state->register_promotion_failure(sz);
1334     }
1335   } else {
1336     // Is in to-space; do copying ourselves.
1337     Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz);




  52 #include "runtime/handles.inline.hpp"
  53 #include "runtime/java.hpp"
  54 #include "runtime/thread.inline.hpp"
  55 #include "utilities/copy.hpp"
  56 #include "utilities/globalDefinitions.hpp"
  57 #include "utilities/workgroup.hpp"
  58 
  59 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
  60 
  61 #ifdef _MSC_VER
  62 #pragma warning( push )
  63 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
  64 #endif
  65 ParScanThreadState::ParScanThreadState(Space* to_space_,
  66                                        ParNewGeneration* gen_,
  67                                        Generation* old_gen_,
  68                                        int thread_num_,
  69                                        ObjToScanQueueSet* work_queue_set_,
  70                                        Stack<oop, mtGC>* overflow_stacks_,
  71                                        size_t desired_plab_sz_,
  72                                        ParNewTracer* gc_tracer,
  73                                        ParallelTaskTerminator& term_) :
  74   _to_space(to_space_), _old_gen(old_gen_), _young_gen(gen_), _thread_num(thread_num_),
  75   _work_queue(work_queue_set_->queue(thread_num_)), _to_space_full(false),
  76   _overflow_stack(overflow_stacks_ ? overflow_stacks_ + thread_num_ : NULL),
  77   _ageTable(false), // false ==> not the global age table, no perf data.
  78   _to_space_alloc_buffer(desired_plab_sz_),
  79   _to_space_closure(gen_, this), _old_gen_closure(gen_, this),
  80   _to_space_root_closure(gen_, this), _old_gen_root_closure(gen_, this),
  81   _older_gen_closure(gen_, this),
  82   _evacuate_followers(this, &_to_space_closure, &_old_gen_closure,
  83                       &_to_space_root_closure, gen_, &_old_gen_root_closure,
  84                       work_queue_set_, &term_),
  85   _is_alive_closure(gen_), _scan_weak_ref_closure(gen_, this),
  86   _keep_alive_closure(&_scan_weak_ref_closure),
  87   _gc_tracer(gc_tracer),
  88   _strong_roots_time(0.0), _term_time(0.0)
  89 {
  90   #if TASKQUEUE_STATS
  91   _term_attempts = 0;
  92   _overflow_refills = 0;
  93   _overflow_refill_objs = 0;
  94   #endif // TASKQUEUE_STATS
  95 
  96   _survivor_chunk_array =
  97     (ChunkArray*) old_gen()->get_data_recorder(thread_num());
  98   _hash_seed = 17;  // Might want to take time-based random value.
  99   _start = os::elapsedTime();
 100   _old_gen_closure.set_generation(old_gen_);
 101   _old_gen_root_closure.set_generation(old_gen_);
 102 }
 103 #ifdef _MSC_VER
 104 #pragma warning( pop )
 105 #endif
 106 
 107 void ParScanThreadState::record_survivor_plab(HeapWord* plab_start,


 207     assert(Universe::heap()->is_in_reserved(cur), "Should be in heap");
 208     assert(!old_gen()->is_in_reserved(cur), "Should be in young gen");
 209     assert(Universe::heap()->is_in_reserved(obj_to_push), "Should be in heap");
 210     if (should_be_partially_scanned(obj_to_push, cur)) {
 211       assert(arrayOop(cur)->length() == 0, "entire array remaining to be scanned");
 212       obj_to_push = cur;
 213     }
 214     bool ok = queue->push(obj_to_push);
 215     assert(ok, "Should have succeeded");
 216   }
 217   assert(young_gen()->overflow_list() == NULL, "Error");
 218   return num_take_elems > 0;  // was something transferred?
 219 }
 220 
 221 void ParScanThreadState::push_on_overflow_stack(oop p) {
 222   assert(ParGCUseLocalOverflow, "Else should not call");
 223   overflow_stack()->push(p);
 224   assert(young_gen()->overflow_list() == NULL, "Error");
 225 }
 226 
 227 HeapWord* ParScanThreadState::alloc_in_to_space_slow(size_t word_sz, 
 228                                                      const oop old, 
 229                                                      const uint age) {
 230 
 231   // Otherwise, if the object is small enough, try to reallocate the
 232   // buffer.
 233   HeapWord* obj = NULL;
 234   if (!_to_space_full) {
 235     ParGCAllocBuffer* const plab = to_space_alloc_buffer();
 236     Space*            const sp   = to_space();
 237     if (word_sz * 100 <
 238         ParallelGCBufferWastePct * plab->word_sz()) {
 239       // Is small enough; abandon this buffer and start a new one.
 240       plab->retire(false, false);
 241       size_t buf_size = plab->word_sz();
 242       HeapWord* buf_space = sp->par_allocate(buf_size);
 243       if (buf_space == NULL) {
 244         const size_t min_bytes =
 245           ParGCAllocBuffer::min_size() << LogHeapWordSize;
 246         size_t free_bytes = sp->free();
 247         while(buf_space == NULL && free_bytes >= min_bytes) {
 248           buf_size = free_bytes >> LogHeapWordSize;
 249           assert(buf_size == (size_t)align_object_size(buf_size),
 250                  "Invariant");
 251           buf_space  = sp->par_allocate(buf_size);
 252           free_bytes = sp->free();
 253         }
 254       }
 255       if (buf_space != NULL) {
 256         plab->set_word_size(buf_size);
 257         plab->set_buf(buf_space);
 258         record_survivor_plab(buf_space, buf_size);
 259         gc_tracer()->report_promotion_in_new_plab_event(old, word_sz, age, false,
 260                                                         buf_size);
 261         obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes);
 262         // Note that we cannot compare buf_size < word_sz below
 263         // because of AlignmentReserve (see ParGCAllocBuffer::allocate()).
 264         assert(obj != NULL || plab->words_remaining() < word_sz,
 265                "Else should have been able to allocate");
 266         // It's conceivable that we may be able to use the
 267         // buffer we just grabbed for subsequent small requests
 268         // even if not for this one.
 269       } else {
 270         // We're used up.
 271         _to_space_full = true;
 272       }
 273 
 274     } else {
 275       // Too large; allocate the object individually.
 276       gc_tracer()->report_promotion_outside_plab_event(old, word_sz, age, false);
 277       obj = sp->par_allocate(word_sz);
 278     }
 279   }
 280   return obj;
 281 }
 282 
 283 
 284 void ParScanThreadState::undo_alloc_in_to_space(HeapWord* obj,
 285                                                 size_t word_sz) {
 286   // Is the alloc in the current alloc buffer?
 287   if (to_space_alloc_buffer()->contains(obj)) {
 288     assert(to_space_alloc_buffer()->contains(obj + word_sz - 1),
 289            "Should contain whole object.");
 290     to_space_alloc_buffer()->undo_allocation(obj, word_sz);
 291   } else {
 292     CollectedHeap::fill_with_object(obj, word_sz);
 293   }
 294 }
 295 
 296 void ParScanThreadState::print_promotion_failure_size() {
 297   if (_promotion_failed_info.has_failed() && PrintPromotionFailure) {
 298     gclog_or_tty->print(" (%d: promotion failure size = " SIZE_FORMAT ") ",
 299                         _thread_num, _promotion_failed_info.first_size());
 300   }
 301 }
 302 
 303 class ParScanThreadStateSet: private ResourceArray {
 304 public:
 305   // Initializes states for the specified number of threads;
 306   ParScanThreadStateSet(int                     num_threads,
 307                         Space&                  to_space,
 308                         ParNewGeneration&       gen,
 309                         Generation&             old_gen,
 310                         ObjToScanQueueSet&      queue_set,
 311                         Stack<oop, mtGC>*       overflow_stacks_,
 312                         size_t                  desired_plab_sz,
 313                         ParNewTracer*           gc_tracer,
 314                         ParallelTaskTerminator& term);
 315 
 316   ~ParScanThreadStateSet() { TASKQUEUE_STATS_ONLY(reset_stats()); }
 317 
 318   inline ParScanThreadState& thread_state(int i);
 319 
 320   void trace_promotion_failed(YoungGCTracer& gc_tracer);
 321   void reset(int active_workers, bool promotion_failed);
 322   void flush();
 323 
 324   #if TASKQUEUE_STATS
 325   static void
 326     print_termination_stats_hdr(outputStream* const st = gclog_or_tty);
 327   void print_termination_stats(outputStream* const st = gclog_or_tty);
 328   static void
 329     print_taskqueue_stats_hdr(outputStream* const st = gclog_or_tty);
 330   void print_taskqueue_stats(outputStream* const st = gclog_or_tty);
 331   void reset_stats();
 332   #endif // TASKQUEUE_STATS
 333 
 334 private:
 335   ParallelTaskTerminator& _term;
 336   ParNewGeneration&       _gen;
 337   Generation&             _next_gen;
 338  public:
 339   bool is_valid(int id) const { return id < length(); }
 340   ParallelTaskTerminator* terminator() { return &_term; }
 341 };
 342 
 343 
 344 ParScanThreadStateSet::ParScanThreadStateSet(
 345   int num_threads, Space& to_space, ParNewGeneration& gen,
 346   Generation& old_gen, ObjToScanQueueSet& queue_set,
 347   Stack<oop, mtGC>* overflow_stacks,
 348   size_t desired_plab_sz, ParNewTracer* gc_tracer, 
 349   ParallelTaskTerminator& term)
 350   : ResourceArray(sizeof(ParScanThreadState), num_threads),
 351     _gen(gen), _next_gen(old_gen), _term(term)
 352 {
 353   assert(num_threads > 0, "sanity check!");
 354   assert(ParGCUseLocalOverflow == (overflow_stacks != NULL),
 355          "overflow_stack allocation mismatch");
 356   // Initialize states.
 357   for (int i = 0; i < num_threads; ++i) {
 358     new ((ParScanThreadState*)_data + i)
 359         ParScanThreadState(&to_space, &gen, &old_gen, i, &queue_set,
 360                            overflow_stacks, desired_plab_sz, gc_tracer, term);
 361   }
 362 }
 363 
 364 inline ParScanThreadState& ParScanThreadStateSet::thread_state(int i)
 365 {
 366   assert(i >= 0 && i < length(), "sanity check!");
 367   return ((ParScanThreadState*)_data)[i];
 368 }
 369 
 370 void ParScanThreadStateSet::trace_promotion_failed(YoungGCTracer& gc_tracer) {
 371   for (int i = 0; i < length(); ++i) {
 372     if (thread_state(i).promotion_failed()) {
 373       gc_tracer.report_promotion_failed(thread_state(i).promotion_failed_info());
 374       thread_state(i).promotion_failed_info().reset();
 375     }
 376   }
 377 }
 378 
 379 void ParScanThreadStateSet::reset(int active_threads, bool promotion_failed)
 380 {


 972   // Capture heap used before collection (for printing).
 973   size_t gch_prev_used = gch->used();
 974 
 975   SpecializationStats::clear();
 976 
 977   age_table()->clear();
 978   to()->clear(SpaceDecorator::Mangle);
 979 
 980   gch->save_marks();
 981   assert(workers != NULL, "Need parallel worker threads.");
 982   int n_workers = active_workers;
 983 
 984   // Set the correct parallelism (number of queues) in the reference processor
 985   ref_processor()->set_active_mt_degree(n_workers);
 986 
 987   // Always set the terminator for the active number of workers
 988   // because only those workers go through the termination protocol.
 989   ParallelTaskTerminator _term(n_workers, task_queues());
 990   ParScanThreadStateSet thread_state_set(workers->active_workers(),
 991                                          *to(), *this, *_next_gen, *task_queues(),
 992                                          _overflow_stacks, desired_plab_sz(), 
 993                                          &gc_tracer, _term);
 994 
 995   ParNewGenTask tsk(this, _next_gen, reserved().end(), &thread_state_set);
 996   gch->set_par_threads(n_workers);
 997   gch->rem_set()->prepare_for_younger_refs_iterate(true);
 998   // It turns out that even when we're using 1 thread, doing the work in a
 999   // separate thread causes wide variance in run times.  We can't help this
1000   // in the multi-threaded case, but we special-case n=1 here to get
1001   // repeatable measurements of the 1-thread overhead of the parallel code.
1002   if (n_workers > 1) {
1003     GenCollectedHeap::StrongRootsScope srs(gch);
1004     workers->run_task(&tsk);
1005   } else {
1006     GenCollectedHeap::StrongRootsScope srs(gch);
1007     tsk.work(0);
1008   }
1009   thread_state_set.reset(0 /* Bad value in debug if not reset */,
1010                          promotion_failed());
1011 
1012   // Process (weak) reference objects found during scavenge.
1013   ReferenceProcessor* rp = ref_processor();


1171         ParScanThreadState* par_scan_state, oop old, size_t sz, markOop m) {
1172   // In the sequential version, this assert also says that the object is
1173   // not forwarded.  That might not be the case here.  It is the case that
1174   // the caller observed it to be not forwarded at some time in the past.
1175   assert(is_in_reserved(old), "shouldn't be scavenging this oop");
1176 
1177   // The sequential code read "old->age()" below.  That doesn't work here,
1178   // since the age is in the mark word, and that might be overwritten with
1179   // a forwarding pointer by a parallel thread.  So we must save the mark
1180   // word in a local and then analyze it.
1181   oopDesc dummyOld;
1182   dummyOld.set_mark(m);
1183   assert(!dummyOld.is_forwarded(),
1184          "should not be called with forwarding pointer mark word.");
1185 
1186   oop new_obj = NULL;
1187   oop forward_ptr;
1188 
1189   // Try allocating obj in to-space (unless too old)
1190   if (dummyOld.age() < tenuring_threshold()) {
1191     new_obj = (oop)par_scan_state->alloc_in_to_space(sz, old, dummyOld.age());
1192     if (new_obj == NULL) {
1193       set_survivor_overflow(true);
1194     }
1195   }
1196 
1197   if (new_obj == NULL) {
1198     // Either to-space is full or we decided to promote
1199     // try allocating obj tenured
1200 
1201     // Attempt to install a null forwarding pointer (atomically),
1202     // to claim the right to install the real forwarding pointer.
1203     forward_ptr = old->forward_to_atomic(ClaimedForwardPtr);
1204     if (forward_ptr != NULL) {
1205       // someone else beat us to it.
1206         return real_forwardee(old);
1207     }
1208 
1209     new_obj = _next_gen->par_promote(par_scan_state->gc_tracer(),
1210                                      par_scan_state->thread_num(),
1211                                      old, m, sz);
1212 
1213     if (new_obj == NULL) {
1214       // promotion failed, forward to self
1215       _promotion_failed = true;
1216       new_obj = old;
1217 
1218       preserve_mark_if_necessary(old, m);
1219       par_scan_state->register_promotion_failure(sz);
1220     }
1221 
1222     old->forward_to(new_obj);
1223     forward_ptr = NULL;
1224   } else {
1225     // Is in to-space; do copying ourselves.
1226     Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz);
1227     forward_ptr = old->forward_to_atomic(new_obj);
1228     // Restore the mark word copied above.
1229     new_obj->set_mark(m);
1230     // Increment age if obj still in new generation


1299   // In the sequential version, this assert also says that the object is
1300   // not forwarded.  That might not be the case here.  It is the case that
1301   // the caller observed it to be not forwarded at some time in the past.
1302   assert(is_in_reserved(old), "shouldn't be scavenging this oop");
1303 
1304   // The sequential code read "old->age()" below.  That doesn't work here,
1305   // since the age is in the mark word, and that might be overwritten with
1306   // a forwarding pointer by a parallel thread.  So we must save the mark
1307   // word here, install it in a local oopDesc, and then analyze it.
1308   oopDesc dummyOld;
1309   dummyOld.set_mark(m);
1310   assert(!dummyOld.is_forwarded(),
1311          "should not be called with forwarding pointer mark word.");
1312 
1313   bool failed_to_promote = false;
1314   oop new_obj = NULL;
1315   oop forward_ptr;
1316 
1317   // Try allocating obj in to-space (unless too old)
1318   if (dummyOld.age() < tenuring_threshold()) {
1319     new_obj = (oop)par_scan_state->alloc_in_to_space(sz, old, dummyOld.age());
1320     if (new_obj == NULL) {
1321       set_survivor_overflow(true);
1322     }
1323   }
1324 
1325   if (new_obj == NULL) {
1326     // Either to-space is full or we decided to promote
1327     // try allocating obj tenured
1328     new_obj = _next_gen->par_promote(par_scan_state->gc_tracer(),
1329                                      par_scan_state->thread_num(),
1330                                      old, m, sz);
1331 
1332     if (new_obj == NULL) {
1333       // promotion failed, forward to self
1334       forward_ptr = old->forward_to_atomic(old);
1335       new_obj = old;
1336 
1337       if (forward_ptr != NULL) {
1338         return forward_ptr;   // someone else succeeded
1339       }
1340 
1341       _promotion_failed = true;
1342       failed_to_promote = true;
1343 
1344       preserve_mark_if_necessary(old, m);
1345       par_scan_state->register_promotion_failure(sz);
1346     }
1347   } else {
1348     // Is in to-space; do copying ourselves.
1349     Copy::aligned_disjoint_words((HeapWord*)old, (HeapWord*)new_obj, sz);