105   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 106     bool oops_into_cset = G1CollectedHeap::heap()->g1_rem_set()->refine_card(card_ptr, worker_i, false);
 107     // This path is executed by the concurrent refine or mutator threads,
 108     // concurrently, and so we do not care if card_ptr contains references
 109     // that point into the collection set.
 110     assert(!oops_into_cset, "should be");
 111 
 112     if (_concurrent && SuspendibleThreadSet::should_yield()) {
 113       // Caller will actually yield.
 114       return false;
 115     }
 116     // Otherwise, we finished successfully; return true.
 117     return true;
 118   }
 119 
 120   void set_concurrent(bool b) { _concurrent = b; }
 121 };
 122 
 123 
 124 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {
 125   int _calls;
 126   G1CollectedHeap* _g1h;
 127   CardTableModRefBS* _ctbs;
 128   int _histo[256];
 129 public:

 130   ClearLoggedCardTableEntryClosure() :
 131     _calls(0), _g1h(G1CollectedHeap::heap()), _ctbs(_g1h->g1_barrier_set())
 132   {
 133     for (int i = 0; i < 256; i++) _histo[i] = 0;
 134   }

 135   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 136     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {
 137       _calls++;
 138       unsigned char* ujb = (unsigned char*)card_ptr;
 139       int ind = (int)(*ujb);
 140       _histo[ind]++;
 141       *card_ptr = -1;
 142     }


 143     return true;
 144   }
 145   int calls() { return _calls; }


 146   void print_histo() {
 147     gclog_or_tty->print_cr("Card table value histogram:");
 148     for (int i = 0; i < 256; i++) {
 149       if (_histo[i] != 0) {
 150         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);
 151       }
 152     }
 153   }
 154 };
 155 
 156 class RedirtyLoggedCardTableEntryClosure: public CardTableEntryClosure {
 157   int _calls;
 158   G1CollectedHeap* _g1h;
 159   CardTableModRefBS* _ctbs;
 160 public:
 161   RedirtyLoggedCardTableEntryClosure() :
 162     _calls(0), _g1h(G1CollectedHeap::heap()), _ctbs(_g1h->g1_barrier_set()) {}
 163 
 164   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 165     if (_g1h->is_in_reserved(_ctbs->addr_for(card_ptr))) {
 166       _calls++;
 167       *card_ptr = 0;
 168     }
 169     return true;
 170   }
 171   int calls() { return _calls; }

 172 };
 173 
 174 YoungList::YoungList(G1CollectedHeap* g1h) :
 175     _g1h(g1h), _head(NULL), _length(0), _last_sampled_rs_lengths(0),
 176     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0) {
 177   guarantee(check_list_empty(false), "just making sure...");
 178 }
 179 
 180 void YoungList::push_region(HeapRegion *hr) {
 181   assert(!hr->is_young(), "should not already be young");
 182   assert(hr->get_next_young_region() == NULL, "cause it should!");
 183 
 184   hr->set_next_young_region(_head);
 185   _head = hr;
 186 
 187   _g1h->g1_policy()->set_region_eden(hr, (int) _length);
 188   ++_length;
 189 }
 190 
 191 void YoungList::add_survivor_region(HeapRegion* hr) {

 472 
 473   // First clear the logged cards.
 474   ClearLoggedCardTableEntryClosure clear;
 475   dcqs.apply_closure_to_all_completed_buffers(&clear);
 476   dcqs.iterate_closure_all_threads(&clear, false);
 477   clear.print_histo();
 478 
 479   // Now ensure that there's no dirty cards.
 480   CountNonCleanMemRegionClosure count2(this);
 481   ct_bs->mod_card_iterate(&count2);
 482   if (count2.n() != 0) {
 483     gclog_or_tty->print_cr("Card table has %d entries; %d originally",
 484                            count2.n(), orig_count);
 485   }
 486   guarantee(count2.n() == 0, "Card table should be clean.");
 487 
 488   RedirtyLoggedCardTableEntryClosure redirty;
 489   dcqs.apply_closure_to_all_completed_buffers(&redirty);
 490   dcqs.iterate_closure_all_threads(&redirty, false);
 491   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",
 492                          clear.calls(), orig_count);
 493   guarantee(redirty.calls() == clear.calls(),
 494             "Or else mechanism is broken.");

 495 
 496   CountNonCleanMemRegionClosure count3(this);
 497   ct_bs->mod_card_iterate(&count3);
 498   if (count3.n() != orig_count) {
 499     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",
 500                            orig_count, count3.n());
 501     guarantee(count3.n() >= orig_count, "Should have restored them all.");
 502   }
 503 }
 504 
 505 // Private class members.
 506 
 507 G1CollectedHeap* G1CollectedHeap::_g1h;
 508 
 509 // Private methods.
 510 
 511 HeapRegion*
 512 G1CollectedHeap::new_region_try_secondary_free_list(bool is_old) {
 513   MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
 514   while (!_secondary_free_list.is_empty() || free_regions_coming()) {

5259   if (G1CollectedHeap::use_parallel_gc_threads()) {
5260     set_par_threads(n_workers);
5261     workers()->run_task(&g1_unlink_task);
5262     set_par_threads(0);
5263   } else {
5264     g1_unlink_task.work(0);
5265   }
5266   if (G1TraceStringSymbolTableScrubbing) {
5267     gclog_or_tty->print_cr("Cleaned string and symbol table, "
5268                            "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, "
5269                            "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed",
5270                            g1_unlink_task.strings_processed(), g1_unlink_task.strings_removed(),
5271                            g1_unlink_task.symbols_processed(), g1_unlink_task.symbols_removed());
5272   }
5273 
5274   if (G1StringDedup::is_enabled()) {
5275     G1StringDedup::unlink(is_alive);
5276   }
5277 }
5278 
5279 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure {
5280  private:
5281   size_t _num_processed;
5282 
5283  public:
5284   RedirtyLoggedCardTableEntryFastClosure() : CardTableEntryClosure(), _num_processed(0) { }
5285 
5286   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
5287     *card_ptr = CardTableModRefBS::dirty_card_val();
5288     _num_processed++;
5289     return true;
5290   }
5291 
5292   size_t num_processed() const { return _num_processed; }
5293 };
5294 
5295 class G1RedirtyLoggedCardsTask : public AbstractGangTask {
5296  private:
5297   DirtyCardQueueSet* _queue;
5298  public:
5299   G1RedirtyLoggedCardsTask(DirtyCardQueueSet* queue) : AbstractGangTask("Redirty Cards"), _queue(queue) { }
5300 
5301   virtual void work(uint worker_id) {
5302     double start_time = os::elapsedTime();
5303 
5304     RedirtyLoggedCardTableEntryFastClosure cl;
5305     if (G1CollectedHeap::heap()->use_parallel_gc_threads()) {
5306       _queue->par_apply_closure_to_all_completed_buffers(&cl);
5307     } else {
5308       _queue->apply_closure_to_all_completed_buffers(&cl);
5309     }
5310 
5311     G1GCPhaseTimes* timer = G1CollectedHeap::heap()->g1_policy()->phase_times();
5312     timer->record_redirty_logged_cards_time_ms(worker_id, (os::elapsedTime() - start_time) * 1000.0);
5313     timer->record_redirty_logged_cards_processed_cards(worker_id, cl.num_processed());
5314   }
5315 };
5316 
5317 void G1CollectedHeap::redirty_logged_cards() {
5318   guarantee(G1DeferredRSUpdate, "Must only be called when using deferred RS updates.");
5319   double redirty_logged_cards_start = os::elapsedTime();
5320 
5321   uint n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
5322                    _g1h->workers()->active_workers() : 1);
5323 
5324   G1RedirtyLoggedCardsTask redirty_task(&dirty_card_queue_set());

 105   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 106     bool oops_into_cset = G1CollectedHeap::heap()->g1_rem_set()->refine_card(card_ptr, worker_i, false);
 107     // This path is executed by the concurrent refine or mutator threads,
 108     // concurrently, and so we do not care if card_ptr contains references
 109     // that point into the collection set.
 110     assert(!oops_into_cset, "should be");
 111 
 112     if (_concurrent && SuspendibleThreadSet::should_yield()) {
 113       // Caller will actually yield.
 114       return false;
 115     }
 116     // Otherwise, we finished successfully; return true.
 117     return true;
 118   }
 119 
 120   void set_concurrent(bool b) { _concurrent = b; }
 121 };
 122 
 123 
 124 class ClearLoggedCardTableEntryClosure: public CardTableEntryClosure {
 125   size_t _num_processed;

 126   CardTableModRefBS* _ctbs;
 127   int _histo[256];
 128 
 129  public:
 130   ClearLoggedCardTableEntryClosure() :
 131     _num_processed(0), _ctbs(G1CollectedHeap::heap()->g1_barrier_set())
 132   {
 133     for (int i = 0; i < 256; i++) _histo[i] = 0;
 134   }
 135 
 136   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {


 137     unsigned char* ujb = (unsigned char*)card_ptr;
 138     int ind = (int)(*ujb);
 139     _histo[ind]++;
 140 
 141     *card_ptr = (jbyte)CardTableModRefBS::clean_card_val();
 142     _num_processed++;
 143 
 144     return true;
 145   }
 146 
 147   size_t num_processed() { return _num_processed; }
 148 
 149   void print_histo() {
 150     gclog_or_tty->print_cr("Card table value histogram:");
 151     for (int i = 0; i < 256; i++) {
 152       if (_histo[i] != 0) {
 153         gclog_or_tty->print_cr("  %d: %d", i, _histo[i]);
 154       }
 155     }
 156   }
 157 };
 158 
 159 class RedirtyLoggedCardTableEntryClosure : public CardTableEntryClosure {
 160  private:
 161   size_t _num_processed;
 162 
 163  public:
 164   RedirtyLoggedCardTableEntryClosure() : CardTableEntryClosure(), _num_processed(0) { }

 165 
 166   bool do_card_ptr(jbyte* card_ptr, uint worker_i) {
 167     *card_ptr = CardTableModRefBS::dirty_card_val();
 168     _num_processed++;


 169     return true;
 170   }
 171 
 172   size_t num_processed() const { return _num_processed; }
 173 };
 174 
 175 YoungList::YoungList(G1CollectedHeap* g1h) :
 176     _g1h(g1h), _head(NULL), _length(0), _last_sampled_rs_lengths(0),
 177     _survivor_head(NULL), _survivor_tail(NULL), _survivor_length(0) {
 178   guarantee(check_list_empty(false), "just making sure...");
 179 }
 180 
 181 void YoungList::push_region(HeapRegion *hr) {
 182   assert(!hr->is_young(), "should not already be young");
 183   assert(hr->get_next_young_region() == NULL, "cause it should!");
 184 
 185   hr->set_next_young_region(_head);
 186   _head = hr;
 187 
 188   _g1h->g1_policy()->set_region_eden(hr, (int) _length);
 189   ++_length;
 190 }
 191 
 192 void YoungList::add_survivor_region(HeapRegion* hr) {

 473 
 474   // First clear the logged cards.
 475   ClearLoggedCardTableEntryClosure clear;
 476   dcqs.apply_closure_to_all_completed_buffers(&clear);
 477   dcqs.iterate_closure_all_threads(&clear, false);
 478   clear.print_histo();
 479 
 480   // Now ensure that there's no dirty cards.
 481   CountNonCleanMemRegionClosure count2(this);
 482   ct_bs->mod_card_iterate(&count2);
 483   if (count2.n() != 0) {
 484     gclog_or_tty->print_cr("Card table has %d entries; %d originally",
 485                            count2.n(), orig_count);
 486   }
 487   guarantee(count2.n() == 0, "Card table should be clean.");
 488 
 489   RedirtyLoggedCardTableEntryClosure redirty;
 490   dcqs.apply_closure_to_all_completed_buffers(&redirty);
 491   dcqs.iterate_closure_all_threads(&redirty, false);
 492   gclog_or_tty->print_cr("Log entries = %d, dirty cards = %d.",
 493                          clear.num_processed(), orig_count);
 494   guarantee(redirty.num_processed() == clear.num_processed(),
 495             err_msg("Redirtied "SIZE_FORMAT" cards, bug cleared "SIZE_FORMAT,
 496                     redirty.num_processed(), clear.num_processed()));
 497 
 498   CountNonCleanMemRegionClosure count3(this);
 499   ct_bs->mod_card_iterate(&count3);
 500   if (count3.n() != orig_count) {
 501     gclog_or_tty->print_cr("Should have restored them all: orig = %d, final = %d.",
 502                            orig_count, count3.n());
 503     guarantee(count3.n() >= orig_count, "Should have restored them all.");
 504   }
 505 }
 506 
 507 // Private class members.
 508 
 509 G1CollectedHeap* G1CollectedHeap::_g1h;
 510 
 511 // Private methods.
 512 
 513 HeapRegion*
 514 G1CollectedHeap::new_region_try_secondary_free_list(bool is_old) {
 515   MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
 516   while (!_secondary_free_list.is_empty() || free_regions_coming()) {

5261   if (G1CollectedHeap::use_parallel_gc_threads()) {
5262     set_par_threads(n_workers);
5263     workers()->run_task(&g1_unlink_task);
5264     set_par_threads(0);
5265   } else {
5266     g1_unlink_task.work(0);
5267   }
5268   if (G1TraceStringSymbolTableScrubbing) {
5269     gclog_or_tty->print_cr("Cleaned string and symbol table, "
5270                            "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, "
5271                            "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed",
5272                            g1_unlink_task.strings_processed(), g1_unlink_task.strings_removed(),
5273                            g1_unlink_task.symbols_processed(), g1_unlink_task.symbols_removed());
5274   }
5275 
5276   if (G1StringDedup::is_enabled()) {
5277     G1StringDedup::unlink(is_alive);
5278   }
5279 }
5280 
















5281 class G1RedirtyLoggedCardsTask : public AbstractGangTask {
5282  private:
5283   DirtyCardQueueSet* _queue;
5284  public:
5285   G1RedirtyLoggedCardsTask(DirtyCardQueueSet* queue) : AbstractGangTask("Redirty Cards"), _queue(queue) { }
5286 
5287   virtual void work(uint worker_id) {
5288     double start_time = os::elapsedTime();
5289 
5290     RedirtyLoggedCardTableEntryClosure cl;
5291     if (G1CollectedHeap::heap()->use_parallel_gc_threads()) {
5292       _queue->par_apply_closure_to_all_completed_buffers(&cl);
5293     } else {
5294       _queue->apply_closure_to_all_completed_buffers(&cl);
5295     }
5296 
5297     G1GCPhaseTimes* timer = G1CollectedHeap::heap()->g1_policy()->phase_times();
5298     timer->record_redirty_logged_cards_time_ms(worker_id, (os::elapsedTime() - start_time) * 1000.0);
5299     timer->record_redirty_logged_cards_processed_cards(worker_id, cl.num_processed());
5300   }
5301 };
5302 
5303 void G1CollectedHeap::redirty_logged_cards() {
5304   guarantee(G1DeferredRSUpdate, "Must only be called when using deferred RS updates.");
5305   double redirty_logged_cards_start = os::elapsedTime();
5306 
5307   uint n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
5308                    _g1h->workers()->active_workers() : 1);
5309 
5310   G1RedirtyLoggedCardsTask redirty_task(&dirty_card_queue_set());