1 /*
   2  * Copyright (c) 2014, 2017, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #include "precompiled.hpp"
  25 
  26 #include "classfile/javaClasses.inline.hpp"
  27 #include "code/codeCache.hpp"
  28 #include "gc/shared/gcTraceTime.inline.hpp"
  29 #include "gc/shenandoah/brooksPointer.hpp"
  30 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
  31 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  32 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  33 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
  34 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  35 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  36 #include "gc/shenandoah/shenandoahHeap.hpp"
  37 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  38 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
  39 #include "gc/shenandoah/shenandoahStringDedup.hpp"
  40 #include "gc/shenandoah/shenandoahUtils.hpp"
  41 #include "gc/shenandoah/shenandoahVerifier.hpp"
  42 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  43 #include "gc/shenandoah/vm_operations_shenandoah.hpp"
  44 #include "oops/oop.inline.hpp"
  45 #include "runtime/biasedLocking.hpp"
  46 #include "runtime/thread.hpp"
  47 #include "utilities/copy.hpp"
  48 #include "gc/shared/taskqueue.inline.hpp"
  49 #include "gc/shared/workgroup.hpp"
  50 
  51 class ShenandoahMarkCompactBarrierSet : public ShenandoahBarrierSet {
  52 public:
  53   ShenandoahMarkCompactBarrierSet(ShenandoahHeap* heap) : ShenandoahBarrierSet(heap) {
  54   }
  55   oop read_barrier(oop src) {
  56     return src;
  57   }
  58 #ifdef ASSERT
  59   bool is_safe(oop o) {
  60     if (o == NULL) return true;
  61     if (! oopDesc::unsafe_equals(o, read_barrier(o))) {
  62       return false;
  63     }
  64     return true;
  65   }
  66   bool is_safe(narrowOop o) {
  67     oop obj = oopDesc::decode_heap_oop(o);
  68     return is_safe(obj);
  69   }
  70 #endif
  71 };
  72 
  73 class ShenandoahClearRegionStatusClosure: public ShenandoahHeapRegionClosure {
  74 private:
  75   ShenandoahHeap* _heap;
  76 
  77 public:
  78   ShenandoahClearRegionStatusClosure() : _heap(ShenandoahHeap::heap()) {}
  79 
  80   bool heap_region_do(ShenandoahHeapRegion *r) {
  81     _heap->set_top_at_mark_start(r->bottom(), r->top());
  82     r->clear_live_data();
  83     r->set_concurrent_iteration_safe_limit(r->top());
  84     return false;
  85   }
  86 };
  87 
  88 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
  89 private:
  90   ShenandoahHeap* _heap;
  91 
  92 public:
  93   ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
  94   bool heap_region_do(ShenandoahHeapRegion* r) {
  95     if (r->is_trash()) {
  96       r->recycle();
  97     }
  98     if (r->is_empty()) {
  99       r->make_regular_bypass();
 100     }
 101     assert (r->is_active(), "only active regions in heap now");
 102     return false;
 103   }
 104 };
 105 
 106 STWGCTimer* ShenandoahMarkCompact::_gc_timer = NULL;
 107 
 108 void ShenandoahMarkCompact::initialize() {
 109   _gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer();
 110 }
 111 
 112 void ShenandoahMarkCompact::do_mark_compact(GCCause::Cause gc_cause) {
 113   ShenandoahHeap* heap = ShenandoahHeap::heap();
 114 
 115   // Default, use number of parallel GC threads
 116   WorkGang* workers = heap->workers();
 117   uint nworkers = ShenandoahWorkerPolicy::calc_workers_for_fullgc();
 118   ShenandoahWorkerScope full_gc_worker_scope(workers, nworkers);
 119 
 120   {
 121     ShenandoahGCSession session(/* is_full_gc */true);
 122     heap->set_full_gc_in_progress(true);
 123 
 124     assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
 125     assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
 126 
 127     {
 128       ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
 129       heap->pre_full_gc_dump(_gc_timer);
 130     }
 131 
 132     {
 133       ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
 134       // Full GC is supposed to recover from any GC state:
 135 
 136       // a. Cancel concurrent mark, if in progress
 137       if (heap->concurrent_mark_in_progress()) {
 138         heap->concurrentMark()->cancel();
 139         heap->stop_concurrent_marking();
 140       }
 141       assert(!heap->concurrent_mark_in_progress(), "sanity");
 142 
 143       // b. Cancel evacuation, if in progress
 144       if (heap->is_evacuation_in_progress()) {
 145         heap->set_evacuation_in_progress_at_safepoint(false);
 146       }
 147       assert(!heap->is_evacuation_in_progress(), "sanity");
 148 
 149       // c. Reset the bitmaps for new marking
 150       heap->reset_mark_bitmap(heap->workers());
 151       assert(heap->is_bitmap_clear(), "sanity");
 152 
 153       // d. Abandon reference discovery and clear all discovered references.
 154       ReferenceProcessor* rp = heap->ref_processor();
 155       rp->disable_discovery();
 156       rp->abandon_partial_discovery();
 157       rp->verify_no_references_recorded();
 158 
 159       // e. Verify heap before changing the regions
 160       if (ShenandoahVerify) {
 161         // Full GC should only be called between regular concurrent cycles, therefore
 162         // those verifications should be valid.
 163         heap->verifier()->verify_before_fullgc();
 164       }
 165 
 166       {
 167         ShenandoahHeapLocker lock(heap->lock());
 168 
 169         // f. Make sure all regions are active. This is needed because we are potentially
 170         // sliding the data through them
 171         ShenandoahEnsureHeapActiveClosure ecl;
 172         heap->heap_region_iterate(&ecl, false, false);
 173 
 174         // g. Clear region statuses, including collection set status
 175         ShenandoahClearRegionStatusClosure cl;
 176         heap->heap_region_iterate(&cl, false, false);
 177       }
 178     }
 179 
 180     BarrierSet* old_bs = oopDesc::bs();
 181     ShenandoahMarkCompactBarrierSet bs(heap);
 182     oopDesc::set_bs(&bs);
 183 
 184     {
 185       GCTraceTime(Info, gc) time("Pause Full", _gc_timer, gc_cause, true);
 186 
 187       if (UseTLAB) {
 188         heap->ensure_parsability(true);
 189       }
 190 
 191       CodeCache::gc_prologue();
 192 
 193       // We should save the marks of the currently locked biased monitors.
 194       // The marking doesn't preserve the marks of biased objects.
 195       //BiasedLocking::preserve_marks();
 196 
 197       heap->set_need_update_refs(true);
 198 
 199       // Setup workers for phase 1
 200       {
 201         OrderAccess::fence();
 202 
 203         ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
 204         phase1_mark_heap();
 205       }
 206 
 207       // Setup workers for the rest
 208       {
 209         OrderAccess::fence();
 210 
 211         ShenandoahHeapRegionSet** copy_queues = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
 212 
 213         {
 214           ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
 215           phase2_calculate_target_addresses(copy_queues);
 216         }
 217 
 218         OrderAccess::fence();
 219 
 220         {
 221           ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
 222           phase3_update_references();
 223         }
 224 
 225         if (ShenandoahStringDedup::is_enabled()) {
 226           ShenandoahGCPhase update_str_dedup_table(ShenandoahPhaseTimings::full_gc_update_str_dedup_table);
 227           ShenandoahStringDedup::parallel_full_gc_update_or_unlink();
 228         }
 229 
 230 
 231         {
 232           ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
 233           phase4_compact_objects(copy_queues);
 234         }
 235 
 236         FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, copy_queues);
 237 
 238         CodeCache::gc_epilogue();
 239         JvmtiExport::gc_epilogue();
 240       }
 241 
 242       // refs processing: clean slate
 243       // rp.enqueue_discovered_references();
 244 
 245       if (ShenandoahVerify) {
 246         heap->verifier()->verify_after_fullgc();
 247       }
 248 
 249       heap->set_bytes_allocated_since_cm(0);
 250 
 251       heap->set_need_update_refs(false);
 252 
 253       heap->set_full_gc_in_progress(false);
 254     }
 255 
 256     {
 257       ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
 258       heap->post_full_gc_dump(_gc_timer);
 259     }
 260 
 261     if (UseTLAB) {
 262       ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_resize_tlabs);
 263       heap->resize_all_tlabs();
 264     }
 265 
 266     oopDesc::set_bs(old_bs);
 267   }
 268 
 269 
 270   if (UseShenandoahMatrix && PrintShenandoahMatrix) {
 271     LogTarget(Info, gc) lt;
 272     LogStream ls(lt);
 273     heap->connection_matrix()->print_on(&ls);
 274   }
 275 }
 276 
 277 void ShenandoahMarkCompact::phase1_mark_heap() {
 278   GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
 279   ShenandoahHeap* _heap = ShenandoahHeap::heap();
 280 
 281   ShenandoahConcurrentMark* cm = _heap->concurrentMark();
 282 
 283   // Do not trust heuristics, because this can be our last resort collection.
 284   // Only ignore processing references and class unloading if explicitly disabled.
 285   cm->set_process_references(ShenandoahRefProcFrequency != 0);
 286   cm->set_unload_classes(ShenandoahUnloadClassesFrequency != 0);
 287 
 288   ReferenceProcessor* rp = _heap->ref_processor();
 289   // enable ("weak") refs discovery
 290   rp->enable_discovery(true /*verify_no_refs*/);
 291   rp->setup_policy(true); // snapshot the soft ref policy to be used in this cycle
 292   rp->set_active_mt_degree(_heap->workers()->active_workers());
 293 
 294   cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
 295   cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
 296   cm->shared_finish_mark_from_roots(/* full_gc = */ true);
 297 
 298   if (UseShenandoahMatrix && PrintShenandoahMatrix) {
 299     LogTarget(Info, gc) lt;
 300     LogStream ls(lt);
 301     _heap->connection_matrix()->print_on(&ls);
 302   }
 303 
 304   if (VerifyDuringGC) {
 305     HandleMark hm;  // handle scope
 306     _heap->prepare_for_verify();
 307     // Note: we can verify only the heap here. When an object is
 308     // marked, the previous value of the mark word (including
 309     // identity hash values, ages, etc) is preserved, and the mark
 310     // word is set to markOop::marked_value - effectively removing
 311     // any hash values from the mark word. These hash values are
 312     // used when verifying the dictionaries and so removing them
 313     // from the mark word can make verification of the dictionaries
 314     // fail. At the end of the GC, the original mark word values
 315     // (including hash values) are restored to the appropriate
 316     // objects.
 317     _heap->verify(VerifyOption_G1UseMarkWord);
 318   }
 319   _heap->set_bitmap_valid(true);
 320 }
 321 
 322 class ShenandoahMCReclaimHumongousRegionClosure : public ShenandoahHeapRegionClosure {
 323 private:
 324   ShenandoahHeap* _heap;
 325 public:
 326   ShenandoahMCReclaimHumongousRegionClosure() : _heap(ShenandoahHeap::heap()) {
 327   }
 328 
 329   bool heap_region_do(ShenandoahHeapRegion* r) {
 330     if (r->is_humongous_start()) {
 331       oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
 332       if (! _heap->is_marked(humongous_obj)) {
 333         _heap->trash_humongous_region_at(r);
 334       }
 335     }
 336     return false;
 337   }
 338 };
 339 
 340 
 341 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
 342 
 343 private:
 344 
 345   ShenandoahHeap* _heap;
 346   ShenandoahHeapRegionSet* _to_regions;
 347   ShenandoahHeapRegion* _to_region;
 348   ShenandoahHeapRegion* _from_region;
 349   HeapWord* _compact_point;
 350 
 351 public:
 352 
 353   ShenandoahPrepareForCompactionObjectClosure(ShenandoahHeapRegionSet* to_regions, ShenandoahHeapRegion* to_region) :
 354     _heap(ShenandoahHeap::heap()),
 355     _to_regions(to_regions),
 356     _to_region(to_region),
 357     _from_region(NULL),
 358     _compact_point(to_region->bottom()) {
 359   }
 360 
 361   void set_from_region(ShenandoahHeapRegion* from_region) {
 362     _from_region = from_region;
 363   }
 364 
 365   ShenandoahHeapRegion* to_region() const {
 366     return _to_region;
 367   }
 368   HeapWord* compact_point() const {
 369     return _compact_point;
 370   }
 371   void do_object(oop p) {
 372     assert(_from_region != NULL, "must set before work");
 373     assert(_heap->is_marked(p), "must be marked");
 374     assert(! _heap->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
 375     size_t obj_size = p->size() + BrooksPointer::word_size();
 376     if (_compact_point + obj_size > _to_region->end()) {
 377       // Object doesn't fit. Pick next to-region and start compacting there.
 378       _to_region->set_new_top(_compact_point);
 379       ShenandoahHeapRegion* new_to_region = _to_regions->current();
 380       _to_regions->next();
 381       if (new_to_region == NULL) {
 382         new_to_region = _from_region;
 383       }
 384       assert(new_to_region != _to_region, "must not reuse same to-region");
 385       assert(new_to_region != NULL, "must not be NULL");
 386       _to_region = new_to_region;
 387       _compact_point = _to_region->bottom();
 388     }
 389     assert(_compact_point + obj_size <= _to_region->end(), "must fit");
 390     assert(oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p)),
 391            "expect forwarded oop");
 392     BrooksPointer::set_raw(p, _compact_point + BrooksPointer::word_size());
 393     _compact_point += obj_size;
 394   }
 395 };
 396 
 397 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
 398 private:
 399 
 400   ShenandoahHeapRegionSet** _copy_queues;
 401   ShenandoahHeapRegionSet* _from_regions;
 402 
 403   ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* copy_queue) {
 404     ShenandoahHeapRegion* from_region = _from_regions->claim_next();
 405 
 406     // Note: During Full GC after cancelled conc GC, we might have incoming regions
 407     // in the collection set. Otherwise we would have just taken care of regular regions.
 408     while (from_region != NULL && !(from_region->is_regular() || from_region->is_cset())) {
 409       from_region = _from_regions->claim_next();
 410     }
 411     if (from_region != NULL) {
 412       assert(copy_queue != NULL, "sanity");
 413       assert(from_region->is_regular() || from_region->is_cset(), "only regular/cset regions in mark-compact");
 414       copy_queue->add_region(from_region);
 415     }
 416     return from_region;
 417   }
 418 
 419 public:
 420   ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet* from_regions, ShenandoahHeapRegionSet** copy_queues) :
 421     AbstractGangTask("Shenandoah Prepare For Compaction Task"),
 422     _from_regions(from_regions), _copy_queues(copy_queues) {
 423   }
 424 
 425   void work(uint worker_id) {
 426     ShenandoahHeap* heap = ShenandoahHeap::heap();
 427     ShenandoahHeapRegionSet* copy_queue = _copy_queues[worker_id];
 428     ShenandoahHeapRegion* from_region = next_from_region(copy_queue);
 429     if (from_region == NULL) return;
 430     ShenandoahHeapRegionSet* to_regions = new ShenandoahHeapRegionSet(ShenandoahHeap::heap()->num_regions());
 431     ShenandoahPrepareForCompactionObjectClosure cl(to_regions, from_region);
 432     while (from_region != NULL) {
 433       assert(from_region != NULL, "sanity");
 434       cl.set_from_region(from_region);
 435       heap->marked_object_iterate(from_region, &cl);
 436       if (from_region != cl.to_region()) {
 437         assert(from_region != NULL, "sanity");
 438         to_regions->add_region(from_region);
 439       }
 440       from_region = next_from_region(copy_queue);
 441     }
 442     assert(cl.to_region() != NULL, "should not happen");
 443     cl.to_region()->set_new_top(cl.compact_point());
 444     while (to_regions->count() > 0) {
 445       ShenandoahHeapRegion* r = to_regions->current();
 446       to_regions->next();
 447       assert(r != NULL, "should not happen");
 448       r->set_new_top(r->bottom());
 449     }
 450     delete to_regions;
 451   }
 452 };
 453 
 454 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** copy_queues) {
 455   GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
 456   ShenandoahHeap* heap = ShenandoahHeap::heap();
 457 
 458   {
 459     ShenandoahHeapLocker lock(heap->lock());
 460 
 461     ShenandoahMCReclaimHumongousRegionClosure cl;
 462     heap->heap_region_iterate(&cl);
 463 
 464     // After some humongous regions were reclaimed, we need to ensure their
 465     // backing storage is active. This is needed because we are potentially
 466     // sliding the data through them.
 467     ShenandoahEnsureHeapActiveClosure ecl;
 468     heap->heap_region_iterate(&ecl, false, false);
 469   }
 470 
 471   // Initialize copy queues.
 472   for (uint i = 0; i < heap->max_workers(); i++) {
 473     copy_queues[i] = new ShenandoahHeapRegionSet(heap->num_regions());
 474   }
 475 
 476   ShenandoahHeapRegionSet* from_regions = heap->regions();
 477   from_regions->clear_current_index();
 478   ShenandoahPrepareForCompactionTask prepare_task(from_regions, copy_queues);
 479   heap->workers()->run_task(&prepare_task);
 480 }
 481 
 482 class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure {
 483 private:
 484   ShenandoahHeap* _heap;
 485   size_t _new_obj_offset;
 486 public:
 487 
 488   ShenandoahAdjustPointersClosure() :
 489           _heap(ShenandoahHeap::heap()),
 490           _new_obj_offset(SIZE_MAX)  {
 491   }
 492 
 493 private:
 494   template <class T>
 495   inline void do_oop_work(T* p) {
 496     T o = oopDesc::load_heap_oop(p);
 497     if (! oopDesc::is_null(o)) {
 498       oop obj = oopDesc::decode_heap_oop_not_null(o);
 499       assert(_heap->is_marked(obj), "must be marked");
 500       oop forw = oop(BrooksPointer::get_raw(obj));
 501       oopDesc::encode_store_heap_oop(p, forw);
 502       if (UseShenandoahMatrix) {
 503         if (_heap->is_in_reserved(p)) {
 504           assert(_heap->is_in_reserved(forw), "must be in heap");
 505           assert (_new_obj_offset != SIZE_MAX, "should be set");
 506           // We're moving a to a', which points to b, about to be moved to b'.
 507           // We already know b' from the fwd pointer of b.
 508           // In the object closure, we see a, and we know a' (by looking at its
 509           // fwd ptr). We store the offset in the OopClosure, which is going
 510           // to visit all of a's fields, and then, when we see each field, we
 511           // subtract the offset from each field address to get the final ptr.
 512           _heap->connection_matrix()->set_connected(((HeapWord*) p) - _new_obj_offset, forw);
 513         }
 514       }
 515     }
 516   }
 517 public:
 518   void do_oop(oop* p) {
 519     do_oop_work(p);
 520   }
 521   void do_oop(narrowOop* p) {
 522     do_oop_work(p);
 523   }
 524   void set_new_obj_offset(size_t new_obj_offset) {
 525     _new_obj_offset = new_obj_offset;
 526   }
 527 };
 528 
 529 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
 530 private:
 531   ShenandoahAdjustPointersClosure _cl;
 532   ShenandoahHeap* _heap;
 533 public:
 534   ShenandoahAdjustPointersObjectClosure() :
 535     _heap(ShenandoahHeap::heap()) {
 536   }
 537   void do_object(oop p) {
 538     assert(_heap->is_marked(p), "must be marked");
 539     HeapWord* forw = BrooksPointer::get_raw(p);
 540     _cl.set_new_obj_offset(pointer_delta((HeapWord*) p, forw));
 541     p->oop_iterate(&_cl);
 542   }
 543 };
 544 
 545 class ShenandoahAdjustPointersTask : public AbstractGangTask {
 546 private:
 547   ShenandoahHeapRegionSet* _regions;
 548 public:
 549 
 550   ShenandoahAdjustPointersTask(ShenandoahHeapRegionSet* regions) :
 551     AbstractGangTask("Shenandoah Adjust Pointers Task"),
 552     _regions(regions) {
 553   }
 554 
 555   void work(uint worker_id) {
 556     ShenandoahHeap* heap = ShenandoahHeap::heap();
 557     ShenandoahHeapRegion* r = _regions->claim_next();
 558     ShenandoahAdjustPointersObjectClosure obj_cl;
 559     while (r != NULL) {
 560       if (! r->is_humongous_continuation()) {
 561         heap->marked_object_iterate(r, &obj_cl);
 562       }
 563       r = _regions->claim_next();
 564     }
 565   }
 566 };
 567 
 568 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
 569 private:
 570   ShenandoahRootProcessor* _rp;
 571 
 572 public:
 573 
 574   ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp) :
 575     AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
 576     _rp(rp) {
 577   }
 578 
 579   void work(uint worker_id) {
 580     ShenandoahAdjustPointersClosure cl;
 581     CLDToOopClosure adjust_cld_closure(&cl, true);
 582     MarkingCodeBlobClosure adjust_code_closure(&cl,
 583                                              CodeBlobToOopClosure::FixRelocations);
 584 
 585     _rp->process_all_roots(&cl, &cl,
 586                            &adjust_cld_closure,
 587                            &adjust_code_closure, worker_id);
 588   }
 589 };
 590 
 591 void ShenandoahMarkCompact::phase3_update_references() {
 592   GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
 593   ShenandoahHeap* heap = ShenandoahHeap::heap();
 594 
 595   if (UseShenandoahMatrix) {
 596     heap->connection_matrix()->clear_all();
 597   }
 598 
 599   WorkGang* workers = heap->workers();
 600   uint nworkers = workers->active_workers();
 601   {
 602 #if defined(COMPILER2) || INCLUDE_JVMCI
 603     DerivedPointerTable::clear();
 604 #endif
 605     ShenandoahRootProcessor rp(heap, nworkers, ShenandoahPhaseTimings::full_gc_roots);
 606     ShenandoahAdjustRootPointersTask task(&rp);
 607     workers->run_task(&task);
 608 #if defined(COMPILER2) || INCLUDE_JVMCI
 609     DerivedPointerTable::update_pointers();
 610 #endif
 611   }
 612 
 613   ShenandoahHeapRegionSet* regions = heap->regions();
 614   regions->clear_current_index();
 615   ShenandoahAdjustPointersTask adjust_pointers_task(regions);
 616   workers->run_task(&adjust_pointers_task);
 617 }
 618 
 619 class ShenandoahCompactObjectsClosure : public ObjectClosure {
 620 private:
 621   ShenandoahHeap* _heap;
 622   bool            _str_dedup;
 623   uint            _worker_id;
 624 public:
 625   ShenandoahCompactObjectsClosure(uint worker_id) : _heap(ShenandoahHeap::heap()),
 626     _str_dedup(ShenandoahStringDedup::is_enabled()), _worker_id(worker_id) {
 627   }
 628   void do_object(oop p) {
 629     assert(_heap->is_marked(p), "must be marked");
 630     size_t size = (size_t)p->size();
 631     HeapWord* compact_to = BrooksPointer::get_raw(p);
 632     HeapWord* compact_from = (HeapWord*) p;
 633     if (compact_from != compact_to) {
 634       Copy::aligned_conjoint_words(compact_from, compact_to, size);
 635     }
 636     oop new_obj = oop(compact_to);
 637     // new_obj->init_mark();
 638     BrooksPointer::initialize(new_obj);
 639 
 640     // String Dedup support
 641     if(_str_dedup && java_lang_String::is_instance_inlined(new_obj)) {
 642       new_obj->incr_age();
 643       if (ShenandoahStringDedup::is_candidate(new_obj)) {
 644         ShenandoahStringDedup::enqueue_from_safepoint(new_obj, _worker_id);
 645       }
 646     }
 647   }
 648 };
 649 
 650 class ShenandoahCompactObjectsTask : public AbstractGangTask {
 651   ShenandoahHeapRegionSet** _regions;
 652 public:
 653   ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** regions) :
 654     AbstractGangTask("Shenandoah Compact Objects Task"),
 655     _regions(regions) {
 656   }
 657   void work(uint worker_id) {
 658     ShenandoahHeap* heap = ShenandoahHeap::heap();
 659     ShenandoahHeapRegionSet* copy_queue = _regions[worker_id];
 660     copy_queue->clear_current_index();
 661     ShenandoahCompactObjectsClosure cl(worker_id);
 662     ShenandoahHeapRegion* r = copy_queue->current();
 663     copy_queue->next();
 664     while (r != NULL) {
 665       assert(! r->is_humongous(), "must not get humongous regions here");
 666       heap->marked_object_iterate(r, &cl);
 667       r->set_top(r->new_top());
 668       r = copy_queue->current();
 669       copy_queue->next();
 670     }
 671   }
 672 };
 673 
 674 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
 675   size_t _live;
 676   ShenandoahHeap* _heap;
 677 public:
 678 
 679   ShenandoahPostCompactClosure() : _live(0), _heap(ShenandoahHeap::heap()) {
 680     _heap->clear_free_regions();
 681   }
 682 
 683   bool heap_region_do(ShenandoahHeapRegion* r) {
 684     // Need to reset the complete-top-at-mark-start pointer here because
 685     // the complete marking bitmap is no longer valid. This ensures
 686     // size-based iteration in marked_object_iterate().
 687     _heap->set_top_at_mark_start(r->bottom(), r->bottom());
 688 
 689     size_t live = r->used();
 690 
 691     // Turn any lingering non-empty cset regions into regular regions.
 692     // This must be the leftover from the cancelled concurrent GC.
 693     if (r->is_cset() && live != 0) {
 694       r->make_regular_bypass();
 695     }
 696 
 697     // Reclaim regular/cset regions that became empty
 698     if ((r->is_regular() || r->is_cset()) && live == 0) {
 699       r->make_trash();
 700     }
 701 
 702     // Recycle all trash regions
 703     if (r->is_trash()) {
 704       live = 0;
 705       r->recycle();
 706     }
 707 
 708     // Finally, add all suitable regions into the free set
 709     if (r->is_alloc_allowed()) {
 710       if (_heap->collection_set()->is_in(r)) {
 711         _heap->collection_set()->remove_region(r);
 712       }
 713       _heap->add_free_region(r);
 714     }
 715 
 716     r->set_live_data(live);
 717     r->reset_alloc_stats_to_shared();
 718     _live += live;
 719     return false;
 720   }
 721 
 722   size_t get_live() { return _live; }
 723 
 724 };
 725 
 726 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** copy_queues) {
 727   GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
 728   ShenandoahHeap* heap = ShenandoahHeap::heap();
 729   ShenandoahCompactObjectsTask compact_task(copy_queues);
 730   heap->workers()->run_task(&compact_task);
 731 
 732   // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
 733   // and must ensure the bitmap is in sync.
 734   heap->reset_mark_bitmap(heap->workers());
 735 
 736   {
 737     ShenandoahHeapLocker lock(heap->lock());
 738     ShenandoahPostCompactClosure post_compact;
 739     heap->heap_region_iterate(&post_compact);
 740 
 741     heap->set_used(post_compact.get_live());
 742   }
 743 
 744   heap->collection_set()->clear();
 745   heap->clear_cancelled_concgc();
 746 
 747   for (uint i = 0; i < heap->max_workers(); i++) {
 748     delete copy_queues[i];
 749   }
 750 
 751 }