1 /*
2 * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "code/codeCache.hpp"
25 #include "gc/shared/gcTraceTime.inline.hpp"
26 #include "gc/shared/isGCActiveMark.hpp"
27 #include "gc/shenandoah/brooksPointer.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
30 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
36 #include "gc/shenandoah/vm_operations_shenandoah.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/thread.hpp"
40 #include "utilities/copy.hpp"
41 #include "gc/shared/taskqueue.inline.hpp"
42 #include "gc/shared/workgroup.hpp"
43
44 class ShenandoahMarkCompactBarrierSet : public ShenandoahBarrierSet {
45 public:
46 ShenandoahMarkCompactBarrierSet(ShenandoahHeap* heap) : ShenandoahBarrierSet(heap) {
47 }
48 oop read_barrier(oop src) {
49 return src;
50 }
51 #ifdef ASSERT
52 bool is_safe(oop o) {
53 if (o == NULL) return true;
54 if (! oopDesc::unsafe_equals(o, read_barrier(o))) {
55 return false;
56 }
57 return true;
58 }
59 bool is_safe(narrowOop o) {
60 oop obj = oopDesc::decode_heap_oop(o);
61 return is_safe(obj);
62 }
63 #endif
64 };
65
66 class ClearInCollectionSetHeapRegionClosure: public ShenandoahHeapRegionClosure {
67 private:
68 ShenandoahHeap* _heap;
69 public:
70
71 ClearInCollectionSetHeapRegionClosure() : _heap(ShenandoahHeap::heap()) {
72 }
73
74 bool doHeapRegion(ShenandoahHeapRegion* r) {
75 _heap->set_next_top_at_mark_start(r->bottom(), r->top());
76 r->clear_live_data();
77 r->set_concurrent_iteration_safe_limit(r->top());
78 return false;
79 }
80 };
81
82 STWGCTimer* ShenandoahMarkCompact::_gc_timer = NULL;
83
84 void ShenandoahMarkCompact::initialize() {
85 _gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer();
86 }
87
88 void ShenandoahMarkCompact::do_mark_compact(GCCause::Cause gc_cause) {
89
90 ShenandoahHeap* _heap = ShenandoahHeap::heap();
91 ShenandoahCollectorPolicy* policy = _heap->shenandoahPolicy();
92
93 _gc_timer->register_gc_start();
94
95 _heap->set_full_gc_in_progress(true);
96
97 assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
98 IsGCActiveMark is_active;
99
100 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
101
102 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc);
103
104 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_heapdumps);
105 _heap->pre_full_gc_dump(_gc_timer);
106 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_heapdumps);
107
108 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_prepare);
109
110 // Full GC is supposed to recover from any GC state:
111
112 // a. Cancel concurrent mark, if in progress
113 if (_heap->concurrent_mark_in_progress()) {
114 _heap->concurrentMark()->cancel();
115 _heap->stop_concurrent_marking();
116 }
117 assert(!_heap->concurrent_mark_in_progress(), "sanity");
118
119 // b. Cancel evacuation, if in progress
120 if (_heap->is_evacuation_in_progress()) {
121 _heap->set_evacuation_in_progress_at_safepoint(false);
122 }
123 assert(!_heap->is_evacuation_in_progress(), "sanity");
124
125 // c. Reset the bitmaps for new marking
126 _heap->reset_next_mark_bitmap(_heap->workers());
127 assert(_heap->is_next_bitmap_clear(), "sanity");
128
129 ClearInCollectionSetHeapRegionClosure cl;
130 _heap->heap_region_iterate(&cl, false, false);
131
132 /*
133 if (ShenandoahVerify) {
134 // Full GC should only be called between regular concurrent cycles, therefore
135 // those verifications should be valid.
136 _heap->verify_heap_after_evacuation();
137 _heap->verify_heap_after_update_refs();
138 }
139 */
140
141 BarrierSet* old_bs = oopDesc::bs();
142 ShenandoahMarkCompactBarrierSet bs(_heap);
143 oopDesc::set_bs(&bs);
144
145 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_prepare);
146
147 {
148 GCTraceTime(Info, gc) time("Pause Full", _gc_timer, gc_cause, true);
149
150 if (UseTLAB) {
151 _heap->ensure_parsability(true);
152 }
153
154 CodeCache::gc_prologue();
155
156 // We should save the marks of the currently locked biased monitors.
157 // The marking doesn't preserve the marks of biased objects.
158 //BiasedLocking::preserve_marks();
159
160 _heap->set_need_update_refs(true);
161 WorkGang* workers = _heap->workers();
162
163 // Setup workers for phase 1
164 {
165 uint nworkers = ShenandoahCollectorPolicy::calc_workers_for_init_marking(
166 workers->active_workers(), Threads::number_of_non_daemon_threads());
167 workers->update_active_workers(nworkers);
168 ShenandoahWorkerScope scope(workers, nworkers);
169
170 OrderAccess::fence();
171
172 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_mark);
173 phase1_mark_heap();
174 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_mark);
175 }
176
177 // Setup workers for the rest
178 {
179 uint nworkers = ShenandoahCollectorPolicy::calc_workers_for_parallel_evacuation(
180 workers->active_workers(), Threads::number_of_non_daemon_threads());
181
182 ShenandoahWorkerScope scope(workers, nworkers);
183
184 OrderAccess::fence();
185
186 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_calculate_addresses);
187 ShenandoahHeapRegionSet* copy_queues[_heap->max_workers()];
188 phase2_calculate_target_addresses(copy_queues);
189 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_calculate_addresses);
190
191 OrderAccess::fence();
192
193 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_adjust_pointers);
194 phase3_update_references();
195 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_adjust_pointers);
196
197 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_copy_objects);
198 phase4_compact_objects(copy_queues);
199 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_copy_objects);
200
201 CodeCache::gc_epilogue();
202 JvmtiExport::gc_epilogue();
203 }
204
205 // refs processing: clean slate
206 // rp.enqueue_discovered_references();
207
208 if (ShenandoahVerify) {
209 _heap->verify_heap_after_evacuation();
210 }
211
212 _heap->set_bytes_allocated_since_cm(0);
213
214 _heap->set_need_update_refs(false);
215
216 _heap->set_full_gc_in_progress(false);
217 }
218
219 _gc_timer->register_gc_end();
220
221 policy->record_full_gc();
222
223 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_heapdumps);
224 _heap->post_full_gc_dump(_gc_timer);
225 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_heapdumps);
226
227 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc);
228
229 oopDesc::set_bs(old_bs);
230
231 if (UseShenandoahMatrix) {
232 if (PrintShenandoahMatrix) {
233 outputStream* log = Log(gc)::info_stream();
234 _heap->connection_matrix()->print_on(log);
235 }
236 }
237 }
238
239 #ifdef ASSERT
240 class VerifyNotForwardedPointersClosure : public MetadataAwareOopClosure {
241 private:
242 template <class T>
243 inline void do_oop_work(T* p) {
244 T o = oopDesc::load_heap_oop(p);
245 if (! oopDesc::is_null(o)) {
246 oop obj = oopDesc::decode_heap_oop_not_null(o);
247 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
248 "expect forwarded oop");
249 ShenandoahHeap* heap = ShenandoahHeap::heap();
250 if (! heap->is_marked_complete(obj)) {
251 tty->print_cr("ref region humongous? %s", BOOL_TO_STR(heap->heap_region_containing(p)->is_humongous()));
252 }
253 assert(heap->is_marked_complete(obj), "must be marked");
254 assert(! heap->allocated_after_complete_mark_start((HeapWord*) obj), "must be truly marked");
255 }
256 }
257 public:
258 void do_oop(oop* p) {
259 do_oop_work(p);
260 }
261 void do_oop(narrowOop* p) {
262 do_oop_work(p);
263 }
264 };
265
266 class ShenandoahMCVerifyAfterMarkingObjectClosure : public ObjectClosure {
267 public:
268 void do_object(oop p) {
269 ShenandoahHeap* heap = ShenandoahHeap::heap();
270 assert(oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p)),
271 "expect forwarded oop");
272 assert(heap->is_marked_complete(p), "must be marked");
273 assert(! heap->allocated_after_complete_mark_start((HeapWord*) p), "must be truly marked");
274 VerifyNotForwardedPointersClosure cl;
275 p->oop_iterate(&cl);
276 }
277 };
278
279 class ShenandoahMCVerifyAfterMarkingRegionClosure : public ShenandoahHeapRegionClosure {
280 bool doHeapRegion(ShenandoahHeapRegion* r) {
281 ShenandoahMCVerifyAfterMarkingObjectClosure cl;
282 if (! r->is_humongous_continuation()) {
283 ShenandoahHeap::heap()->marked_object_iterate(r, &cl);
284 }
285 return false;
286 }
287 };
288
289 #endif
290
291 void ShenandoahMarkCompact::phase1_mark_heap() {
292 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
293 ShenandoahHeap* _heap = ShenandoahHeap::heap();
294
295 ShenandoahConcurrentMark* cm = _heap->concurrentMark();
296
297 cm->set_process_references(true);
298 cm->set_unload_classes(true);
299
300 ReferenceProcessor* rp = _heap->ref_processor();
301 // enable ("weak") refs discovery
302 rp->enable_discovery(true /*verify_no_refs*/);
303 rp->setup_policy(true); // snapshot the soft ref policy to be used in this cycle
304 rp->set_active_mt_degree(_heap->workers()->active_workers());
305
306 COMPILER2_PRESENT(DerivedPointerTable::clear());
307 cm->update_roots();
308 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
309
310 cm->mark_roots();
311 cm->shared_finish_mark_from_roots(/* full_gc = */ true);
312
313 _heap->swap_mark_bitmaps();
314
315 if (UseShenandoahMatrix) {
316 if (PrintShenandoahMatrix) {
317 outputStream* log = Log(gc)::info_stream();
318 _heap->connection_matrix()->print_on(log);
319 }
320 if (VerifyShenandoahMatrix) {
321 _heap->verify_matrix();
322 }
323 }
324
325 if (VerifyDuringGC) {
326 HandleMark hm; // handle scope
327 // Universe::heap()->prepare_for_verify();
328 _heap->prepare_for_verify();
329 // Note: we can verify only the heap here. When an object is
330 // marked, the previous value of the mark word (including
331 // identity hash values, ages, etc) is preserved, and the mark
332 // word is set to markOop::marked_value - effectively removing
333 // any hash values from the mark word. These hash values are
334 // used when verifying the dictionaries and so removing them
335 // from the mark word can make verification of the dictionaries
336 // fail. At the end of the GC, the original mark word values
337 // (including hash values) are restored to the appropriate
338 // objects.
339 // Universe::heap()->verify(VerifySilently, VerifyOption_G1UseMarkWord);
340 _heap->verify(VerifyOption_G1UseMarkWord);
341 }
342
343 #ifdef ASSERT
344 ShenandoahMCVerifyAfterMarkingRegionClosure cl;
345 _heap->heap_region_iterate(&cl);
346 #endif
347 }
348
349 class ShenandoahMCReclaimHumongousRegionClosure : public ShenandoahHeapRegionClosure {
350 private:
351 ShenandoahHeap* _heap;
352 public:
353 ShenandoahMCReclaimHumongousRegionClosure() : _heap(ShenandoahHeap::heap()) {
354 }
355
356 bool doHeapRegion(ShenandoahHeapRegion* r) {
357 if (r->is_humongous_start()) {
358 oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
359 if (! _heap->is_marked_complete(humongous_obj)) {
360 _heap->reclaim_humongous_region_at(r);
361 }
362 }
363 return false;
364 }
365 };
366
367
368 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
369
370 private:
371
372 ShenandoahHeap* _heap;
373 ShenandoahHeapRegionSet* _to_regions;
374 ShenandoahHeapRegion* _to_region;
375 ShenandoahHeapRegion* _from_region;
376 HeapWord* _compact_point;
377
378 public:
379
380 ShenandoahPrepareForCompactionObjectClosure(ShenandoahHeapRegionSet* to_regions, ShenandoahHeapRegion* to_region) :
381 _heap(ShenandoahHeap::heap()),
382 _to_regions(to_regions),
383 _to_region(to_region),
384 _from_region(NULL),
385 _compact_point(to_region->bottom()) {
386 }
387
388 void set_from_region(ShenandoahHeapRegion* from_region) {
389 _from_region = from_region;
390 }
391
392 ShenandoahHeapRegion* to_region() const {
393 return _to_region;
394 }
395 HeapWord* compact_point() const {
396 return _compact_point;
397 }
398 void do_object(oop p) {
399 assert(_from_region != NULL, "must set before work");
400 assert(_heap->is_marked_complete(p), "must be marked");
401 assert(! _heap->allocated_after_complete_mark_start((HeapWord*) p), "must be truly marked");
402 size_t size = p->size();
403 size_t obj_size = size + BrooksPointer::word_size();
404 if (_compact_point + obj_size > _to_region->end()) {
405 // Object doesn't fit. Pick next to-region and start compacting there.
406 _to_region->set_new_top(_compact_point);
407 ShenandoahHeapRegion* new_to_region = _to_regions->current();
408 _to_regions->next();
409 if (new_to_region == NULL) {
410 new_to_region = _from_region;
411 }
412 assert(new_to_region != _to_region, "must not reuse same to-region");
413 assert(new_to_region != NULL, "must not be NULL");
414 _to_region = new_to_region;
415 _compact_point = _to_region->bottom();
416 }
417 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
418 assert(oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p)),
419 "expect forwarded oop");
420 BrooksPointer::set_raw(p, _compact_point + BrooksPointer::word_size());
421 _compact_point += obj_size;
422 }
423 };
424
425 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
426 private:
427
428 ShenandoahHeapRegionSet** _copy_queues;
429 ShenandoahHeapRegionSet* _from_regions;
430
431 ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* copy_queue) {
432 ShenandoahHeapRegion* from_region = _from_regions->claim_next();
433 while (from_region != NULL && (from_region->is_humongous() || from_region->is_pinned())) {
434 from_region = _from_regions->claim_next();
435 }
436 if (from_region != NULL) {
437 assert(copy_queue != NULL, "sanity");
438 assert(! from_region->is_humongous(), "must not get humongous regions here");
439 assert(! from_region->is_pinned(), "no pinned region in mark-compact");
440 copy_queue->add_region(from_region);
441 }
442 return from_region;
443 }
444
445 public:
446 ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet* from_regions, ShenandoahHeapRegionSet** copy_queues) :
447 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
448 _from_regions(from_regions), _copy_queues(copy_queues) {
449 }
450
451 void work(uint worker_id) {
452 ShenandoahHeap* heap = ShenandoahHeap::heap();
453 ShenandoahHeapRegionSet* copy_queue = _copy_queues[worker_id];
454 ShenandoahHeapRegion* from_region = next_from_region(copy_queue);
455 if (from_region == NULL) return;
456 ShenandoahHeapRegionSet* to_regions = new ShenandoahHeapRegionSet(ShenandoahHeap::heap()->max_regions());
457 ShenandoahPrepareForCompactionObjectClosure cl(to_regions, from_region);
458 while (from_region != NULL) {
459 assert(from_region != NULL, "sanity");
460 cl.set_from_region(from_region);
461 heap->marked_object_iterate(from_region, &cl);
462 if (from_region != cl.to_region()) {
463 assert(from_region != NULL, "sanity");
464 to_regions->add_region(from_region);
465 }
466 from_region = next_from_region(copy_queue);
467 }
468 assert(cl.to_region() != NULL, "should not happen");
469 cl.to_region()->set_new_top(cl.compact_point());
470 while (to_regions->count() > 0) {
471 ShenandoahHeapRegion* r = to_regions->current();
472 to_regions->next();
473 if (r == NULL) {
474 to_regions->print();
475 }
476 assert(r != NULL, "should not happen");
477 r->set_new_top(r->bottom());
478 }
479 delete to_regions;
480 }
481 };
482
483 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** copy_queues) {
484 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
485 ShenandoahHeap* heap = ShenandoahHeap::heap();
486
487 ShenandoahMCReclaimHumongousRegionClosure cl;
488 heap->heap_region_iterate(&cl);
489
490 // Initialize copy queues.
491 for (uint i = 0; i < heap->max_workers(); i++) {
492 copy_queues[i] = new ShenandoahHeapRegionSet(heap->max_regions());
493 }
494
495 ShenandoahHeapRegionSet* from_regions = heap->regions();
496 from_regions->clear_current_index();
497 ShenandoahPrepareForCompactionTask prepare_task(from_regions, copy_queues);
498 heap->workers()->run_task(&prepare_task);
499 }
500
501 class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure {
502 private:
503 ShenandoahHeap* _heap;
504 uint _from_idx;
505 public:
506
507 ShenandoahAdjustPointersClosure() : _heap(ShenandoahHeap::heap()) {
508 }
509
510 private:
511 template <class T>
512 inline void do_oop_work(T* p) {
513 T o = oopDesc::load_heap_oop(p);
514 if (! oopDesc::is_null(o)) {
515 oop obj = oopDesc::decode_heap_oop_not_null(o);
516 assert(_heap->is_marked_complete(obj), "must be marked");
517 oop forw = oop(BrooksPointer::get_raw(obj));
518 oopDesc::encode_store_heap_oop(p, forw);
519 if (UseShenandoahMatrix) {
520 if (_heap->is_in_reserved(p)) {
521 assert(_heap->is_in_reserved(forw), "must be in heap");
522 uint to_idx = _heap->heap_region_index_containing(forw);
523 _heap->connection_matrix()->set_connected(_from_idx, to_idx, true);
524 }
525 }
526 }
527 }
528 public:
529 void do_oop(oop* p) {
530 do_oop_work(p);
531 }
532 void do_oop(narrowOop* p) {
533 do_oop_work(p);
534 }
535 void set_from_idx(uint from_idx) {
536 _from_idx = from_idx;
537 }
538 };
539
540 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
541 private:
542 ShenandoahAdjustPointersClosure _cl;
543 ShenandoahHeap* _heap;
544 public:
545 ShenandoahAdjustPointersObjectClosure() :
546 _heap(ShenandoahHeap::heap()) {
547 }
548 void do_object(oop p) {
549 assert(_heap->is_marked_complete(p), "must be marked");
550 oop forw = oop(BrooksPointer::get_raw(p));
551 uint from_idx = _heap->heap_region_index_containing(forw);
552 _cl.set_from_idx(from_idx);
553 p->oop_iterate(&_cl);
554 }
555 };
556
557 class ShenandoahAdjustPointersTask : public AbstractGangTask {
558 private:
559 ShenandoahHeapRegionSet* _regions;
560 public:
561
562 ShenandoahAdjustPointersTask(ShenandoahHeapRegionSet* regions) :
563 AbstractGangTask("Shenandoah Adjust Pointers Task"),
564 _regions(regions) {
565 }
566
567 void work(uint worker_id) {
568 ShenandoahHeap* heap = ShenandoahHeap::heap();
569 ShenandoahHeapRegion* r = _regions->claim_next();
570 ShenandoahAdjustPointersObjectClosure obj_cl;
571 while (r != NULL) {
572 if (! r->is_humongous_continuation()) {
573 heap->marked_object_iterate(r, &obj_cl);
574 }
575 r = _regions->claim_next();
576 }
577 }
578 };
579
580 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
581 private:
582 ShenandoahRootProcessor* _rp;
583
584 public:
585
586 ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp) :
587 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
588 _rp(rp) {
589 }
590
591 void work(uint worker_id) {
592 ShenandoahAdjustPointersClosure cl;
593 CLDToOopClosure adjust_cld_closure(&cl, true);
594 MarkingCodeBlobClosure adjust_code_closure(&cl,
595 CodeBlobToOopClosure::FixRelocations);
596
597 _rp->process_all_roots(&cl, &cl,
598 &adjust_cld_closure,
599 &adjust_code_closure, worker_id);
600 }
601 };
602
603 void ShenandoahMarkCompact::phase3_update_references() {
604 GCTraceTime(Info, gc, phases) time("Phase 2: Adjust pointers", _gc_timer);
605 ShenandoahHeap* heap = ShenandoahHeap::heap();
606
607 if (UseShenandoahMatrix) {
608 heap->connection_matrix()->clear_all();
609 }
610
611 // Need cleared claim bits for the roots processing
612 ClassLoaderDataGraph::clear_claimed_marks();
613
614 WorkGang* workers = heap->workers();
615 uint nworkers = workers->active_workers();
616 {
617 COMPILER2_PRESENT(DerivedPointerTable::clear());
618
619 ShenandoahRootProcessor rp(heap, nworkers);
620 ShenandoahAdjustRootPointersTask task(&rp);
621 workers->run_task(&task);
622 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
623 }
624
625 ShenandoahHeapRegionSet* regions = heap->regions();
626 regions->clear_current_index();
627 ShenandoahAdjustPointersTask adjust_pointers_task(regions);
628 workers->run_task(&adjust_pointers_task);
629 }
630
631 class ShenandoahCompactObjectsClosure : public ObjectClosure {
632 private:
633 ShenandoahHeap* _heap;
634 public:
635 ShenandoahCompactObjectsClosure() : _heap(ShenandoahHeap::heap()) {
636 }
637 void do_object(oop p) {
638 assert(_heap->is_marked_complete(p), "must be marked");
639 size_t size = p->size();
640 HeapWord* compact_to = BrooksPointer::get_raw(p);
641 HeapWord* compact_from = (HeapWord*) p;
642 if (compact_from != compact_to) {
643 Copy::aligned_conjoint_words(compact_from, compact_to, size);
644 }
645 oop new_obj = oop(compact_to);
646 // new_obj->init_mark();
647 BrooksPointer::initialize(new_obj);
648 }
649 };
650
651 class ShenandoahCompactObjectsTask : public AbstractGangTask {
652 ShenandoahHeapRegionSet** _regions;
653 public:
654 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** regions) :
655 AbstractGangTask("Shenandoah Compact Objects Task"),
656 _regions(regions) {
657 }
658 void work(uint worker_id) {
659 ShenandoahHeap* heap = ShenandoahHeap::heap();
660 ShenandoahHeapRegionSet* copy_queue = _regions[worker_id];
661 copy_queue->clear_current_index();
662 ShenandoahCompactObjectsClosure cl;
663 ShenandoahHeapRegion* r = copy_queue->current();
664 copy_queue->next();
665 while (r != NULL) {
666 assert(! r->is_humongous(), "must not get humongous regions here");
667 heap->marked_object_iterate(r, &cl);
668 r->set_top(r->new_top());
669 r = copy_queue->current();
670 copy_queue->next();
671 }
672 }
673 };
674
675 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
676 size_t _live;
677 ShenandoahHeap* _heap;
678 public:
679
680 ShenandoahPostCompactClosure() : _live(0), _heap(ShenandoahHeap::heap()) {
681 _heap->clear_free_regions();
682 }
683
684 bool doHeapRegion(ShenandoahHeapRegion* r) {
685 // Need to reset the complete-top-at-mark-start pointer here because
686 // the complete marking bitmap is no longer valid. This ensures
687 // size-based iteration in marked_object_iterate().
688 _heap->set_complete_top_at_mark_start(r->bottom(), r->bottom());
689 r->set_in_collection_set(false);
690 if (r->is_humongous()) {
691 _live += ShenandoahHeapRegion::RegionSizeBytes;
692 } else {
693 size_t live = r->used();
694 if (live == 0) {
695 r->recycle();
696 _heap->add_free_region(r);
697 }
698 r->set_live_data(live);
699 _live += live;
700 }
701 return false;
702 }
703
704 size_t get_live() { return _live; }
705
706 };
707
708 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** copy_queues) {
709 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
710 ShenandoahHeap* heap = ShenandoahHeap::heap();
711 ShenandoahCompactObjectsTask compact_task(copy_queues);
712 heap->workers()->run_task(&compact_task);
713
714 heap->clear_cset_fast_test();
715
716 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
717 // and must ensure the bitmap is in sync.
718 heap->reset_complete_mark_bitmap(heap->workers());
719
720 {
721 ShenandoahHeap::ShenandoahHeapLock lock(heap);
722 ShenandoahPostCompactClosure post_compact;
723 heap->heap_region_iterate(&post_compact);
724
725 heap->set_used(post_compact.get_live());
726
727 }
728
729 heap->clear_cancelled_concgc();
730
731 // Also clear the next bitmap in preparation for next marking.
732 heap->reset_next_mark_bitmap(heap->workers());
733
734 for (uint i = 0; i < heap->max_workers(); i++) {
735 delete copy_queues[i];
736 }
737
738 }