1 /*
2 * Copyright (c) 2014, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "code/codeCache.hpp"
25 #include "gc/shared/gcTraceTime.inline.hpp"
26 #include "gc/shared/isGCActiveMark.hpp"
27 #include "gc/shenandoah/brooksPointer.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
30 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
33 #include "gc/shenandoah/shenandoahHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
36 #include "gc/shenandoah/vm_operations_shenandoah.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "runtime/biasedLocking.hpp"
39 #include "runtime/thread.hpp"
40 #include "utilities/copy.hpp"
41 #include "gc/shared/taskqueue.inline.hpp"
42 #include "gc/shared/workgroup.hpp"
43
44 class ShenandoahMarkCompactBarrierSet : public ShenandoahBarrierSet {
45 public:
46 ShenandoahMarkCompactBarrierSet(ShenandoahHeap* heap) : ShenandoahBarrierSet(heap) {
47 }
48 oop read_barrier(oop src) {
49 return src;
50 }
51 #ifdef ASSERT
52 bool is_safe(oop o) {
53 if (o == NULL) return true;
54 if (! oopDesc::unsafe_equals(o, read_barrier(o))) {
55 return false;
56 }
57 return true;
58 }
59 bool is_safe(narrowOop o) {
60 oop obj = oopDesc::decode_heap_oop(o);
61 return is_safe(obj);
62 }
63 #endif
64 };
65
66 class ClearInCollectionSetHeapRegionClosure: public ShenandoahHeapRegionClosure {
67 private:
68 ShenandoahHeap* _heap;
69 public:
70
71 ClearInCollectionSetHeapRegionClosure() : _heap(ShenandoahHeap::heap()) {
72 }
73
74 bool doHeapRegion(ShenandoahHeapRegion* r) {
75 _heap->set_next_top_at_mark_start(r->bottom(), r->top());
76 r->clear_live_data();
77 r->set_concurrent_iteration_safe_limit(r->top());
78 return false;
79 }
80 };
81
82 STWGCTimer* ShenandoahMarkCompact::_gc_timer = NULL;
83
84 void ShenandoahMarkCompact::initialize() {
85 _gc_timer = new (ResourceObj::C_HEAP, mtGC) STWGCTimer();
86 }
87
88 void ShenandoahMarkCompact::do_mark_compact(GCCause::Cause gc_cause) {
89
90 ShenandoahHeap* _heap = ShenandoahHeap::heap();
91 ShenandoahCollectorPolicy* policy = _heap->shenandoahPolicy();
92
93 _gc_timer->register_gc_start();
94
95 _heap->set_full_gc_in_progress(true);
96
97 assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
98 IsGCActiveMark is_active;
99
100 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
101
102 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc);
103
104 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_heapdumps);
105 _heap->pre_full_gc_dump(_gc_timer);
106 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_heapdumps);
107
108 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_prepare);
109
110 // Full GC is supposed to recover from any GC state:
111
112 // a. Cancel concurrent mark, if in progress
113 if (_heap->concurrent_mark_in_progress()) {
114 _heap->concurrentMark()->cancel();
115 _heap->stop_concurrent_marking();
116 }
117 assert(!_heap->concurrent_mark_in_progress(), "sanity");
118
119 // b. Cancel evacuation, if in progress
120 if (_heap->is_evacuation_in_progress()) {
121 _heap->set_evacuation_in_progress_at_safepoint(false);
122 }
123 assert(!_heap->is_evacuation_in_progress(), "sanity");
124
125 // c. Reset the bitmaps for new marking
126 _heap->reset_next_mark_bitmap(_heap->workers());
127 assert(_heap->is_next_bitmap_clear(), "sanity");
128
129 ClearInCollectionSetHeapRegionClosure cl;
130 _heap->heap_region_iterate(&cl, false, false);
131
132 /*
133 if (ShenandoahVerify) {
134 // Full GC should only be called between regular concurrent cycles, therefore
135 // those verifications should be valid.
136 _heap->verify_heap_after_evacuation();
137 _heap->verify_heap_after_update_refs();
138 }
139 */
140
141 BarrierSet* old_bs = oopDesc::bs();
142 ShenandoahMarkCompactBarrierSet bs(_heap);
143 oopDesc::set_bs(&bs);
144
145 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_prepare);
146
147 {
148 GCTraceTime(Info, gc) time("Pause Full", _gc_timer, gc_cause, true);
149
150 if (UseTLAB) {
151 _heap->ensure_parsability(true);
152 }
153
154 CodeCache::gc_prologue();
155
156 // We should save the marks of the currently locked biased monitors.
157 // The marking doesn't preserve the marks of biased objects.
158 //BiasedLocking::preserve_marks();
159
160 _heap->set_need_update_refs(true);
161 WorkGang* workers = _heap->workers();
162
163 // Setup workers for phase 1
164 {
165 uint nworkers = ShenandoahCollectorPolicy::calc_workers_for_init_marking(
166 workers->active_workers(), Threads::number_of_non_daemon_threads());
167 workers->update_active_workers(nworkers);
168 ShenandoahWorkerScope scope(workers, nworkers);
169
170 OrderAccess::fence();
171
172 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_mark);
173 phase1_mark_heap();
174 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_mark);
175 }
176
177 // Setup workers for the rest
178 {
179 uint nworkers = ShenandoahCollectorPolicy::calc_workers_for_parallel_evacuation(
180 workers->active_workers(), Threads::number_of_non_daemon_threads());
181
182 ShenandoahWorkerScope scope(workers, nworkers);
183
184 OrderAccess::fence();
185
186 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_calculate_addresses);
187 ShenandoahHeapRegionSet* copy_queues[_heap->max_workers()];
188 phase2_calculate_target_addresses(copy_queues);
189 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_calculate_addresses);
190
191 OrderAccess::fence();
192
193 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_adjust_pointers);
194 phase3_update_references();
195 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_adjust_pointers);
196
197 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_copy_objects);
198 phase4_compact_objects(copy_queues);
199 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_copy_objects);
200
201 CodeCache::gc_epilogue();
202 JvmtiExport::gc_epilogue();
203 }
204
205 // refs processing: clean slate
206 // rp.enqueue_discovered_references();
207
208 if (ShenandoahVerify) {
209 _heap->verify_heap_after_evacuation();
210 }
211
212 _heap->set_bytes_allocated_since_cm(0);
213
214 _heap->set_need_update_refs(false);
215
216 _heap->set_full_gc_in_progress(false);
217 }
218
219 _gc_timer->register_gc_end();
220
221 policy->record_full_gc();
222
223 policy->record_phase_start(ShenandoahCollectorPolicy::full_gc_heapdumps);
224 _heap->post_full_gc_dump(_gc_timer);
225 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc_heapdumps);
226
227 policy->record_phase_end(ShenandoahCollectorPolicy::full_gc);
228
229 oopDesc::set_bs(old_bs);
230 }
231
232 #ifdef ASSERT
233 class VerifyNotForwardedPointersClosure : public MetadataAwareOopClosure {
234 private:
235 template <class T>
236 inline void do_oop_work(T* p) {
237 T o = oopDesc::load_heap_oop(p);
238 if (! oopDesc::is_null(o)) {
239 oop obj = oopDesc::decode_heap_oop_not_null(o);
240 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
241 "expect forwarded oop");
242 ShenandoahHeap* heap = ShenandoahHeap::heap();
243 if (! heap->is_marked_complete(obj)) {
244 tty->print_cr("ref region humongous? %s", BOOL_TO_STR(heap->heap_region_containing(p)->is_humongous()));
245 }
246 assert(heap->is_marked_complete(obj), "must be marked");
247 assert(! heap->allocated_after_complete_mark_start((HeapWord*) obj), "must be truly marked");
248 }
249 }
250 public:
251 void do_oop(oop* p) {
252 do_oop_work(p);
253 }
254 void do_oop(narrowOop* p) {
255 do_oop_work(p);
256 }
257 };
258
259 class ShenandoahMCVerifyAfterMarkingObjectClosure : public ObjectClosure {
260 public:
261 void do_object(oop p) {
262 ShenandoahHeap* heap = ShenandoahHeap::heap();
263 assert(oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p)),
264 "expect forwarded oop");
265 assert(heap->is_marked_complete(p), "must be marked");
266 assert(! heap->allocated_after_complete_mark_start((HeapWord*) p), "must be truly marked");
267 VerifyNotForwardedPointersClosure cl;
268 p->oop_iterate(&cl);
269 }
270 };
271
272 class ShenandoahMCVerifyAfterMarkingRegionClosure : public ShenandoahHeapRegionClosure {
273 bool doHeapRegion(ShenandoahHeapRegion* r) {
274 ShenandoahMCVerifyAfterMarkingObjectClosure cl;
275 if (! r->is_humongous_continuation()) {
276 ShenandoahHeap::heap()->marked_object_iterate(r, &cl);
277 }
278 return false;
279 }
280 };
281
282 #endif
283
284 void ShenandoahMarkCompact::phase1_mark_heap() {
285 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
286 ShenandoahHeap* _heap = ShenandoahHeap::heap();
287
288 ShenandoahConcurrentMark* cm = _heap->concurrentMark();
289
290 cm->set_process_references(true);
291 cm->set_unload_classes(true);
292
293 ReferenceProcessor* rp = _heap->ref_processor();
294 // enable ("weak") refs discovery
295 rp->enable_discovery(true /*verify_no_refs*/);
296 rp->setup_policy(true); // snapshot the soft ref policy to be used in this cycle
297 rp->set_active_mt_degree(_heap->workers()->active_workers());
298
299 COMPILER2_PRESENT(DerivedPointerTable::clear());
300 cm->update_roots();
301 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
302
303 cm->mark_roots();
304 cm->shared_finish_mark_from_roots(/* full_gc = */ true);
305
306 _heap->swap_mark_bitmaps();
307
308 if (UseShenandoahMatrix) {
309 if (PrintShenandoahMatrix) {
310 _heap->connection_matrix()->print_on(tty);
311 }
312 if (VerifyShenandoahMatrix) {
313 _heap->verify_matrix();
314 }
315 }
316
317 if (VerifyDuringGC) {
318 HandleMark hm; // handle scope
319 // Universe::heap()->prepare_for_verify();
320 _heap->prepare_for_verify();
321 // Note: we can verify only the heap here. When an object is
322 // marked, the previous value of the mark word (including
323 // identity hash values, ages, etc) is preserved, and the mark
324 // word is set to markOop::marked_value - effectively removing
325 // any hash values from the mark word. These hash values are
326 // used when verifying the dictionaries and so removing them
327 // from the mark word can make verification of the dictionaries
328 // fail. At the end of the GC, the original mark word values
329 // (including hash values) are restored to the appropriate
330 // objects.
331 // Universe::heap()->verify(VerifySilently, VerifyOption_G1UseMarkWord);
332 _heap->verify(VerifyOption_G1UseMarkWord);
333 }
334
335 #ifdef ASSERT
336 ShenandoahMCVerifyAfterMarkingRegionClosure cl;
337 _heap->heap_region_iterate(&cl);
338 #endif
339 }
340
341 class ShenandoahMCReclaimHumongousRegionClosure : public ShenandoahHeapRegionClosure {
342 private:
343 ShenandoahHeap* _heap;
344 public:
345 ShenandoahMCReclaimHumongousRegionClosure() : _heap(ShenandoahHeap::heap()) {
346 }
347
348 bool doHeapRegion(ShenandoahHeapRegion* r) {
349 if (r->is_humongous_start()) {
350 oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
351 if (! _heap->is_marked_complete(humongous_obj)) {
352 _heap->reclaim_humongous_region_at(r);
353 }
354 }
355 return false;
356 }
357 };
358
359
360 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
361
362 private:
363
364 ShenandoahHeap* _heap;
365 ShenandoahHeapRegionSet* _to_regions;
366 ShenandoahHeapRegion* _to_region;
367 ShenandoahHeapRegion* _from_region;
368 HeapWord* _compact_point;
369
370 public:
371
372 ShenandoahPrepareForCompactionObjectClosure(ShenandoahHeapRegionSet* to_regions, ShenandoahHeapRegion* to_region) :
373 _heap(ShenandoahHeap::heap()),
374 _to_regions(to_regions),
375 _to_region(to_region),
376 _from_region(NULL),
377 _compact_point(to_region->bottom()) {
378 }
379
380 void set_from_region(ShenandoahHeapRegion* from_region) {
381 _from_region = from_region;
382 }
383
384 ShenandoahHeapRegion* to_region() const {
385 return _to_region;
386 }
387 HeapWord* compact_point() const {
388 return _compact_point;
389 }
390 void do_object(oop p) {
391 assert(_from_region != NULL, "must set before work");
392 assert(_heap->is_marked_complete(p), "must be marked");
393 assert(! _heap->allocated_after_complete_mark_start((HeapWord*) p), "must be truly marked");
394 size_t size = p->size();
395 size_t obj_size = size + BrooksPointer::word_size();
396 if (_compact_point + obj_size > _to_region->end()) {
397 // Object doesn't fit. Pick next to-region and start compacting there.
398 _to_region->set_new_top(_compact_point);
399 ShenandoahHeapRegion* new_to_region = _to_regions->current();
400 _to_regions->next();
401 if (new_to_region == NULL) {
402 new_to_region = _from_region;
403 }
404 assert(new_to_region != _to_region, "must not reuse same to-region");
405 assert(new_to_region != NULL, "must not be NULL");
406 _to_region = new_to_region;
407 _compact_point = _to_region->bottom();
408 }
409 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
410 assert(oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p)),
411 "expect forwarded oop");
412 BrooksPointer::set_raw(p, _compact_point + BrooksPointer::word_size());
413 _compact_point += obj_size;
414 }
415 };
416
417 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
418 private:
419
420 ShenandoahHeapRegionSet** _copy_queues;
421 ShenandoahHeapRegionSet* _from_regions;
422
423 ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* copy_queue) {
424 ShenandoahHeapRegion* from_region = _from_regions->claim_next();
425 while (from_region != NULL && (from_region->is_humongous() || from_region->is_pinned())) {
426 from_region = _from_regions->claim_next();
427 }
428 if (from_region != NULL) {
429 assert(copy_queue != NULL, "sanity");
430 assert(! from_region->is_humongous(), "must not get humongous regions here");
431 assert(! from_region->is_pinned(), "no pinned region in mark-compact");
432 copy_queue->add_region(from_region);
433 }
434 return from_region;
435 }
436
437 public:
438 ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet* from_regions, ShenandoahHeapRegionSet** copy_queues) :
439 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
440 _from_regions(from_regions), _copy_queues(copy_queues) {
441 }
442
443 void work(uint worker_id) {
444 ShenandoahHeap* heap = ShenandoahHeap::heap();
445 ShenandoahHeapRegionSet* copy_queue = _copy_queues[worker_id];
446 ShenandoahHeapRegion* from_region = next_from_region(copy_queue);
447 if (from_region == NULL) return;
448 ShenandoahHeapRegionSet* to_regions = new ShenandoahHeapRegionSet(ShenandoahHeap::heap()->max_regions());
449 ShenandoahPrepareForCompactionObjectClosure cl(to_regions, from_region);
450 while (from_region != NULL) {
451 assert(from_region != NULL, "sanity");
452 cl.set_from_region(from_region);
453 heap->marked_object_iterate(from_region, &cl);
454 if (from_region != cl.to_region()) {
455 assert(from_region != NULL, "sanity");
456 to_regions->add_region(from_region);
457 }
458 from_region = next_from_region(copy_queue);
459 }
460 assert(cl.to_region() != NULL, "should not happen");
461 cl.to_region()->set_new_top(cl.compact_point());
462 while (to_regions->count() > 0) {
463 ShenandoahHeapRegion* r = to_regions->current();
464 to_regions->next();
465 if (r == NULL) {
466 to_regions->print();
467 }
468 assert(r != NULL, "should not happen");
469 r->set_new_top(r->bottom());
470 }
471 delete to_regions;
472 }
473 };
474
475 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** copy_queues) {
476 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
477 ShenandoahHeap* heap = ShenandoahHeap::heap();
478
479 ShenandoahMCReclaimHumongousRegionClosure cl;
480 heap->heap_region_iterate(&cl);
481
482 // Initialize copy queues.
483 for (uint i = 0; i < heap->max_workers(); i++) {
484 copy_queues[i] = new ShenandoahHeapRegionSet(heap->max_regions());
485 }
486
487 ShenandoahHeapRegionSet* from_regions = heap->regions();
488 from_regions->clear_current_index();
489 ShenandoahPrepareForCompactionTask prepare_task(from_regions, copy_queues);
490 heap->workers()->run_task(&prepare_task);
491 }
492
493 class ShenandoahAdjustPointersClosure : public MetadataAwareOopClosure {
494 private:
495 ShenandoahHeap* _heap;
496
497 public:
498
499 ShenandoahAdjustPointersClosure() : _heap(ShenandoahHeap::heap()) {
500 }
501
502 private:
503 template <class T>
504 inline void do_oop_work(T* p) {
505 T o = oopDesc::load_heap_oop(p);
506 if (! oopDesc::is_null(o)) {
507 oop obj = oopDesc::decode_heap_oop_not_null(o);
508 assert(_heap->is_marked_complete(obj), "must be marked");
509 oop forw = oop(BrooksPointer::get_raw(obj));
510 oopDesc::encode_store_heap_oop(p, forw);
511 }
512 }
513 public:
514 void do_oop(oop* p) {
515 do_oop_work(p);
516 }
517 void do_oop(narrowOop* p) {
518 do_oop_work(p);
519 }
520 };
521
522 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
523 private:
524 ShenandoahAdjustPointersClosure* _cl;
525 ShenandoahHeap* _heap;
526 public:
527 ShenandoahAdjustPointersObjectClosure(ShenandoahAdjustPointersClosure* cl) :
528 _cl(cl), _heap(ShenandoahHeap::heap()) {
529 }
530 void do_object(oop p) {
531 assert(_heap->is_marked_complete(p), "must be marked");
532 p->oop_iterate(_cl);
533 }
534 };
535
536 class ShenandoahAdjustPointersTask : public AbstractGangTask {
537 private:
538 ShenandoahHeapRegionSet* _regions;
539 public:
540
541 ShenandoahAdjustPointersTask(ShenandoahHeapRegionSet* regions) :
542 AbstractGangTask("Shenandoah Adjust Pointers Task"),
543 _regions(regions) {
544 }
545
546 void work(uint worker_id) {
547 ShenandoahHeap* heap = ShenandoahHeap::heap();
548 ShenandoahHeapRegion* r = _regions->claim_next();
549 ShenandoahAdjustPointersClosure cl;
550 ShenandoahAdjustPointersObjectClosure obj_cl(&cl);
551 while (r != NULL) {
552 if (! r->is_humongous_continuation()) {
553 heap->marked_object_iterate(r, &obj_cl);
554 }
555 r = _regions->claim_next();
556 }
557 }
558 };
559
560 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
561 private:
562 ShenandoahRootProcessor* _rp;
563
564 public:
565
566 ShenandoahAdjustRootPointersTask(ShenandoahRootProcessor* rp) :
567 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
568 _rp(rp) {
569 }
570
571 void work(uint worker_id) {
572 ShenandoahAdjustPointersClosure cl;
573 CLDToOopClosure adjust_cld_closure(&cl, true);
574 MarkingCodeBlobClosure adjust_code_closure(&cl,
575 CodeBlobToOopClosure::FixRelocations);
576
577 _rp->process_all_roots(&cl, &cl,
578 &adjust_cld_closure,
579 &adjust_code_closure, worker_id);
580 }
581 };
582
583 void ShenandoahMarkCompact::phase3_update_references() {
584 GCTraceTime(Info, gc, phases) time("Phase 2: Adjust pointers", _gc_timer);
585 ShenandoahHeap* heap = ShenandoahHeap::heap();
586
587 // Need cleared claim bits for the roots processing
588 ClassLoaderDataGraph::clear_claimed_marks();
589
590 WorkGang* workers = heap->workers();
591 uint nworkers = workers->active_workers();
592 {
593 COMPILER2_PRESENT(DerivedPointerTable::clear());
594
595 ShenandoahRootProcessor rp(heap, nworkers);
596 ShenandoahAdjustRootPointersTask task(&rp);
597 workers->run_task(&task);
598 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
599 }
600
601 ShenandoahHeapRegionSet* regions = heap->regions();
602 regions->clear_current_index();
603 ShenandoahAdjustPointersTask adjust_pointers_task(regions);
604 workers->run_task(&adjust_pointers_task);
605 }
606
607 class ShenandoahCompactObjectsClosure : public ObjectClosure {
608 private:
609 ShenandoahHeap* _heap;
610 public:
611 ShenandoahCompactObjectsClosure() : _heap(ShenandoahHeap::heap()) {
612 }
613 void do_object(oop p) {
614 assert(_heap->is_marked_complete(p), "must be marked");
615 size_t size = p->size();
616 HeapWord* compact_to = BrooksPointer::get_raw(p);
617 HeapWord* compact_from = (HeapWord*) p;
618 if (compact_from != compact_to) {
619 Copy::aligned_conjoint_words(compact_from, compact_to, size);
620 }
621 oop new_obj = oop(compact_to);
622 // new_obj->init_mark();
623 BrooksPointer::initialize(new_obj);
624 }
625 };
626
627 class ShenandoahCompactObjectsTask : public AbstractGangTask {
628 ShenandoahHeapRegionSet** _regions;
629 public:
630 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** regions) :
631 AbstractGangTask("Shenandoah Compact Objects Task"),
632 _regions(regions) {
633 }
634 void work(uint worker_id) {
635 ShenandoahHeap* heap = ShenandoahHeap::heap();
636 ShenandoahHeapRegionSet* copy_queue = _regions[worker_id];
637 copy_queue->clear_current_index();
638 ShenandoahCompactObjectsClosure cl;
639 ShenandoahHeapRegion* r = copy_queue->current();
640 copy_queue->next();
641 while (r != NULL) {
642 assert(! r->is_humongous(), "must not get humongous regions here");
643 heap->marked_object_iterate(r, &cl);
644 r->set_top(r->new_top());
645 r = copy_queue->current();
646 copy_queue->next();
647 }
648 }
649 };
650
651 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
652 size_t _live;
653 ShenandoahHeap* _heap;
654 public:
655
656 ShenandoahPostCompactClosure() : _live(0), _heap(ShenandoahHeap::heap()) {
657 _heap->clear_free_regions();
658 }
659
660 bool doHeapRegion(ShenandoahHeapRegion* r) {
661 // Need to reset the complete-top-at-mark-start pointer here because
662 // the complete marking bitmap is no longer valid. This ensures
663 // size-based iteration in marked_object_iterate().
664 _heap->set_complete_top_at_mark_start(r->bottom(), r->bottom());
665 r->set_in_collection_set(false);
666 if (r->is_humongous()) {
667 _live += ShenandoahHeapRegion::RegionSizeBytes;
668 } else {
669 size_t live = r->used();
670 if (live == 0) {
671 r->recycle();
672 _heap->add_free_region(r);
673 }
674 r->set_live_data(live);
675 _live += live;
676 }
677 return false;
678 }
679
680 size_t get_live() { return _live; }
681
682 };
683
684 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** copy_queues) {
685 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
686 ShenandoahHeap* heap = ShenandoahHeap::heap();
687 ShenandoahCompactObjectsTask compact_task(copy_queues);
688 heap->workers()->run_task(&compact_task);
689
690 heap->clear_cset_fast_test();
691
692 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
693 // and must ensure the bitmap is in sync.
694 heap->reset_complete_mark_bitmap(heap->workers());
695
696 {
697 ShenandoahHeap::ShenandoahHeapLock lock(heap);
698 ShenandoahPostCompactClosure post_compact;
699 heap->heap_region_iterate(&post_compact);
700
701 heap->set_used(post_compact.get_live());
702
703 }
704
705 heap->clear_cancelled_concgc();
706
707 // Also clear the next bitmap in preparation for next marking.
708 heap->reset_next_mark_bitmap(heap->workers());
709
710 for (uint i = 0; i < heap->max_workers(); i++) {
711 delete copy_queues[i];
712 }
713
714 }