1 /*
2 * Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "code/codeCache.hpp"
27 #include "gc/shared/gcTraceTime.inline.hpp"
28 #include "gc/shared/preservedMarks.inline.hpp"
29 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
30 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
31 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
33 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
34 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
35 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeuristics.hpp"
38 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
39 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
40 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
41 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
42 #include "gc/shenandoah/shenandoahUtils.hpp"
43 #include "gc/shenandoah/shenandoahVerifier.hpp"
44 #include "gc/shenandoah/shenandoahVMOperations.hpp"
45 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
46 #include "memory/metaspace.hpp"
47 #include "oops/oop.inline.hpp"
48 #include "runtime/biasedLocking.hpp"
49 #include "runtime/thread.hpp"
50 #include "utilities/copy.hpp"
51 #include "utilities/growableArray.hpp"
52 #include "gc/shared/workgroup.hpp"
53
54 ShenandoahMarkCompact::ShenandoahMarkCompact() :
55 _gc_timer(NULL),
56 _preserved_marks(new PreservedMarksSet(true)) {}
57
58 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
59 _gc_timer = gc_timer;
60 }
61
62 void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
63 ShenandoahHeap* heap = ShenandoahHeap::heap();
64
65 if (ShenandoahVerify) {
66 heap->verifier()->verify_before_fullgc();
67 }
68
69 if (VerifyBeforeGC) {
70 Universe::verify();
71 }
72
73 heap->set_full_gc_in_progress(true);
74
75 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
76 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
77
78 {
79 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
80 heap->pre_full_gc_dump(_gc_timer);
81 }
82
83 {
84 ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
85 // Full GC is supposed to recover from any GC state:
86
87 // a0. Remember if we have forwarded objects
88 bool has_forwarded_objects = heap->has_forwarded_objects();
89
90 // a1. Cancel evacuation, if in progress
91 if (heap->is_evacuation_in_progress()) {
92 heap->set_evacuation_in_progress(false);
93 }
94 assert(!heap->is_evacuation_in_progress(), "sanity");
95
96 // a2. Cancel update-refs, if in progress
97 if (heap->is_update_refs_in_progress()) {
98 heap->set_update_refs_in_progress(false);
99 }
100 assert(!heap->is_update_refs_in_progress(), "sanity");
101
102 // a3. Cancel concurrent traversal GC, if in progress
103 if (heap->is_concurrent_traversal_in_progress()) {
104 heap->traversal_gc()->reset();
105 heap->set_concurrent_traversal_in_progress(false);
106 }
107
108 // b. Cancel concurrent mark, if in progress
109 if (heap->is_concurrent_mark_in_progress()) {
110 heap->concurrent_mark()->cancel();
111 heap->stop_concurrent_marking();
112 }
113 assert(!heap->is_concurrent_mark_in_progress(), "sanity");
114
115 // c. Reset the bitmaps for new marking
116 heap->reset_mark_bitmap();
117 assert(heap->marking_context()->is_bitmap_clear(), "sanity");
118 assert(!heap->marking_context()->is_complete(), "sanity");
119
120 // d. Abandon reference discovery and clear all discovered references.
121 ReferenceProcessor* rp = heap->ref_processor();
122 rp->disable_discovery();
123 rp->abandon_partial_discovery();
124 rp->verify_no_references_recorded();
125
126 // e. Set back forwarded objects bit back, in case some steps above dropped it.
127 heap->set_has_forwarded_objects(has_forwarded_objects);
128
129 // f. Sync pinned region status from the CP marks
130 heap->sync_pinned_region_status();
131
132 // The rest of prologue:
133 BiasedLocking::preserve_marks();
134 _preserved_marks->init(heap->workers()->active_workers());
135 }
136
137 heap->make_parsable(true);
138
139 CodeCache::gc_prologue();
140
141 OrderAccess::fence();
142
143 phase1_mark_heap();
144
145 // Once marking is done, which may have fixed up forwarded objects, we can drop it.
146 // Coming out of Full GC, we would not have any forwarded objects.
147 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
148 heap->set_has_forwarded_objects(false);
149
150 heap->set_full_gc_move_in_progress(true);
151
152 // Setup workers for the rest
153 OrderAccess::fence();
154
155 // Initialize worker slices
156 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
157 for (uint i = 0; i < heap->max_workers(); i++) {
158 worker_slices[i] = new ShenandoahHeapRegionSet();
159 }
160
161 {
162 // The rest of code performs region moves, where region status is undefined
163 // until all phases run together.
164 ShenandoahHeapLocker lock(heap->lock());
165
166 phase2_calculate_target_addresses(worker_slices);
167
168 OrderAccess::fence();
169
170 phase3_update_references();
171
172 phase4_compact_objects(worker_slices);
173 }
174
175 {
176 // Epilogue
177 SharedRestorePreservedMarksTaskExecutor exec(heap->workers());
178 _preserved_marks->restore(&exec);
179 BiasedLocking::restore_marks();
180 _preserved_marks->reclaim();
181
182 CodeCache::gc_epilogue();
183 }
184
185 // Resize metaspace
186 MetaspaceGC::compute_new_size();
187
188 // Free worker slices
189 for (uint i = 0; i < heap->max_workers(); i++) {
190 delete worker_slices[i];
191 }
192 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
193
194 heap->set_full_gc_move_in_progress(false);
195 heap->set_full_gc_in_progress(false);
196
197 if (ShenandoahVerify) {
198 heap->verifier()->verify_after_fullgc();
199 }
200
201 if (VerifyAfterGC) {
202 Universe::verify();
203 }
204
205 {
206 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
207 heap->post_full_gc_dump(_gc_timer);
208 }
209 }
210
211 class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
212 private:
213 ShenandoahMarkingContext* const _ctx;
214
215 public:
216 ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
217
218 void heap_region_do(ShenandoahHeapRegion *r) {
219 _ctx->capture_top_at_mark_start(r);
220 r->clear_live_data();
221 r->set_concurrent_iteration_safe_limit(r->top());
222 }
223 };
224
225 void ShenandoahMarkCompact::phase1_mark_heap() {
226 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
227 ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
228
229 ShenandoahHeap* heap = ShenandoahHeap::heap();
230
231 ShenandoahPrepareForMarkClosure cl;
232 heap->heap_region_iterate(&cl);
233
234 ShenandoahConcurrentMark* cm = heap->concurrent_mark();
235
236 heap->set_process_references(heap->heuristics()->can_process_references());
237 heap->set_unload_classes(heap->heuristics()->can_unload_classes());
238
239 ReferenceProcessor* rp = heap->ref_processor();
240 // enable ("weak") refs discovery
241 rp->enable_discovery(true /*verify_no_refs*/);
242 rp->setup_policy(true); // forcefully purge all soft references
243 rp->set_active_mt_degree(heap->workers()->active_workers());
244
245 cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
246 cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
247 cm->finish_mark_from_roots(/* full_gc = */ true);
248
249 heap->mark_complete_marking_context();
250 }
251
252 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
253 private:
254 PreservedMarks* const _preserved_marks;
255 ShenandoahHeap* const _heap;
256 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
257 int _empty_regions_pos;
258 ShenandoahHeapRegion* _to_region;
259 ShenandoahHeapRegion* _from_region;
260 HeapWord* _compact_point;
261
262 public:
263 ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
264 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
265 ShenandoahHeapRegion* to_region) :
266 _preserved_marks(preserved_marks),
267 _heap(ShenandoahHeap::heap()),
268 _empty_regions(empty_regions),
269 _empty_regions_pos(0),
270 _to_region(to_region),
271 _from_region(NULL),
272 _compact_point(to_region->bottom()) {}
273
274 void set_from_region(ShenandoahHeapRegion* from_region) {
275 _from_region = from_region;
276 }
277
278 void finish_region() {
279 assert(_to_region != NULL, "should not happen");
280 _to_region->set_new_top(_compact_point);
281 }
282
283 bool is_compact_same_region() {
284 return _from_region == _to_region;
285 }
286
287 int empty_regions_pos() {
288 return _empty_regions_pos;
289 }
290
291 void do_object(oop p) {
292 assert(_from_region != NULL, "must set before work");
293 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
294 assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
295
296 size_t obj_size = p->size();
297 if (_compact_point + obj_size > _to_region->end()) {
298 finish_region();
299
300 // Object doesn't fit. Pick next empty region and start compacting there.
301 ShenandoahHeapRegion* new_to_region;
302 if (_empty_regions_pos < _empty_regions.length()) {
303 new_to_region = _empty_regions.at(_empty_regions_pos);
304 _empty_regions_pos++;
305 } else {
306 // Out of empty region? Compact within the same region.
307 new_to_region = _from_region;
308 }
309
310 assert(new_to_region != _to_region, "must not reuse same to-region");
311 assert(new_to_region != NULL, "must not be NULL");
312 _to_region = new_to_region;
313 _compact_point = _to_region->bottom();
314 }
315
316 // Object fits into current region, record new location:
317 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
318 shenandoah_assert_not_forwarded(NULL, p);
319 _preserved_marks->push_if_necessary(p, p->mark_raw());
320 p->forward_to(oop(_compact_point));
321 _compact_point += obj_size;
322 }
323 };
324
325 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
326 private:
327 PreservedMarksSet* const _preserved_marks;
328 ShenandoahHeap* const _heap;
329 ShenandoahHeapRegionSet** const _worker_slices;
330 ShenandoahRegionIterator _heap_regions;
331
332 ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
333 ShenandoahHeapRegion* from_region = _heap_regions.next();
334
335 // Look for next candidate for this slice:
336 while (from_region != NULL) {
337 // Empty region: get it into the slice to defragment the slice itself.
338 // We could have skipped this without violating correctness, but we really
339 // want to compact all live regions to the start of the heap, which sometimes
340 // means moving them into the fully empty regions.
341 if (from_region->is_empty()) break;
342
343 // Can move the region, and this is not the humongous region. Humongous
344 // moves are special cased here, because their moves are handled separately.
345 if (from_region->is_stw_move_allowed() && !from_region->is_humongous()) break;
346
347 from_region = _heap_regions.next();
348 }
349
350 if (from_region != NULL) {
351 assert(slice != NULL, "sanity");
352 assert(!from_region->is_humongous(), "this path cannot handle humongous regions");
353 assert(from_region->is_empty() || from_region->is_stw_move_allowed(), "only regions that can be moved in mark-compact");
354 slice->add_region(from_region);
355 }
356
357 return from_region;
358 }
359
360 public:
361 ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
362 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
363 _preserved_marks(preserved_marks),
364 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
365 }
366
367 void work(uint worker_id) {
368 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
369 ShenandoahHeapRegion* from_region = next_from_region(slice);
370 // No work?
371 if (from_region == NULL) {
372 return;
373 }
374
375 // Sliding compaction. Walk all regions in the slice, and compact them.
376 // Remember empty regions and reuse them as needed.
377 ResourceMark rm;
378 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
379 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
380 while (from_region != NULL) {
381 cl.set_from_region(from_region);
382 if (from_region->has_live()) {
383 _heap->marked_object_iterate(from_region, &cl);
384 }
385
386 // Compacted the region to somewhere else? From-region is empty then.
387 if (!cl.is_compact_same_region()) {
388 empty_regions.append(from_region);
389 }
390 from_region = next_from_region(slice);
391 }
392 cl.finish_region();
393
394 // Mark all remaining regions as empty
395 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
396 ShenandoahHeapRegion* r = empty_regions.at(pos);
397 r->set_new_top(r->bottom());
398 }
399 }
400 };
401
402 void ShenandoahMarkCompact::calculate_target_humongous_objects() {
403 ShenandoahHeap* heap = ShenandoahHeap::heap();
404
405 // Compute the new addresses for humongous objects. We need to do this after addresses
406 // for regular objects are calculated, and we know what regions in heap suffix are
407 // available for humongous moves.
408 //
409 // Scan the heap backwards, because we are compacting humongous regions towards the end.
410 // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
411 // humongous start there.
412 //
413 // The complication is potential non-movable regions during the scan. If such region is
414 // detected, then sliding restarts towards that non-movable region.
415
416 size_t to_begin = heap->num_regions();
417 size_t to_end = heap->num_regions();
418
419 for (size_t c = heap->num_regions(); c > 0; c--) {
420 ShenandoahHeapRegion *r = heap->get_region(c - 1);
421 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
422 // To-region candidate: record this, and continue scan
423 to_begin = r->region_number();
424 continue;
425 }
426
427 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
428 // From-region candidate: movable humongous region
429 oop old_obj = oop(r->bottom());
430 size_t words_size = old_obj->size();
431 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
432
433 size_t start = to_end - num_regions;
434
435 if (start >= to_begin && start != r->region_number()) {
436 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
437 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark_raw());
438 old_obj->forward_to(oop(heap->get_region(start)->bottom()));
439 to_end = start;
440 continue;
441 }
442 }
443
444 // Failed to fit. Scan starting from current region.
445 to_begin = r->region_number();
446 to_end = r->region_number();
447 }
448 }
449
450 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
451 private:
452 ShenandoahHeap* const _heap;
453
454 public:
455 ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
456 void heap_region_do(ShenandoahHeapRegion* r) {
457 if (r->is_trash()) {
458 r->recycle();
459 }
460 if (r->is_cset()) {
461 r->make_regular_bypass();
462 }
463 if (r->is_empty_uncommitted()) {
464 r->make_committed_bypass();
465 }
466 assert (r->is_committed(), "only committed regions in heap now, see region " SIZE_FORMAT, r->region_number());
467
468 // Record current region occupancy: this communicates empty regions are free
469 // to the rest of Full GC code.
470 r->set_new_top(r->top());
471 }
472 };
473
474 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
475 private:
476 ShenandoahHeap* const _heap;
477 ShenandoahMarkingContext* const _ctx;
478
479 public:
480 ShenandoahTrashImmediateGarbageClosure() :
481 _heap(ShenandoahHeap::heap()),
482 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
483
484 void heap_region_do(ShenandoahHeapRegion* r) {
485 if (r->is_humongous_start()) {
486 oop humongous_obj = oop(r->bottom());
487 if (!_ctx->is_marked(humongous_obj)) {
488 assert(!r->has_live(),
489 "Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
490 _heap->trash_humongous_region_at(r);
491 } else {
492 assert(r->has_live(),
493 "Region " SIZE_FORMAT " should have live", r->region_number());
494 }
495 } else if (r->is_humongous_continuation()) {
496 // If we hit continuation, the non-live humongous starts should have been trashed already
497 assert(r->humongous_start_region()->has_live(),
498 "Region " SIZE_FORMAT " should have live", r->region_number());
499 } else if (r->is_regular()) {
500 if (!r->has_live()) {
501 r->make_trash_immediate();
502 }
503 }
504 }
505 };
506
507 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
508 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
509 ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
510
511 ShenandoahHeap* heap = ShenandoahHeap::heap();
512
513 // About to figure out which regions can be compacted, make sure pinning status
514 // had been updated in GC prologue.
515 heap->assert_pinned_region_status();
516
517 {
518 // Trash the immediately collectible regions before computing addresses
519 ShenandoahTrashImmediateGarbageClosure tigcl;
520 heap->heap_region_iterate(&tigcl);
521
522 // Make sure regions are in good state: committed, active, clean.
523 // This is needed because we are potentially sliding the data through them.
524 ShenandoahEnsureHeapActiveClosure ecl;
525 heap->heap_region_iterate(&ecl);
526 }
527
528 // Compute the new addresses for regular objects
529 {
530 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
531 ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
532 heap->workers()->run_task(&prepare_task);
533 }
534
535 // Compute the new addresses for humongous objects
536 {
537 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
538 calculate_target_humongous_objects();
539 }
540 }
541
542 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
543 private:
544 ShenandoahHeap* const _heap;
545 ShenandoahMarkingContext* const _ctx;
546
547 template <class T>
548 inline void do_oop_work(T* p) {
549 T o = RawAccess<>::oop_load(p);
550 if (!CompressedOops::is_null(o)) {
551 oop obj = CompressedOops::decode_not_null(o);
552 assert(_ctx->is_marked(obj), "must be marked");
553 if (obj->is_forwarded()) {
554 oop forw = obj->forwardee();
555 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
556 }
557 }
558 }
559
560 public:
561 ShenandoahAdjustPointersClosure() :
562 _heap(ShenandoahHeap::heap()),
563 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
564
565 void do_oop(oop* p) { do_oop_work(p); }
566 void do_oop(narrowOop* p) { do_oop_work(p); }
567 };
568
569 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
570 private:
571 ShenandoahHeap* const _heap;
572 ShenandoahAdjustPointersClosure _cl;
573
574 public:
575 ShenandoahAdjustPointersObjectClosure() :
576 _heap(ShenandoahHeap::heap()) {
577 }
578 void do_object(oop p) {
579 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
580 p->oop_iterate(&_cl);
581 }
582 };
583
584 class ShenandoahAdjustPointersTask : public AbstractGangTask {
585 private:
586 ShenandoahHeap* const _heap;
587 ShenandoahRegionIterator _regions;
588
589 public:
590 ShenandoahAdjustPointersTask() :
591 AbstractGangTask("Shenandoah Adjust Pointers Task"),
592 _heap(ShenandoahHeap::heap()) {
593 }
594
595 void work(uint worker_id) {
596 ShenandoahAdjustPointersObjectClosure obj_cl;
597 ShenandoahHeapRegion* r = _regions.next();
598 while (r != NULL) {
599 if (!r->is_humongous_continuation() && r->has_live()) {
600 _heap->marked_object_iterate(r, &obj_cl);
601 }
602 r = _regions.next();
603 }
604 }
605 };
606
607 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
608 private:
609 ShenandoahRootAdjuster* _rp;
610 PreservedMarksSet* _preserved_marks;
611 public:
612 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
613 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
614 _rp(rp),
615 _preserved_marks(preserved_marks) {}
616
617 void work(uint worker_id) {
618 ShenandoahAdjustPointersClosure cl;
619 _rp->roots_do(worker_id, &cl);
620 _preserved_marks->get(worker_id)->adjust_during_full_gc();
621 }
622 };
623
624 void ShenandoahMarkCompact::phase3_update_references() {
625 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
626 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
627
628 ShenandoahHeap* heap = ShenandoahHeap::heap();
629
630 WorkGang* workers = heap->workers();
631 uint nworkers = workers->active_workers();
632 {
633 #if COMPILER2_OR_JVMCI
634 DerivedPointerTable::clear();
635 #endif
636 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_roots);
637 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
638 workers->run_task(&task);
639 #if COMPILER2_OR_JVMCI
640 DerivedPointerTable::update_pointers();
641 #endif
642 }
643
644 ShenandoahAdjustPointersTask adjust_pointers_task;
645 workers->run_task(&adjust_pointers_task);
646 }
647
648 class ShenandoahCompactObjectsClosure : public ObjectClosure {
649 private:
650 ShenandoahHeap* const _heap;
651 uint const _worker_id;
652
653 public:
654 ShenandoahCompactObjectsClosure(uint worker_id) :
655 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
656
657 void do_object(oop p) {
658 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
659 size_t size = (size_t)p->size();
660 if (p->is_forwarded()) {
661 HeapWord* compact_from = (HeapWord*) p;
662 HeapWord* compact_to = (HeapWord*) p->forwardee();
663 Copy::aligned_conjoint_words(compact_from, compact_to, size);
664 oop new_obj = oop(compact_to);
665 new_obj->init_mark_raw();
666 }
667 }
668 };
669
670 class ShenandoahCompactObjectsTask : public AbstractGangTask {
671 private:
672 ShenandoahHeap* const _heap;
673 ShenandoahHeapRegionSet** const _worker_slices;
674
675 public:
676 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
677 AbstractGangTask("Shenandoah Compact Objects Task"),
678 _heap(ShenandoahHeap::heap()),
679 _worker_slices(worker_slices) {
680 }
681
682 void work(uint worker_id) {
683 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
684
685 ShenandoahCompactObjectsClosure cl(worker_id);
686 ShenandoahHeapRegion* r = slice.next();
687 while (r != NULL) {
688 assert(!r->is_humongous(), "must not get humongous regions here");
689 if (r->has_live()) {
690 _heap->marked_object_iterate(r, &cl);
691 }
692 r->set_top(r->new_top());
693 r = slice.next();
694 }
695 }
696 };
697
698 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
699 private:
700 ShenandoahHeap* const _heap;
701 size_t _live;
702
703 public:
704 ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
705 _heap->free_set()->clear();
706 }
707
708 void heap_region_do(ShenandoahHeapRegion* r) {
709 assert (!r->is_cset(), "cset regions should have been demoted already");
710
711 // Need to reset the complete-top-at-mark-start pointer here because
712 // the complete marking bitmap is no longer valid. This ensures
713 // size-based iteration in marked_object_iterate().
714 // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
715 // pinned regions.
716 if (!r->is_pinned()) {
717 _heap->complete_marking_context()->reset_top_at_mark_start(r);
718 }
719
720 size_t live = r->used();
721
722 // Make empty regions that have been allocated into regular
723 if (r->is_empty() && live > 0) {
724 r->make_regular_bypass();
725 }
726
727 // Reclaim regular regions that became empty
728 if (r->is_regular() && live == 0) {
729 r->make_trash();
730 }
731
732 // Recycle all trash regions
733 if (r->is_trash()) {
734 live = 0;
735 r->recycle();
736 }
737
738 r->set_live_data(live);
739 r->reset_alloc_metadata_to_shared();
740 _live += live;
741 }
742
743 size_t get_live() {
744 return _live;
745 }
746 };
747
748 void ShenandoahMarkCompact::compact_humongous_objects() {
749 // Compact humongous regions, based on their fwdptr objects.
750 //
751 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
752 // humongous regions are already compacted, and do not require further moves, which alleviates
753 // sliding costs. We may consider doing this in parallel in future.
754
755 ShenandoahHeap* heap = ShenandoahHeap::heap();
756
757 for (size_t c = heap->num_regions(); c > 0; c--) {
758 ShenandoahHeapRegion* r = heap->get_region(c - 1);
759 if (r->is_humongous_start()) {
760 oop old_obj = oop(r->bottom());
761 if (!old_obj->is_forwarded()) {
762 // No need to move the object, it stays at the same slot
763 continue;
764 }
765 size_t words_size = old_obj->size();
766 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
767
768 size_t old_start = r->region_number();
769 size_t old_end = old_start + num_regions - 1;
770 size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
771 size_t new_end = new_start + num_regions - 1;
772 assert(old_start != new_start, "must be real move");
773 assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->region_number());
774
775 Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
776 heap->get_region(new_start)->bottom(),
777 ShenandoahHeapRegion::region_size_words()*num_regions);
778
779 oop new_obj = oop(heap->get_region(new_start)->bottom());
780 new_obj->init_mark_raw();
781
782 {
783 for (size_t c = old_start; c <= old_end; c++) {
784 ShenandoahHeapRegion* r = heap->get_region(c);
785 r->make_regular_bypass();
786 r->set_top(r->bottom());
787 }
788
789 for (size_t c = new_start; c <= new_end; c++) {
790 ShenandoahHeapRegion* r = heap->get_region(c);
791 if (c == new_start) {
792 r->make_humongous_start_bypass();
793 } else {
794 r->make_humongous_cont_bypass();
795 }
796
797 // Trailing region may be non-full, record the remainder there
798 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
799 if ((c == new_end) && (remainder != 0)) {
800 r->set_top(r->bottom() + remainder);
801 } else {
802 r->set_top(r->end());
803 }
804
805 r->reset_alloc_metadata_to_shared();
806 }
807 }
808 }
809 }
810 }
811
812 // This is slightly different to ShHeap::reset_next_mark_bitmap:
813 // we need to remain able to walk pinned regions.
814 // Since pinned region do not move and don't get compacted, we will get holes with
815 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
816 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
817 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
818 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
819 class ShenandoahMCResetCompleteBitmapTask : public AbstractGangTask {
820 private:
821 ShenandoahRegionIterator _regions;
822
823 public:
824 ShenandoahMCResetCompleteBitmapTask() :
825 AbstractGangTask("Parallel Reset Bitmap Task") {
826 }
827
828 void work(uint worker_id) {
829 ShenandoahHeapRegion* region = _regions.next();
830 ShenandoahHeap* heap = ShenandoahHeap::heap();
831 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
832 while (region != NULL) {
833 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
834 ctx->clear_bitmap(region);
835 }
836 region = _regions.next();
837 }
838 }
839 };
840
841 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
842 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
843 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
844
845 ShenandoahHeap* heap = ShenandoahHeap::heap();
846
847 // Compact regular objects first
848 {
849 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
850 ShenandoahCompactObjectsTask compact_task(worker_slices);
851 heap->workers()->run_task(&compact_task);
852 }
853
854 // Compact humongous objects after regular object moves
855 {
856 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
857 compact_humongous_objects();
858 }
859
860 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
861 // and must ensure the bitmap is in sync.
862 {
863 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
864 ShenandoahMCResetCompleteBitmapTask task;
865 heap->workers()->run_task(&task);
866 }
867
868 // Bring regions in proper states after the collection, and set heap properties.
869 {
870 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
871
872 ShenandoahPostCompactClosure post_compact;
873 heap->heap_region_iterate(&post_compact);
874 heap->set_used(post_compact.get_live());
875
876 heap->collection_set()->clear();
877 heap->free_set()->rebuild();
878 }
879
880 heap->clear_cancelled_gc();
881 }