1 /*
2 * Copyright (c) 2014, 2020, Red Hat, Inc. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "code/codeCache.hpp"
28 #include "gc/shared/gcTraceTime.inline.hpp"
29 #include "gc/shared/preservedMarks.inline.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
31 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
32 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
33 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
34 #include "gc/shenandoah/shenandoahFreeSet.hpp"
35 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
36 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
38 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
39 #include "gc/shenandoah/shenandoahHeuristics.hpp"
40 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
41 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
42 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
43 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
44 #include "gc/shenandoah/shenandoahUtils.hpp"
45 #include "gc/shenandoah/shenandoahVerifier.hpp"
46 #include "gc/shenandoah/shenandoahVMOperations.hpp"
47 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
48 #include "memory/metaspace.hpp"
49 #include "memory/universe.hpp"
50 #include "oops/compressedOops.inline.hpp"
51 #include "oops/oop.inline.hpp"
52 #include "runtime/biasedLocking.hpp"
53 #include "runtime/orderAccess.hpp"
54 #include "runtime/thread.hpp"
55 #include "utilities/copy.hpp"
56 #include "utilities/growableArray.hpp"
57 #include "gc/shared/workgroup.hpp"
58
59 ShenandoahMarkCompact::ShenandoahMarkCompact() :
60 _gc_timer(NULL),
61 _preserved_marks(new PreservedMarksSet(true)) {}
62
63 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
64 _gc_timer = gc_timer;
65 }
66
67 void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
68 ShenandoahHeap* heap = ShenandoahHeap::heap();
69
70 if (ShenandoahVerify) {
71 heap->verifier()->verify_before_fullgc();
72 }
73
74 if (VerifyBeforeGC) {
75 Universe::verify();
76 }
77
78 // Degenerated GC may carry concurrent_root_in_progress flag when upgrading to
79 // full GC. We need to reset it before mutators resume.
80 if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
81 heap->set_concurrent_root_in_progress(false);
82 }
83
84 heap->set_full_gc_in_progress(true);
85
86 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
87 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
88
89 {
90 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
91 heap->pre_full_gc_dump(_gc_timer);
92 }
93
94 {
95 ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
96 // Full GC is supposed to recover from any GC state:
97
98 // a0. Remember if we have forwarded objects
99 bool has_forwarded_objects = heap->has_forwarded_objects();
100
101 // a1. Cancel evacuation, if in progress
102 if (heap->is_evacuation_in_progress()) {
103 heap->set_evacuation_in_progress(false);
104 }
105 assert(!heap->is_evacuation_in_progress(), "sanity");
106
107 // a2. Cancel update-refs, if in progress
108 if (heap->is_update_refs_in_progress()) {
109 heap->set_update_refs_in_progress(false);
110 }
111 assert(!heap->is_update_refs_in_progress(), "sanity");
112
113 // a3. Cancel concurrent traversal GC, if in progress
114 if (heap->is_concurrent_traversal_in_progress()) {
115 heap->traversal_gc()->reset();
116 heap->set_concurrent_traversal_in_progress(false);
117 }
118
119 // b. Cancel concurrent mark, if in progress
120 if (heap->is_concurrent_mark_in_progress()) {
121 heap->concurrent_mark()->cancel();
122 heap->set_concurrent_mark_in_progress(false);
123 }
124 assert(!heap->is_concurrent_mark_in_progress(), "sanity");
125
126 // c. Reset the bitmaps for new marking
127 heap->reset_mark_bitmap();
128 assert(heap->marking_context()->is_bitmap_clear(), "sanity");
129 assert(!heap->marking_context()->is_complete(), "sanity");
130
131 // d. Abandon reference discovery and clear all discovered references.
132 ReferenceProcessor* rp = heap->ref_processor();
133 rp->disable_discovery();
134 rp->abandon_partial_discovery();
135 rp->verify_no_references_recorded();
136
137 // e. Set back forwarded objects bit back, in case some steps above dropped it.
138 heap->set_has_forwarded_objects(has_forwarded_objects);
139
140 // f. Sync pinned region status from the CP marks
141 heap->sync_pinned_region_status();
142
143 // The rest of prologue:
144 BiasedLocking::preserve_marks();
145 _preserved_marks->init(heap->workers()->active_workers());
146 }
147
148 heap->make_parsable(true);
149
150 OrderAccess::fence();
151
152 phase1_mark_heap();
153
154 // Once marking is done, which may have fixed up forwarded objects, we can drop it.
155 // Coming out of Full GC, we would not have any forwarded objects.
156 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
157 heap->set_has_forwarded_objects(false);
158
159 heap->set_full_gc_move_in_progress(true);
160
161 // Setup workers for the rest
162 OrderAccess::fence();
163
164 // Initialize worker slices
165 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
166 for (uint i = 0; i < heap->max_workers(); i++) {
167 worker_slices[i] = new ShenandoahHeapRegionSet();
168 }
169
170 {
171 // The rest of code performs region moves, where region status is undefined
172 // until all phases run together.
173 ShenandoahHeapLocker lock(heap->lock());
174
175 phase2_calculate_target_addresses(worker_slices);
176
177 OrderAccess::fence();
178
179 phase3_update_references();
180
181 phase4_compact_objects(worker_slices);
182 }
183
184 {
185 // Epilogue
186 _preserved_marks->restore(heap->workers());
187 BiasedLocking::restore_marks();
188 _preserved_marks->reclaim();
189 }
190
191 // Resize metaspace
192 MetaspaceGC::compute_new_size();
193
194 // Free worker slices
195 for (uint i = 0; i < heap->max_workers(); i++) {
196 delete worker_slices[i];
197 }
198 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
199
200 heap->set_full_gc_move_in_progress(false);
201 heap->set_full_gc_in_progress(false);
202
203 if (ShenandoahVerify) {
204 heap->verifier()->verify_after_fullgc();
205 }
206
207 if (VerifyAfterGC) {
208 Universe::verify();
209 }
210
211 {
212 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
213 heap->post_full_gc_dump(_gc_timer);
214 }
215 }
216
217 class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
218 private:
219 ShenandoahMarkingContext* const _ctx;
220
221 public:
222 ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
223
224 void heap_region_do(ShenandoahHeapRegion *r) {
225 _ctx->capture_top_at_mark_start(r);
226 r->clear_live_data();
227 r->set_concurrent_iteration_safe_limit(r->top());
228 }
229 };
230
231 void ShenandoahMarkCompact::phase1_mark_heap() {
232 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
233 ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
234
235 ShenandoahHeap* heap = ShenandoahHeap::heap();
236
237 ShenandoahPrepareForMarkClosure cl;
238 heap->heap_region_iterate(&cl);
239
240 ShenandoahConcurrentMark* cm = heap->concurrent_mark();
241
242 heap->set_process_references(heap->heuristics()->can_process_references());
243 heap->set_unload_classes(heap->heuristics()->can_unload_classes());
244
245 ReferenceProcessor* rp = heap->ref_processor();
246 // enable ("weak") refs discovery
247 rp->enable_discovery(true /*verify_no_refs*/);
248 rp->setup_policy(true); // forcefully purge all soft references
249 rp->set_active_mt_degree(heap->workers()->active_workers());
250
251 cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
252 cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
253 cm->finish_mark_from_roots(/* full_gc = */ true);
254 heap->mark_complete_marking_context();
255 heap->parallel_cleaning(true /* full_gc */);
256 }
257
258 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
259 private:
260 PreservedMarks* const _preserved_marks;
261 ShenandoahHeap* const _heap;
262 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
263 int _empty_regions_pos;
264 ShenandoahHeapRegion* _to_region;
265 ShenandoahHeapRegion* _from_region;
266 HeapWord* _compact_point;
267
268 public:
269 ShenandoahPrepareForCompactionObjectClosure(PreservedMarks* preserved_marks,
270 GrowableArray<ShenandoahHeapRegion*>& empty_regions,
271 ShenandoahHeapRegion* to_region) :
272 _preserved_marks(preserved_marks),
273 _heap(ShenandoahHeap::heap()),
274 _empty_regions(empty_regions),
275 _empty_regions_pos(0),
276 _to_region(to_region),
277 _from_region(NULL),
278 _compact_point(to_region->bottom()) {}
279
280 void set_from_region(ShenandoahHeapRegion* from_region) {
281 _from_region = from_region;
282 }
283
284 void finish_region() {
285 assert(_to_region != NULL, "should not happen");
286 _to_region->set_new_top(_compact_point);
287 }
288
289 bool is_compact_same_region() {
290 return _from_region == _to_region;
291 }
292
293 int empty_regions_pos() {
294 return _empty_regions_pos;
295 }
296
297 void do_object(oop p) {
298 assert(_from_region != NULL, "must set before work");
299 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
300 assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
301
302 size_t obj_size = p->size();
303 if (_compact_point + obj_size > _to_region->end()) {
304 finish_region();
305
306 // Object doesn't fit. Pick next empty region and start compacting there.
307 ShenandoahHeapRegion* new_to_region;
308 if (_empty_regions_pos < _empty_regions.length()) {
309 new_to_region = _empty_regions.at(_empty_regions_pos);
310 _empty_regions_pos++;
311 } else {
312 // Out of empty region? Compact within the same region.
313 new_to_region = _from_region;
314 }
315
316 assert(new_to_region != _to_region, "must not reuse same to-region");
317 assert(new_to_region != NULL, "must not be NULL");
318 _to_region = new_to_region;
319 _compact_point = _to_region->bottom();
320 }
321
322 // Object fits into current region, record new location:
323 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
324 shenandoah_assert_not_forwarded(NULL, p);
325 _preserved_marks->push_if_necessary(p, p->mark_raw());
326 p->forward_to(oop(_compact_point));
327 _compact_point += obj_size;
328 }
329 };
330
331 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
332 private:
333 PreservedMarksSet* const _preserved_marks;
334 ShenandoahHeap* const _heap;
335 ShenandoahHeapRegionSet** const _worker_slices;
336 ShenandoahRegionIterator _heap_regions;
337
338 ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
339 ShenandoahHeapRegion* from_region = _heap_regions.next();
340
341 // Look for next candidate for this slice:
342 while (from_region != NULL) {
343 // Empty region: get it into the slice to defragment the slice itself.
344 // We could have skipped this without violating correctness, but we really
345 // want to compact all live regions to the start of the heap, which sometimes
346 // means moving them into the fully empty regions.
347 if (from_region->is_empty()) break;
348
349 // Can move the region, and this is not the humongous region. Humongous
350 // moves are special cased here, because their moves are handled separately.
351 if (from_region->is_stw_move_allowed() && !from_region->is_humongous()) break;
352
353 from_region = _heap_regions.next();
354 }
355
356 if (from_region != NULL) {
357 assert(slice != NULL, "sanity");
358 assert(!from_region->is_humongous(), "this path cannot handle humongous regions");
359 assert(from_region->is_empty() || from_region->is_stw_move_allowed(), "only regions that can be moved in mark-compact");
360 slice->add_region(from_region);
361 }
362
363 return from_region;
364 }
365
366 public:
367 ShenandoahPrepareForCompactionTask(PreservedMarksSet* preserved_marks, ShenandoahHeapRegionSet** worker_slices) :
368 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
369 _preserved_marks(preserved_marks),
370 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
371 }
372
373 void work(uint worker_id) {
374 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
375 ShenandoahHeapRegion* from_region = next_from_region(slice);
376 // No work?
377 if (from_region == NULL) {
378 return;
379 }
380
381 // Sliding compaction. Walk all regions in the slice, and compact them.
382 // Remember empty regions and reuse them as needed.
383 ResourceMark rm;
384 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
385 ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
386 while (from_region != NULL) {
387 cl.set_from_region(from_region);
388 if (from_region->has_live()) {
389 _heap->marked_object_iterate(from_region, &cl);
390 }
391
392 // Compacted the region to somewhere else? From-region is empty then.
393 if (!cl.is_compact_same_region()) {
394 empty_regions.append(from_region);
395 }
396 from_region = next_from_region(slice);
397 }
398 cl.finish_region();
399
400 // Mark all remaining regions as empty
401 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
402 ShenandoahHeapRegion* r = empty_regions.at(pos);
403 r->set_new_top(r->bottom());
404 }
405 }
406 };
407
408 void ShenandoahMarkCompact::calculate_target_humongous_objects() {
409 ShenandoahHeap* heap = ShenandoahHeap::heap();
410
411 // Compute the new addresses for humongous objects. We need to do this after addresses
412 // for regular objects are calculated, and we know what regions in heap suffix are
413 // available for humongous moves.
414 //
415 // Scan the heap backwards, because we are compacting humongous regions towards the end.
416 // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
417 // humongous start there.
418 //
419 // The complication is potential non-movable regions during the scan. If such region is
420 // detected, then sliding restarts towards that non-movable region.
421
422 size_t to_begin = heap->num_regions();
423 size_t to_end = heap->num_regions();
424
425 for (size_t c = heap->num_regions(); c > 0; c--) {
426 ShenandoahHeapRegion *r = heap->get_region(c - 1);
427 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
428 // To-region candidate: record this, and continue scan
429 to_begin = r->region_number();
430 continue;
431 }
432
433 if (r->is_humongous_start() && r->is_stw_move_allowed()) {
434 // From-region candidate: movable humongous region
435 oop old_obj = oop(r->bottom());
436 size_t words_size = old_obj->size();
437 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
438
439 size_t start = to_end - num_regions;
440
441 if (start >= to_begin && start != r->region_number()) {
442 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
443 _preserved_marks->get(0)->push_if_necessary(old_obj, old_obj->mark_raw());
444 old_obj->forward_to(oop(heap->get_region(start)->bottom()));
445 to_end = start;
446 continue;
447 }
448 }
449
450 // Failed to fit. Scan starting from current region.
451 to_begin = r->region_number();
452 to_end = r->region_number();
453 }
454 }
455
456 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
457 private:
458 ShenandoahHeap* const _heap;
459
460 public:
461 ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
462 void heap_region_do(ShenandoahHeapRegion* r) {
463 if (r->is_trash()) {
464 r->recycle();
465 }
466 if (r->is_cset()) {
467 r->make_regular_bypass();
468 }
469 if (r->is_empty_uncommitted()) {
470 r->make_committed_bypass();
471 }
472 assert (r->is_committed(), "only committed regions in heap now, see region " SIZE_FORMAT, r->region_number());
473
474 // Record current region occupancy: this communicates empty regions are free
475 // to the rest of Full GC code.
476 r->set_new_top(r->top());
477 }
478 };
479
480 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
481 private:
482 ShenandoahHeap* const _heap;
483 ShenandoahMarkingContext* const _ctx;
484
485 public:
486 ShenandoahTrashImmediateGarbageClosure() :
487 _heap(ShenandoahHeap::heap()),
488 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
489
490 void heap_region_do(ShenandoahHeapRegion* r) {
491 if (r->is_humongous_start()) {
492 oop humongous_obj = oop(r->bottom());
493 if (!_ctx->is_marked(humongous_obj)) {
494 assert(!r->has_live(),
495 "Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
496 _heap->trash_humongous_region_at(r);
497 } else {
498 assert(r->has_live(),
499 "Region " SIZE_FORMAT " should have live", r->region_number());
500 }
501 } else if (r->is_humongous_continuation()) {
502 // If we hit continuation, the non-live humongous starts should have been trashed already
503 assert(r->humongous_start_region()->has_live(),
504 "Region " SIZE_FORMAT " should have live", r->region_number());
505 } else if (r->is_regular()) {
506 if (!r->has_live()) {
507 r->make_trash_immediate();
508 }
509 }
510 }
511 };
512
513 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
514 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
515 ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
516
517 ShenandoahHeap* heap = ShenandoahHeap::heap();
518
519 // About to figure out which regions can be compacted, make sure pinning status
520 // had been updated in GC prologue.
521 heap->assert_pinned_region_status();
522
523 {
524 // Trash the immediately collectible regions before computing addresses
525 ShenandoahTrashImmediateGarbageClosure tigcl;
526 heap->heap_region_iterate(&tigcl);
527
528 // Make sure regions are in good state: committed, active, clean.
529 // This is needed because we are potentially sliding the data through them.
530 ShenandoahEnsureHeapActiveClosure ecl;
531 heap->heap_region_iterate(&ecl);
532 }
533
534 // Compute the new addresses for regular objects
535 {
536 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
537 ShenandoahPrepareForCompactionTask prepare_task(_preserved_marks, worker_slices);
538 heap->workers()->run_task(&prepare_task);
539 }
540
541 // Compute the new addresses for humongous objects
542 {
543 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
544 calculate_target_humongous_objects();
545 }
546 }
547
548 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
549 private:
550 ShenandoahHeap* const _heap;
551 ShenandoahMarkingContext* const _ctx;
552
553 template <class T>
554 inline void do_oop_work(T* p) {
555 T o = RawAccess<>::oop_load(p);
556 if (!CompressedOops::is_null(o)) {
557 oop obj = CompressedOops::decode_not_null(o);
558 assert_object_is_in_heap(p, obj);
559 assert(_ctx->is_marked(obj), "must be marked");
560 if (obj->is_forwarded()) {
561 oop forw = obj->forwardee();
562 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
563 }
564 }
565 }
566
567 public:
568 ShenandoahAdjustPointersClosure() :
569 _heap(ShenandoahHeap::heap()),
570 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
571
572 void do_oop(oop* p) { do_oop_work(p); }
573 void do_oop(narrowOop* p) { do_oop_work(p); }
574 };
575
576 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
577 private:
578 ShenandoahHeap* const _heap;
579 ShenandoahAdjustPointersClosure _cl;
580
581 public:
582 ShenandoahAdjustPointersObjectClosure() :
583 _heap(ShenandoahHeap::heap()) {
584 }
585 void do_object(oop p) {
586 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
587 p->oop_iterate(&_cl);
588 }
589 };
590
591 class ShenandoahAdjustPointersTask : public AbstractGangTask {
592 private:
593 ShenandoahHeap* const _heap;
594 ShenandoahRegionIterator _regions;
595
596 public:
597 ShenandoahAdjustPointersTask() :
598 AbstractGangTask("Shenandoah Adjust Pointers Task"),
599 _heap(ShenandoahHeap::heap()) {
600 }
601
602 void work(uint worker_id) {
603 ShenandoahAdjustPointersObjectClosure obj_cl;
604 ShenandoahHeapRegion* r = _regions.next();
605 while (r != NULL) {
606 if (!r->is_humongous_continuation() && r->has_live()) {
607 _heap->marked_object_iterate(r, &obj_cl);
608 }
609 r = _regions.next();
610 }
611 }
612 };
613
614 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
615 private:
616 ShenandoahRootAdjuster* _rp;
617 PreservedMarksSet* _preserved_marks;
618 public:
619 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp, PreservedMarksSet* preserved_marks) :
620 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
621 _rp(rp),
622 _preserved_marks(preserved_marks) {}
623
624 void work(uint worker_id) {
625 ShenandoahAdjustPointersClosure cl;
626 _rp->roots_do(worker_id, &cl);
627 _preserved_marks->get(worker_id)->adjust_during_full_gc();
628 }
629 };
630
631 void ShenandoahMarkCompact::phase3_update_references() {
632 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
633 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
634
635 ShenandoahHeap* heap = ShenandoahHeap::heap();
636
637 WorkGang* workers = heap->workers();
638 uint nworkers = workers->active_workers();
639 {
640 #if COMPILER2_OR_JVMCI
641 DerivedPointerTable::clear();
642 #endif
643 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_roots);
644 ShenandoahAdjustRootPointersTask task(&rp, _preserved_marks);
645 workers->run_task(&task);
646 #if COMPILER2_OR_JVMCI
647 DerivedPointerTable::update_pointers();
648 #endif
649 }
650
651 ShenandoahAdjustPointersTask adjust_pointers_task;
652 workers->run_task(&adjust_pointers_task);
653 }
654
655 class ShenandoahCompactObjectsClosure : public ObjectClosure {
656 private:
657 ShenandoahHeap* const _heap;
658 uint const _worker_id;
659
660 public:
661 ShenandoahCompactObjectsClosure(uint worker_id) :
662 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
663
664 void do_object(oop p) {
665 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
666 size_t size = (size_t)p->size();
667 if (p->is_forwarded()) {
668 HeapWord* compact_from = (HeapWord*) p;
669 HeapWord* compact_to = (HeapWord*) p->forwardee();
670 Copy::aligned_conjoint_words(compact_from, compact_to, size);
671 oop new_obj = oop(compact_to);
672 new_obj->init_mark_raw();
673 }
674 }
675 };
676
677 class ShenandoahCompactObjectsTask : public AbstractGangTask {
678 private:
679 ShenandoahHeap* const _heap;
680 ShenandoahHeapRegionSet** const _worker_slices;
681
682 public:
683 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
684 AbstractGangTask("Shenandoah Compact Objects Task"),
685 _heap(ShenandoahHeap::heap()),
686 _worker_slices(worker_slices) {
687 }
688
689 void work(uint worker_id) {
690 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
691
692 ShenandoahCompactObjectsClosure cl(worker_id);
693 ShenandoahHeapRegion* r = slice.next();
694 while (r != NULL) {
695 assert(!r->is_humongous(), "must not get humongous regions here");
696 if (r->has_live()) {
697 _heap->marked_object_iterate(r, &cl);
698 }
699 r->set_top(r->new_top());
700 r = slice.next();
701 }
702 }
703 };
704
705 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
706 private:
707 ShenandoahHeap* const _heap;
708 size_t _live;
709
710 public:
711 ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
712 _heap->free_set()->clear();
713 }
714
715 void heap_region_do(ShenandoahHeapRegion* r) {
716 assert (!r->is_cset(), "cset regions should have been demoted already");
717
718 // Need to reset the complete-top-at-mark-start pointer here because
719 // the complete marking bitmap is no longer valid. This ensures
720 // size-based iteration in marked_object_iterate().
721 // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
722 // pinned regions.
723 if (!r->is_pinned()) {
724 _heap->complete_marking_context()->reset_top_at_mark_start(r);
725 }
726
727 size_t live = r->used();
728
729 // Make empty regions that have been allocated into regular
730 if (r->is_empty() && live > 0) {
731 r->make_regular_bypass();
732 }
733
734 // Reclaim regular regions that became empty
735 if (r->is_regular() && live == 0) {
736 r->make_trash();
737 }
738
739 // Recycle all trash regions
740 if (r->is_trash()) {
741 live = 0;
742 r->recycle();
743 }
744
745 r->set_live_data(live);
746 r->reset_alloc_metadata_to_shared();
747 _live += live;
748 }
749
750 size_t get_live() {
751 return _live;
752 }
753 };
754
755 void ShenandoahMarkCompact::compact_humongous_objects() {
756 // Compact humongous regions, based on their fwdptr objects.
757 //
758 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
759 // humongous regions are already compacted, and do not require further moves, which alleviates
760 // sliding costs. We may consider doing this in parallel in future.
761
762 ShenandoahHeap* heap = ShenandoahHeap::heap();
763
764 for (size_t c = heap->num_regions(); c > 0; c--) {
765 ShenandoahHeapRegion* r = heap->get_region(c - 1);
766 if (r->is_humongous_start()) {
767 oop old_obj = oop(r->bottom());
768 if (!old_obj->is_forwarded()) {
769 // No need to move the object, it stays at the same slot
770 continue;
771 }
772 size_t words_size = old_obj->size();
773 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
774
775 size_t old_start = r->region_number();
776 size_t old_end = old_start + num_regions - 1;
777 size_t new_start = heap->heap_region_index_containing(old_obj->forwardee());
778 size_t new_end = new_start + num_regions - 1;
779 assert(old_start != new_start, "must be real move");
780 assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->region_number());
781
782 Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
783 heap->get_region(new_start)->bottom(),
784 ShenandoahHeapRegion::region_size_words()*num_regions);
785
786 oop new_obj = oop(heap->get_region(new_start)->bottom());
787 new_obj->init_mark_raw();
788
789 {
790 for (size_t c = old_start; c <= old_end; c++) {
791 ShenandoahHeapRegion* r = heap->get_region(c);
792 r->make_regular_bypass();
793 r->set_top(r->bottom());
794 }
795
796 for (size_t c = new_start; c <= new_end; c++) {
797 ShenandoahHeapRegion* r = heap->get_region(c);
798 if (c == new_start) {
799 r->make_humongous_start_bypass();
800 } else {
801 r->make_humongous_cont_bypass();
802 }
803
804 // Trailing region may be non-full, record the remainder there
805 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
806 if ((c == new_end) && (remainder != 0)) {
807 r->set_top(r->bottom() + remainder);
808 } else {
809 r->set_top(r->end());
810 }
811
812 r->reset_alloc_metadata_to_shared();
813 }
814 }
815 }
816 }
817 }
818
819 // This is slightly different to ShHeap::reset_next_mark_bitmap:
820 // we need to remain able to walk pinned regions.
821 // Since pinned region do not move and don't get compacted, we will get holes with
822 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
823 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
824 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
825 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
826 class ShenandoahMCResetCompleteBitmapTask : public AbstractGangTask {
827 private:
828 ShenandoahRegionIterator _regions;
829
830 public:
831 ShenandoahMCResetCompleteBitmapTask() :
832 AbstractGangTask("Parallel Reset Bitmap Task") {
833 }
834
835 void work(uint worker_id) {
836 ShenandoahHeapRegion* region = _regions.next();
837 ShenandoahHeap* heap = ShenandoahHeap::heap();
838 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
839 while (region != NULL) {
840 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
841 ctx->clear_bitmap(region);
842 }
843 region = _regions.next();
844 }
845 }
846 };
847
848 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
849 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
850 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
851
852 ShenandoahHeap* heap = ShenandoahHeap::heap();
853
854 // Compact regular objects first
855 {
856 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
857 ShenandoahCompactObjectsTask compact_task(worker_slices);
858 heap->workers()->run_task(&compact_task);
859 }
860
861 // Compact humongous objects after regular object moves
862 {
863 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
864 compact_humongous_objects();
865 }
866
867 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
868 // and must ensure the bitmap is in sync.
869 {
870 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
871 ShenandoahMCResetCompleteBitmapTask task;
872 heap->workers()->run_task(&task);
873 }
874
875 // Bring regions in proper states after the collection, and set heap properties.
876 {
877 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
878
879 ShenandoahPostCompactClosure post_compact;
880 heap->heap_region_iterate(&post_compact);
881 heap->set_used(post_compact.get_live());
882
883 heap->collection_set()->clear();
884 heap->free_set()->rebuild();
885 }
886
887 heap->clear_cancelled_gc();
888 }