1 /*
2 * Copyright (c) 2014, 2019, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25
26 #include "code/codeCache.hpp"
27 #include "gc/shared/gcTraceTime.inline.hpp"
28 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
29 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
30 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
31 #include "gc/shenandoah/shenandoahFreeSet.hpp"
32 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
33 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
34 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
36 #include "gc/shenandoah/shenandoahHeuristics.hpp"
37 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
39 #include "gc/shenandoah/shenandoahTraversalGC.hpp"
40 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
41 #include "gc/shenandoah/shenandoahUtils.hpp"
42 #include "gc/shenandoah/shenandoahVerifier.hpp"
43 #include "gc/shenandoah/shenandoahVMOperations.hpp"
44 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
45 #include "memory/metaspace.hpp"
46 #include "memory/universe.hpp"
47 #include "oops/compressedOops.inline.hpp"
48 #include "oops/oop.inline.hpp"
49 #include "runtime/thread.hpp"
50 #include "utilities/copy.hpp"
51 #include "utilities/growableArray.hpp"
52 #include "gc/shared/workgroup.hpp"
53
54 void ShenandoahMarkCompact::initialize(GCTimer* gc_timer) {
55 _gc_timer = gc_timer;
56 }
57
58 void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) {
59 ShenandoahHeap* heap = ShenandoahHeap::heap();
60
61 if (ShenandoahVerify) {
62 heap->verifier()->verify_before_fullgc();
63 }
64
65 if (VerifyBeforeGC) {
66 Universe::verify();
67 }
68
69 heap->set_full_gc_in_progress(true);
70
71 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
72 assert(Thread::current()->is_VM_thread(), "Do full GC only while world is stopped");
73
74 {
75 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
76 heap->pre_full_gc_dump(_gc_timer);
77 }
78
79 {
80 ShenandoahGCPhase prepare_phase(ShenandoahPhaseTimings::full_gc_prepare);
81 // Full GC is supposed to recover from any GC state:
82
83 // a0. Remember if we have forwarded objects
84 bool has_forwarded_objects = heap->has_forwarded_objects();
85
86 // a1. Cancel evacuation, if in progress
87 if (heap->is_evacuation_in_progress()) {
88 heap->set_evacuation_in_progress(false);
89 }
90 assert(!heap->is_evacuation_in_progress(), "sanity");
91
92 // a2. Cancel update-refs, if in progress
93 if (heap->is_update_refs_in_progress()) {
94 heap->set_update_refs_in_progress(false);
95 }
96 assert(!heap->is_update_refs_in_progress(), "sanity");
97
98 // a3. Cancel concurrent traversal GC, if in progress
99 if (heap->is_concurrent_traversal_in_progress()) {
100 heap->traversal_gc()->reset();
101 heap->set_concurrent_traversal_in_progress(false);
102 }
103
104 // b. Cancel concurrent mark, if in progress
105 if (heap->is_concurrent_mark_in_progress()) {
106 heap->concurrent_mark()->cancel();
107 heap->stop_concurrent_marking();
108 }
109 assert(!heap->is_concurrent_mark_in_progress(), "sanity");
110
111 // c. Reset the bitmaps for new marking
112 heap->reset_mark_bitmap();
113 assert(heap->marking_context()->is_bitmap_clear(), "sanity");
114 assert(!heap->marking_context()->is_complete(), "sanity");
115
116 // d. Abandon reference discovery and clear all discovered references.
117 ReferenceProcessor* rp = heap->ref_processor();
118 rp->disable_discovery();
119 rp->abandon_partial_discovery();
120 rp->verify_no_references_recorded();
121
122 // e. Set back forwarded objects bit back, in case some steps above dropped it.
123 heap->set_has_forwarded_objects(has_forwarded_objects);
124 }
125
126 heap->make_parsable(true);
127
128 OrderAccess::fence();
129
130 phase1_mark_heap();
131
132 // Once marking is done, which may have fixed up forwarded objects, we can drop it.
133 // Coming out of Full GC, we would not have any forwarded objects.
134 // This also prevents resolves with fwdptr from kicking in while adjusting pointers in phase3.
135 heap->set_has_forwarded_objects(false);
136
137 heap->set_full_gc_move_in_progress(true);
138
139 // Setup workers for the rest
140 OrderAccess::fence();
141
142 // Initialize worker slices
143 ShenandoahHeapRegionSet** worker_slices = NEW_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, heap->max_workers(), mtGC);
144 for (uint i = 0; i < heap->max_workers(); i++) {
145 worker_slices[i] = new ShenandoahHeapRegionSet();
146 }
147
148 {
149 // The rest of code performs region moves, where region status is undefined
150 // until all phases run together.
151 ShenandoahHeapLocker lock(heap->lock());
152
153 phase2_calculate_target_addresses(worker_slices);
154
155 OrderAccess::fence();
156
157 phase3_update_references();
158
159 phase4_compact_objects(worker_slices);
160 }
161
162 // Resize metaspace
163 MetaspaceGC::compute_new_size();
164
165 // Free worker slices
166 for (uint i = 0; i < heap->max_workers(); i++) {
167 delete worker_slices[i];
168 }
169 FREE_C_HEAP_ARRAY(ShenandoahHeapRegionSet*, worker_slices);
170
171 JvmtiExport::gc_epilogue();
172
173 heap->set_full_gc_move_in_progress(false);
174 heap->set_full_gc_in_progress(false);
175
176 if (ShenandoahVerify) {
177 heap->verifier()->verify_after_fullgc();
178 }
179
180 if (VerifyAfterGC) {
181 Universe::verify();
182 }
183
184 {
185 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_heapdumps);
186 heap->post_full_gc_dump(_gc_timer);
187 }
188 }
189
190 class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
191 private:
192 ShenandoahMarkingContext* const _ctx;
193
194 public:
195 ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
196
197 void heap_region_do(ShenandoahHeapRegion *r) {
198 _ctx->capture_top_at_mark_start(r);
199 r->clear_live_data();
200 r->set_concurrent_iteration_safe_limit(r->top());
201 }
202 };
203
204 void ShenandoahMarkCompact::phase1_mark_heap() {
205 GCTraceTime(Info, gc, phases) time("Phase 1: Mark live objects", _gc_timer);
206 ShenandoahGCPhase mark_phase(ShenandoahPhaseTimings::full_gc_mark);
207
208 ShenandoahHeap* heap = ShenandoahHeap::heap();
209
210 ShenandoahPrepareForMarkClosure cl;
211 heap->heap_region_iterate(&cl);
212
213 ShenandoahConcurrentMark* cm = heap->concurrent_mark();
214
215 heap->set_process_references(heap->heuristics()->can_process_references());
216 heap->set_unload_classes(heap->heuristics()->can_unload_classes());
217
218 ReferenceProcessor* rp = heap->ref_processor();
219 // enable ("weak") refs discovery
220 rp->enable_discovery(true /*verify_no_refs*/);
221 rp->setup_policy(true); // forcefully purge all soft references
222 rp->set_active_mt_degree(heap->workers()->active_workers());
223
224 cm->update_roots(ShenandoahPhaseTimings::full_gc_roots);
225 cm->mark_roots(ShenandoahPhaseTimings::full_gc_roots);
226 cm->finish_mark_from_roots(/* full_gc = */ true);
227
228 heap->mark_complete_marking_context();
229 }
230
231 class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
232 private:
233 ShenandoahHeap* const _heap;
234 GrowableArray<ShenandoahHeapRegion*>& _empty_regions;
235 int _empty_regions_pos;
236 ShenandoahHeapRegion* _to_region;
237 ShenandoahHeapRegion* _from_region;
238 HeapWord* _compact_point;
239
240 public:
241 ShenandoahPrepareForCompactionObjectClosure(GrowableArray<ShenandoahHeapRegion*>& empty_regions, ShenandoahHeapRegion* to_region) :
242 _heap(ShenandoahHeap::heap()),
243 _empty_regions(empty_regions),
244 _empty_regions_pos(0),
245 _to_region(to_region),
246 _from_region(NULL),
247 _compact_point(to_region->bottom()) {}
248
249 void set_from_region(ShenandoahHeapRegion* from_region) {
250 _from_region = from_region;
251 }
252
253 void finish_region() {
254 assert(_to_region != NULL, "should not happen");
255 _to_region->set_new_top(_compact_point);
256 }
257
258 bool is_compact_same_region() {
259 return _from_region == _to_region;
260 }
261
262 int empty_regions_pos() {
263 return _empty_regions_pos;
264 }
265
266 void do_object(oop p) {
267 assert(_from_region != NULL, "must set before work");
268 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
269 assert(!_heap->complete_marking_context()->allocated_after_mark_start((HeapWord*) p), "must be truly marked");
270
271 size_t obj_size = p->size() + ShenandoahForwarding::word_size();
272 if (_compact_point + obj_size > _to_region->end()) {
273 finish_region();
274
275 // Object doesn't fit. Pick next empty region and start compacting there.
276 ShenandoahHeapRegion* new_to_region;
277 if (_empty_regions_pos < _empty_regions.length()) {
278 new_to_region = _empty_regions.at(_empty_regions_pos);
279 _empty_regions_pos++;
280 } else {
281 // Out of empty region? Compact within the same region.
282 new_to_region = _from_region;
283 }
284
285 assert(new_to_region != _to_region, "must not reuse same to-region");
286 assert(new_to_region != NULL, "must not be NULL");
287 _to_region = new_to_region;
288 _compact_point = _to_region->bottom();
289 }
290
291 // Object fits into current region, record new location:
292 assert(_compact_point + obj_size <= _to_region->end(), "must fit");
293 shenandoah_assert_not_forwarded(NULL, p);
294 ShenandoahForwarding::set_forwardee_raw(p, _compact_point + ShenandoahForwarding::word_size());
295 _compact_point += obj_size;
296 }
297 };
298
299 class ShenandoahPrepareForCompactionTask : public AbstractGangTask {
300 private:
301 ShenandoahHeap* const _heap;
302 ShenandoahHeapRegionSet** const _worker_slices;
303 ShenandoahRegionIterator _heap_regions;
304
305 ShenandoahHeapRegion* next_from_region(ShenandoahHeapRegionSet* slice) {
306 ShenandoahHeapRegion* from_region = _heap_regions.next();
307
308 while (from_region != NULL && (!from_region->is_move_allowed() || from_region->is_humongous())) {
309 from_region = _heap_regions.next();
310 }
311
312 if (from_region != NULL) {
313 assert(slice != NULL, "sanity");
314 assert(!from_region->is_humongous(), "this path cannot handle humongous regions");
315 assert(from_region->is_move_allowed(), "only regions that can be moved in mark-compact");
316 slice->add_region(from_region);
317 }
318
319 return from_region;
320 }
321
322 public:
323 ShenandoahPrepareForCompactionTask(ShenandoahHeapRegionSet** worker_slices) :
324 AbstractGangTask("Shenandoah Prepare For Compaction Task"),
325 _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) {
326 }
327
328 void work(uint worker_id) {
329 ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
330 ShenandoahHeapRegion* from_region = next_from_region(slice);
331 // No work?
332 if (from_region == NULL) {
333 return;
334 }
335
336 // Sliding compaction. Walk all regions in the slice, and compact them.
337 // Remember empty regions and reuse them as needed.
338 ResourceMark rm;
339 GrowableArray<ShenandoahHeapRegion*> empty_regions((int)_heap->num_regions());
340 ShenandoahPrepareForCompactionObjectClosure cl(empty_regions, from_region);
341 while (from_region != NULL) {
342 cl.set_from_region(from_region);
343 if (from_region->has_live()) {
344 _heap->marked_object_iterate(from_region, &cl);
345 }
346
347 // Compacted the region to somewhere else? From-region is empty then.
348 if (!cl.is_compact_same_region()) {
349 empty_regions.append(from_region);
350 }
351 from_region = next_from_region(slice);
352 }
353 cl.finish_region();
354
355 // Mark all remaining regions as empty
356 for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
357 ShenandoahHeapRegion* r = empty_regions.at(pos);
358 r->set_new_top(r->bottom());
359 }
360 }
361 };
362
363 void ShenandoahMarkCompact::calculate_target_humongous_objects() {
364 ShenandoahHeap* heap = ShenandoahHeap::heap();
365
366 // Compute the new addresses for humongous objects. We need to do this after addresses
367 // for regular objects are calculated, and we know what regions in heap suffix are
368 // available for humongous moves.
369 //
370 // Scan the heap backwards, because we are compacting humongous regions towards the end.
371 // Maintain the contiguous compaction window in [to_begin; to_end), so that we can slide
372 // humongous start there.
373 //
374 // The complication is potential non-movable regions during the scan. If such region is
375 // detected, then sliding restarts towards that non-movable region.
376
377 size_t to_begin = heap->num_regions();
378 size_t to_end = heap->num_regions();
379
380 for (size_t c = heap->num_regions() - 1; c > 0; c--) {
381 ShenandoahHeapRegion *r = heap->get_region(c);
382 if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) {
383 // To-region candidate: record this, and continue scan
384 to_begin = r->region_number();
385 continue;
386 }
387
388 if (r->is_humongous_start() && r->is_move_allowed()) {
389 // From-region candidate: movable humongous region
390 oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
391 size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
392 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
393
394 size_t start = to_end - num_regions;
395
396 if (start >= to_begin && start != r->region_number()) {
397 // Fits into current window, and the move is non-trivial. Record the move then, and continue scan.
398 ShenandoahForwarding::set_forwardee_raw(old_obj, heap->get_region(start)->bottom() + ShenandoahForwarding::word_size());
399 to_end = start;
400 continue;
401 }
402 }
403
404 // Failed to fit. Scan starting from current region.
405 to_begin = r->region_number();
406 to_end = r->region_number();
407 }
408 }
409
410 class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure {
411 private:
412 ShenandoahHeap* const _heap;
413
414 public:
415 ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {}
416 void heap_region_do(ShenandoahHeapRegion* r) {
417 if (r->is_trash()) {
418 r->recycle();
419 }
420 if (r->is_cset()) {
421 r->make_regular_bypass();
422 }
423 if (r->is_empty_uncommitted()) {
424 r->make_committed_bypass();
425 }
426 assert (r->is_committed(), "only committed regions in heap now, see region " SIZE_FORMAT, r->region_number());
427
428 // Record current region occupancy: this communicates empty regions are free
429 // to the rest of Full GC code.
430 r->set_new_top(r->top());
431 }
432 };
433
434 class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure {
435 private:
436 ShenandoahHeap* const _heap;
437 ShenandoahMarkingContext* const _ctx;
438
439 public:
440 ShenandoahTrashImmediateGarbageClosure() :
441 _heap(ShenandoahHeap::heap()),
442 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
443
444 void heap_region_do(ShenandoahHeapRegion* r) {
445 if (r->is_humongous_start()) {
446 oop humongous_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
447 if (!_ctx->is_marked(humongous_obj)) {
448 assert(!r->has_live(),
449 "Region " SIZE_FORMAT " is not marked, should not have live", r->region_number());
450 _heap->trash_humongous_region_at(r);
451 } else {
452 assert(r->has_live(),
453 "Region " SIZE_FORMAT " should have live", r->region_number());
454 }
455 } else if (r->is_humongous_continuation()) {
456 // If we hit continuation, the non-live humongous starts should have been trashed already
457 assert(r->humongous_start_region()->has_live(),
458 "Region " SIZE_FORMAT " should have live", r->region_number());
459 } else if (r->is_regular()) {
460 if (!r->has_live()) {
461 r->make_trash_immediate();
462 }
463 }
464 }
465 };
466
467 void ShenandoahMarkCompact::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) {
468 GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer);
469 ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses);
470
471 ShenandoahHeap* heap = ShenandoahHeap::heap();
472
473 {
474 // Trash the immediately collectible regions before computing addresses
475 ShenandoahTrashImmediateGarbageClosure tigcl;
476 heap->heap_region_iterate(&tigcl);
477
478 // Make sure regions are in good state: committed, active, clean.
479 // This is needed because we are potentially sliding the data through them.
480 ShenandoahEnsureHeapActiveClosure ecl;
481 heap->heap_region_iterate(&ecl);
482 }
483
484 // Compute the new addresses for regular objects
485 {
486 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular);
487 ShenandoahPrepareForCompactionTask prepare_task(worker_slices);
488 heap->workers()->run_task(&prepare_task);
489 }
490
491 // Compute the new addresses for humongous objects
492 {
493 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_humong);
494 calculate_target_humongous_objects();
495 }
496 }
497
498 class ShenandoahAdjustPointersClosure : public MetadataVisitingOopIterateClosure {
499 private:
500 ShenandoahHeap* const _heap;
501 ShenandoahMarkingContext* const _ctx;
502
503 template <class T>
504 inline void do_oop_work(T* p) {
505 T o = RawAccess<>::oop_load(p);
506 if (!CompressedOops::is_null(o)) {
507 oop obj = CompressedOops::decode_not_null(o);
508 assert(_ctx->is_marked(obj), "must be marked");
509 oop forw = oop(ShenandoahForwarding::get_forwardee_raw(obj));
510 RawAccess<IS_NOT_NULL>::oop_store(p, forw);
511 }
512 }
513
514 public:
515 ShenandoahAdjustPointersClosure() :
516 _heap(ShenandoahHeap::heap()),
517 _ctx(ShenandoahHeap::heap()->complete_marking_context()) {}
518
519 void do_oop(oop* p) { do_oop_work(p); }
520 void do_oop(narrowOop* p) { do_oop_work(p); }
521 };
522
523 class ShenandoahAdjustPointersObjectClosure : public ObjectClosure {
524 private:
525 ShenandoahHeap* const _heap;
526 ShenandoahAdjustPointersClosure _cl;
527
528 public:
529 ShenandoahAdjustPointersObjectClosure() :
530 _heap(ShenandoahHeap::heap()) {
531 }
532 void do_object(oop p) {
533 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
534 HeapWord* forw = ShenandoahForwarding::get_forwardee_raw(p);
535 p->oop_iterate(&_cl);
536 }
537 };
538
539 class ShenandoahAdjustPointersTask : public AbstractGangTask {
540 private:
541 ShenandoahHeap* const _heap;
542 ShenandoahRegionIterator _regions;
543
544 public:
545 ShenandoahAdjustPointersTask() :
546 AbstractGangTask("Shenandoah Adjust Pointers Task"),
547 _heap(ShenandoahHeap::heap()) {
548 }
549
550 void work(uint worker_id) {
551 ShenandoahAdjustPointersObjectClosure obj_cl;
552 ShenandoahHeapRegion* r = _regions.next();
553 while (r != NULL) {
554 if (!r->is_humongous_continuation() && r->has_live()) {
555 _heap->marked_object_iterate(r, &obj_cl);
556 }
557 r = _regions.next();
558 }
559 }
560 };
561
562 class ShenandoahAdjustRootPointersTask : public AbstractGangTask {
563 private:
564 ShenandoahRootAdjuster* _rp;
565
566 public:
567 ShenandoahAdjustRootPointersTask(ShenandoahRootAdjuster* rp) :
568 AbstractGangTask("Shenandoah Adjust Root Pointers Task"),
569 _rp(rp) {}
570
571 void work(uint worker_id) {
572 ShenandoahAdjustPointersClosure cl;
573 _rp->roots_do(worker_id, &cl);
574 }
575 };
576
577 void ShenandoahMarkCompact::phase3_update_references() {
578 GCTraceTime(Info, gc, phases) time("Phase 3: Adjust pointers", _gc_timer);
579 ShenandoahGCPhase adjust_pointer_phase(ShenandoahPhaseTimings::full_gc_adjust_pointers);
580
581 ShenandoahHeap* heap = ShenandoahHeap::heap();
582
583 WorkGang* workers = heap->workers();
584 uint nworkers = workers->active_workers();
585 {
586 #if COMPILER2_OR_JVMCI
587 DerivedPointerTable::clear();
588 #endif
589 ShenandoahRootAdjuster rp(nworkers, ShenandoahPhaseTimings::full_gc_roots);
590 ShenandoahAdjustRootPointersTask task(&rp);
591 workers->run_task(&task);
592 #if COMPILER2_OR_JVMCI
593 DerivedPointerTable::update_pointers();
594 #endif
595 }
596
597 ShenandoahAdjustPointersTask adjust_pointers_task;
598 workers->run_task(&adjust_pointers_task);
599 }
600
601 class ShenandoahCompactObjectsClosure : public ObjectClosure {
602 private:
603 ShenandoahHeap* const _heap;
604 uint const _worker_id;
605
606 public:
607 ShenandoahCompactObjectsClosure(uint worker_id) :
608 _heap(ShenandoahHeap::heap()), _worker_id(worker_id) {}
609
610 void do_object(oop p) {
611 assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
612 size_t size = (size_t)p->size();
613 HeapWord* compact_to = ShenandoahForwarding::get_forwardee_raw(p);
614 HeapWord* compact_from = (HeapWord*) p;
615 if (compact_from != compact_to) {
616 Copy::aligned_conjoint_words(compact_from, compact_to, size);
617 }
618 oop new_obj = oop(compact_to);
619 ShenandoahForwarding::initialize(new_obj);
620 }
621 };
622
623 class ShenandoahCompactObjectsTask : public AbstractGangTask {
624 private:
625 ShenandoahHeap* const _heap;
626 ShenandoahHeapRegionSet** const _worker_slices;
627
628 public:
629 ShenandoahCompactObjectsTask(ShenandoahHeapRegionSet** worker_slices) :
630 AbstractGangTask("Shenandoah Compact Objects Task"),
631 _heap(ShenandoahHeap::heap()),
632 _worker_slices(worker_slices) {
633 }
634
635 void work(uint worker_id) {
636 ShenandoahHeapRegionSetIterator slice(_worker_slices[worker_id]);
637
638 ShenandoahCompactObjectsClosure cl(worker_id);
639 ShenandoahHeapRegion* r = slice.next();
640 while (r != NULL) {
641 assert(!r->is_humongous(), "must not get humongous regions here");
642 if (r->has_live()) {
643 _heap->marked_object_iterate(r, &cl);
644 }
645 r->set_top(r->new_top());
646 r = slice.next();
647 }
648 }
649 };
650
651 class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
652 private:
653 ShenandoahHeap* const _heap;
654 size_t _live;
655
656 public:
657 ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) {
658 _heap->free_set()->clear();
659 }
660
661 void heap_region_do(ShenandoahHeapRegion* r) {
662 assert (!r->is_cset(), "cset regions should have been demoted already");
663
664 // Need to reset the complete-top-at-mark-start pointer here because
665 // the complete marking bitmap is no longer valid. This ensures
666 // size-based iteration in marked_object_iterate().
667 // NOTE: See blurb at ShenandoahMCResetCompleteBitmapTask on why we need to skip
668 // pinned regions.
669 if (!r->is_pinned()) {
670 _heap->complete_marking_context()->reset_top_at_mark_start(r);
671 }
672
673 size_t live = r->used();
674
675 // Make empty regions that have been allocated into regular
676 if (r->is_empty() && live > 0) {
677 r->make_regular_bypass();
678 }
679
680 // Reclaim regular regions that became empty
681 if (r->is_regular() && live == 0) {
682 r->make_trash();
683 }
684
685 // Recycle all trash regions
686 if (r->is_trash()) {
687 live = 0;
688 r->recycle();
689 }
690
691 r->set_live_data(live);
692 r->reset_alloc_metadata_to_shared();
693 _live += live;
694 }
695
696 size_t get_live() {
697 return _live;
698 }
699 };
700
701 void ShenandoahMarkCompact::compact_humongous_objects() {
702 // Compact humongous regions, based on their fwdptr objects.
703 //
704 // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
705 // humongous regions are already compacted, and do not require further moves, which alleviates
706 // sliding costs. We may consider doing this in parallel in future.
707
708 ShenandoahHeap* heap = ShenandoahHeap::heap();
709
710 for (size_t c = heap->num_regions() - 1; c > 0; c--) {
711 ShenandoahHeapRegion* r = heap->get_region(c);
712 if (r->is_humongous_start()) {
713 oop old_obj = oop(r->bottom() + ShenandoahForwarding::word_size());
714 size_t words_size = old_obj->size() + ShenandoahForwarding::word_size();
715 size_t num_regions = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize);
716
717 size_t old_start = r->region_number();
718 size_t old_end = old_start + num_regions - 1;
719 size_t new_start = heap->heap_region_index_containing(ShenandoahForwarding::get_forwardee_raw(old_obj));
720 size_t new_end = new_start + num_regions - 1;
721
722 if (old_start == new_start) {
723 // No need to move the object, it stays at the same slot
724 continue;
725 }
726
727 assert (r->is_move_allowed(), "should be movable");
728
729 Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(),
730 heap->get_region(new_start)->bottom(),
731 ShenandoahHeapRegion::region_size_words()*num_regions);
732
733 oop new_obj = oop(heap->get_region(new_start)->bottom() + ShenandoahForwarding::word_size());
734 ShenandoahForwarding::initialize(new_obj);
735
736 {
737 for (size_t c = old_start; c <= old_end; c++) {
738 ShenandoahHeapRegion* r = heap->get_region(c);
739 r->make_regular_bypass();
740 r->set_top(r->bottom());
741 }
742
743 for (size_t c = new_start; c <= new_end; c++) {
744 ShenandoahHeapRegion* r = heap->get_region(c);
745 if (c == new_start) {
746 r->make_humongous_start_bypass();
747 } else {
748 r->make_humongous_cont_bypass();
749 }
750
751 // Trailing region may be non-full, record the remainder there
752 size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask();
753 if ((c == new_end) && (remainder != 0)) {
754 r->set_top(r->bottom() + remainder);
755 } else {
756 r->set_top(r->end());
757 }
758
759 r->reset_alloc_metadata_to_shared();
760 }
761 }
762 }
763 }
764 }
765
766 // This is slightly different to ShHeap::reset_next_mark_bitmap:
767 // we need to remain able to walk pinned regions.
768 // Since pinned region do not move and don't get compacted, we will get holes with
769 // unreachable objects in them (which may have pointers to unloaded Klasses and thus
770 // cannot be iterated over using oop->size(). The only way to safely iterate over those is using
771 // a valid marking bitmap and valid TAMS pointer. This class only resets marking
772 // bitmaps for un-pinned regions, and later we only reset TAMS for unpinned regions.
773 class ShenandoahMCResetCompleteBitmapTask : public AbstractGangTask {
774 private:
775 ShenandoahRegionIterator _regions;
776
777 public:
778 ShenandoahMCResetCompleteBitmapTask() :
779 AbstractGangTask("Parallel Reset Bitmap Task") {
780 }
781
782 void work(uint worker_id) {
783 ShenandoahHeapRegion* region = _regions.next();
784 ShenandoahHeap* heap = ShenandoahHeap::heap();
785 ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
786 while (region != NULL) {
787 if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
788 ctx->clear_bitmap(region);
789 }
790 region = _regions.next();
791 }
792 }
793 };
794
795 void ShenandoahMarkCompact::phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices) {
796 GCTraceTime(Info, gc, phases) time("Phase 4: Move objects", _gc_timer);
797 ShenandoahGCPhase compaction_phase(ShenandoahPhaseTimings::full_gc_copy_objects);
798
799 ShenandoahHeap* heap = ShenandoahHeap::heap();
800
801 // Compact regular objects first
802 {
803 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_regular);
804 ShenandoahCompactObjectsTask compact_task(worker_slices);
805 heap->workers()->run_task(&compact_task);
806 }
807
808 // Compact humongous objects after regular object moves
809 {
810 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
811 compact_humongous_objects();
812 }
813
814 // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
815 // and must ensure the bitmap is in sync.
816 {
817 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_reset_complete);
818 ShenandoahMCResetCompleteBitmapTask task;
819 heap->workers()->run_task(&task);
820 }
821
822 // Bring regions in proper states after the collection, and set heap properties.
823 {
824 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
825
826 ShenandoahPostCompactClosure post_compact;
827 heap->heap_region_iterate(&post_compact);
828 heap->set_used(post_compact.get_live());
829
830 heap->collection_set()->clear();
831 heap->free_set()->rebuild();
832 }
833
834 heap->clear_cancelled_gc();
835 }