21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "memory/universe.hpp"
27
28 #include "gc/shared/gcArguments.hpp"
29 #include "gc/shared/gcTimer.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/memAllocator.hpp"
32 #include "gc/shared/parallelCleaning.hpp"
33 #include "gc/shared/plab.hpp"
34
35 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahControlThread.hpp"
42 #include "gc/shenandoah/shenandoahFreeSet.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
47 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
48 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
49 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
50 #include "gc/shenandoah/shenandoahMetrics.hpp"
51 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
52 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
53 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
54 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
55 #include "gc/shenandoah/shenandoahStringDedup.hpp"
56 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
57 #include "gc/shenandoah/shenandoahUtils.hpp"
58 #include "gc/shenandoah/shenandoahVerifier.hpp"
59 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
60 #include "gc/shenandoah/shenandoahVMOperations.hpp"
1054
1055 public:
1056 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1057 AbstractGangTask("Shenandoah evacuate and update roots"),
1058 _rp(rp) {}
1059
1060 void work(uint worker_id) {
1061 ShenandoahParallelWorkerSession worker_session(worker_id);
1062 ShenandoahEvacOOMScope oom_evac_scope;
1063 ShenandoahEvacuateUpdateRootsClosure cl;
1064 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1065 _rp->roots_do(worker_id, &cl);
1066 }
1067 };
1068
1069 void ShenandoahHeap::evacuate_and_update_roots() {
1070 #if COMPILER2_OR_JVMCI
1071 DerivedPointerTable::clear();
1072 #endif
1073 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1074
1075 {
1076 ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
1077 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1078 workers()->run_task(&roots_task);
1079 }
1080
1081 #if COMPILER2_OR_JVMCI
1082 DerivedPointerTable::update_pointers();
1083 #endif
1084 }
1085
1086 // Returns size in bytes
1087 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1088 if (ShenandoahElasticTLAB) {
1089 // With Elastic TLABs, return the max allowed size, and let the allocation path
1090 // figure out the safe size for current allocation.
1091 return ShenandoahHeapRegion::max_tlab_size_bytes();
1092 } else {
1093 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1094 }
1095 }
1096
1500 // If collection set has candidates, start evacuation.
1501 // Otherwise, bypass the rest of the cycle.
1502 if (!collection_set()->is_empty()) {
1503 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1504
1505 if (ShenandoahVerify) {
1506 verifier()->verify_before_evacuation();
1507 }
1508
1509 set_evacuation_in_progress(true);
1510 // From here on, we need to update references.
1511 set_has_forwarded_objects(true);
1512
1513 evacuate_and_update_roots();
1514
1515 if (ShenandoahPacing) {
1516 pacer()->setup_for_evac();
1517 }
1518
1519 if (ShenandoahVerify) {
1520 verifier()->verify_roots_no_forwarded();
1521 verifier()->verify_during_evacuation();
1522 }
1523 } else {
1524 if (ShenandoahVerify) {
1525 verifier()->verify_after_concmark();
1526 }
1527
1528 if (VerifyAfterGC) {
1529 Universe::verify();
1530 }
1531 }
1532
1533 } else {
1534 concurrent_mark()->cancel();
1535 stop_concurrent_marking();
1536
1537 if (process_references()) {
1538 // Abandon reference processing right away: pre-cleaning must have failed.
1539 ReferenceProcessor *rp = ref_processor();
1540 rp->disable_discovery();
1561 }
1562
1563 void ShenandoahHeap::op_conc_evac() {
1564 ShenandoahEvacuationTask task(this, _collection_set, true);
1565 workers()->run_task(&task);
1566 }
1567
1568 void ShenandoahHeap::op_stw_evac() {
1569 ShenandoahEvacuationTask task(this, _collection_set, false);
1570 workers()->run_task(&task);
1571 }
1572
1573 void ShenandoahHeap::op_updaterefs() {
1574 update_heap_references(true);
1575 }
1576
1577 void ShenandoahHeap::op_cleanup() {
1578 free_set()->recycle_trash();
1579 }
1580
1581 void ShenandoahHeap::op_reset() {
1582 reset_mark_bitmap();
1583 }
1584
1585 void ShenandoahHeap::op_preclean() {
1586 concurrent_mark()->preclean_weak_refs();
1587 }
1588
1589 void ShenandoahHeap::op_init_traversal() {
1590 traversal_gc()->init_traversal_collection();
1591 }
1592
1593 void ShenandoahHeap::op_traversal() {
1594 traversal_gc()->concurrent_traversal_collection();
1595 }
1596
1597 void ShenandoahHeap::op_final_traversal() {
1598 traversal_gc()->final_traversal_collection();
1599 }
1600
2542 "concurrent evacuation");
2543
2544 try_inject_alloc_failure();
2545 op_conc_evac();
2546 }
2547
2548 void ShenandoahHeap::entry_updaterefs() {
2549 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2550
2551 static const char* msg = "Concurrent update references";
2552 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2553 EventMark em("%s", msg);
2554
2555 ShenandoahWorkerScope scope(workers(),
2556 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2557 "concurrent reference update");
2558
2559 try_inject_alloc_failure();
2560 op_updaterefs();
2561 }
2562 void ShenandoahHeap::entry_cleanup() {
2563 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2564
2565 static const char* msg = "Concurrent cleanup";
2566 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2567 EventMark em("%s", msg);
2568
2569 // This phase does not use workers, no need for setup
2570
2571 try_inject_alloc_failure();
2572 op_cleanup();
2573 }
2574
2575 void ShenandoahHeap::entry_reset() {
2576 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2577
2578 static const char* msg = "Concurrent reset";
2579 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2580 EventMark em("%s", msg);
2581
|
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "memory/universe.hpp"
27
28 #include "gc/shared/gcArguments.hpp"
29 #include "gc/shared/gcTimer.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/memAllocator.hpp"
32 #include "gc/shared/parallelCleaning.hpp"
33 #include "gc/shared/plab.hpp"
34
35 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
42 #include "gc/shenandoah/shenandoahControlThread.hpp"
43 #include "gc/shenandoah/shenandoahFreeSet.hpp"
44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
45 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
48 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
49 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
50 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
51 #include "gc/shenandoah/shenandoahMetrics.hpp"
52 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
53 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
54 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
55 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
56 #include "gc/shenandoah/shenandoahStringDedup.hpp"
57 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
58 #include "gc/shenandoah/shenandoahUtils.hpp"
59 #include "gc/shenandoah/shenandoahVerifier.hpp"
60 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
61 #include "gc/shenandoah/shenandoahVMOperations.hpp"
1055
1056 public:
1057 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1058 AbstractGangTask("Shenandoah evacuate and update roots"),
1059 _rp(rp) {}
1060
1061 void work(uint worker_id) {
1062 ShenandoahParallelWorkerSession worker_session(worker_id);
1063 ShenandoahEvacOOMScope oom_evac_scope;
1064 ShenandoahEvacuateUpdateRootsClosure cl;
1065 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1066 _rp->roots_do(worker_id, &cl);
1067 }
1068 };
1069
1070 void ShenandoahHeap::evacuate_and_update_roots() {
1071 #if COMPILER2_OR_JVMCI
1072 DerivedPointerTable::clear();
1073 #endif
1074 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1075 {
1076 // Include concurrent roots if current cycle can not process those roots concurrently
1077 ShenandoahRootEvacuator rp(workers()->active_workers(),
1078 ShenandoahPhaseTimings::init_evac,
1079 !ShenandoahConcurrentRoots::should_do_concurrent_roots());
1080 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1081 workers()->run_task(&roots_task);
1082 }
1083
1084 #if COMPILER2_OR_JVMCI
1085 DerivedPointerTable::update_pointers();
1086 #endif
1087 }
1088
1089 // Returns size in bytes
1090 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1091 if (ShenandoahElasticTLAB) {
1092 // With Elastic TLABs, return the max allowed size, and let the allocation path
1093 // figure out the safe size for current allocation.
1094 return ShenandoahHeapRegion::max_tlab_size_bytes();
1095 } else {
1096 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1097 }
1098 }
1099
1503 // If collection set has candidates, start evacuation.
1504 // Otherwise, bypass the rest of the cycle.
1505 if (!collection_set()->is_empty()) {
1506 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1507
1508 if (ShenandoahVerify) {
1509 verifier()->verify_before_evacuation();
1510 }
1511
1512 set_evacuation_in_progress(true);
1513 // From here on, we need to update references.
1514 set_has_forwarded_objects(true);
1515
1516 evacuate_and_update_roots();
1517
1518 if (ShenandoahPacing) {
1519 pacer()->setup_for_evac();
1520 }
1521
1522 if (ShenandoahVerify) {
1523 if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1524 verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::JNIHandleRoots);
1525 } else {
1526 verifier()->verify_roots_no_forwarded();
1527 }
1528 verifier()->verify_during_evacuation();
1529 }
1530 } else {
1531 if (ShenandoahVerify) {
1532 verifier()->verify_after_concmark();
1533 }
1534
1535 if (VerifyAfterGC) {
1536 Universe::verify();
1537 }
1538 }
1539
1540 } else {
1541 concurrent_mark()->cancel();
1542 stop_concurrent_marking();
1543
1544 if (process_references()) {
1545 // Abandon reference processing right away: pre-cleaning must have failed.
1546 ReferenceProcessor *rp = ref_processor();
1547 rp->disable_discovery();
1568 }
1569
1570 void ShenandoahHeap::op_conc_evac() {
1571 ShenandoahEvacuationTask task(this, _collection_set, true);
1572 workers()->run_task(&task);
1573 }
1574
1575 void ShenandoahHeap::op_stw_evac() {
1576 ShenandoahEvacuationTask task(this, _collection_set, false);
1577 workers()->run_task(&task);
1578 }
1579
1580 void ShenandoahHeap::op_updaterefs() {
1581 update_heap_references(true);
1582 }
1583
1584 void ShenandoahHeap::op_cleanup() {
1585 free_set()->recycle_trash();
1586 }
1587
1588 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1589 private:
1590 ShenandoahJNIHandleRoots<true /*concurrent*/> _jni_roots;
1591
1592 public:
1593 ShenandoahConcurrentRootsEvacUpdateTask() :
1594 AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") {
1595 }
1596
1597 void work(uint worker_id) {
1598 ShenandoahEvacOOMScope oom;
1599 ShenandoahEvacuateUpdateRootsClosure cl;
1600 _jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
1601 }
1602 };
1603
1604 void ShenandoahHeap::op_roots() {
1605 if (is_evacuation_in_progress() &&
1606 ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1607 ShenandoahConcurrentRootsEvacUpdateTask task;
1608 workers()->run_task(&task);
1609 }
1610 }
1611
1612 void ShenandoahHeap::op_reset() {
1613 reset_mark_bitmap();
1614 }
1615
1616 void ShenandoahHeap::op_preclean() {
1617 concurrent_mark()->preclean_weak_refs();
1618 }
1619
1620 void ShenandoahHeap::op_init_traversal() {
1621 traversal_gc()->init_traversal_collection();
1622 }
1623
1624 void ShenandoahHeap::op_traversal() {
1625 traversal_gc()->concurrent_traversal_collection();
1626 }
1627
1628 void ShenandoahHeap::op_final_traversal() {
1629 traversal_gc()->final_traversal_collection();
1630 }
1631
2573 "concurrent evacuation");
2574
2575 try_inject_alloc_failure();
2576 op_conc_evac();
2577 }
2578
2579 void ShenandoahHeap::entry_updaterefs() {
2580 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2581
2582 static const char* msg = "Concurrent update references";
2583 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2584 EventMark em("%s", msg);
2585
2586 ShenandoahWorkerScope scope(workers(),
2587 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2588 "concurrent reference update");
2589
2590 try_inject_alloc_failure();
2591 op_updaterefs();
2592 }
2593
2594 void ShenandoahHeap::entry_roots() {
2595 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
2596
2597 static const char* msg = "Concurrent roots processing";
2598 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2599 EventMark em("%s", msg);
2600
2601 ShenandoahWorkerScope scope(workers(),
2602 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2603 "concurrent root processing");
2604
2605 try_inject_alloc_failure();
2606 op_roots();
2607 }
2608
2609 void ShenandoahHeap::entry_cleanup() {
2610 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2611
2612 static const char* msg = "Concurrent cleanup";
2613 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2614 EventMark em("%s", msg);
2615
2616 // This phase does not use workers, no need for setup
2617
2618 try_inject_alloc_failure();
2619 op_cleanup();
2620 }
2621
2622 void ShenandoahHeap::entry_reset() {
2623 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2624
2625 static const char* msg = "Concurrent reset";
2626 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2627 EventMark em("%s", msg);
2628
|