21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "memory/universe.hpp"
27
28 #include "gc/shared/gcArguments.hpp"
29 #include "gc/shared/gcTimer.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/memAllocator.hpp"
32 #include "gc/shared/parallelCleaning.hpp"
33 #include "gc/shared/plab.hpp"
34
35 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahControlThread.hpp"
42 #include "gc/shenandoah/shenandoahFreeSet.hpp"
43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
44 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
47 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
48 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
49 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
50 #include "gc/shenandoah/shenandoahMetrics.hpp"
51 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
52 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
53 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
54 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
55 #include "gc/shenandoah/shenandoahStringDedup.hpp"
56 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
57 #include "gc/shenandoah/shenandoahUtils.hpp"
58 #include "gc/shenandoah/shenandoahVerifier.hpp"
59 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
60 #include "gc/shenandoah/shenandoahVMOperations.hpp"
1054
1055 public:
1056 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1057 AbstractGangTask("Shenandoah evacuate and update roots"),
1058 _rp(rp) {}
1059
1060 void work(uint worker_id) {
1061 ShenandoahParallelWorkerSession worker_session(worker_id);
1062 ShenandoahEvacOOMScope oom_evac_scope;
1063 ShenandoahEvacuateUpdateRootsClosure cl;
1064 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1065 _rp->roots_do(worker_id, &cl);
1066 }
1067 };
1068
1069 void ShenandoahHeap::evacuate_and_update_roots() {
1070 #if COMPILER2_OR_JVMCI
1071 DerivedPointerTable::clear();
1072 #endif
1073 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1074
1075 {
1076 ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
1077 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1078 workers()->run_task(&roots_task);
1079 }
1080
1081 #if COMPILER2_OR_JVMCI
1082 DerivedPointerTable::update_pointers();
1083 #endif
1084 }
1085
1086 // Returns size in bytes
1087 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1088 if (ShenandoahElasticTLAB) {
1089 // With Elastic TLABs, return the max allowed size, and let the allocation path
1090 // figure out the safe size for current allocation.
1091 return ShenandoahHeapRegion::max_tlab_size_bytes();
1092 } else {
1093 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1094 }
1095 }
1096
1500 // If collection set has candidates, start evacuation.
1501 // Otherwise, bypass the rest of the cycle.
1502 if (!collection_set()->is_empty()) {
1503 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1504
1505 if (ShenandoahVerify) {
1506 verifier()->verify_before_evacuation();
1507 }
1508
1509 set_evacuation_in_progress(true);
1510 // From here on, we need to update references.
1511 set_has_forwarded_objects(true);
1512
1513 evacuate_and_update_roots();
1514
1515 if (ShenandoahPacing) {
1516 pacer()->setup_for_evac();
1517 }
1518
1519 if (ShenandoahVerify) {
1520 verifier()->verify_roots_no_forwarded();
1521 verifier()->verify_during_evacuation();
1522 }
1523 } else {
1524 if (ShenandoahVerify) {
1525 verifier()->verify_after_concmark();
1526 }
1527
1528 if (VerifyAfterGC) {
1529 Universe::verify();
1530 }
1531 }
1532
1533 } else {
1534 concurrent_mark()->cancel();
1535 stop_concurrent_marking();
1536
1537 if (process_references()) {
1538 // Abandon reference processing right away: pre-cleaning must have failed.
1539 ReferenceProcessor *rp = ref_processor();
1540 rp->disable_discovery();
1561 }
1562
1563 void ShenandoahHeap::op_conc_evac() {
1564 ShenandoahEvacuationTask task(this, _collection_set, true);
1565 workers()->run_task(&task);
1566 }
1567
1568 void ShenandoahHeap::op_stw_evac() {
1569 ShenandoahEvacuationTask task(this, _collection_set, false);
1570 workers()->run_task(&task);
1571 }
1572
1573 void ShenandoahHeap::op_updaterefs() {
1574 update_heap_references(true);
1575 }
1576
1577 void ShenandoahHeap::op_cleanup() {
1578 free_set()->recycle_trash();
1579 }
1580
1581 void ShenandoahHeap::op_reset() {
1582 reset_mark_bitmap();
1583 }
1584
1585 void ShenandoahHeap::op_preclean() {
1586 concurrent_mark()->preclean_weak_refs();
1587 }
1588
1589 void ShenandoahHeap::op_init_traversal() {
1590 traversal_gc()->init_traversal_collection();
1591 }
1592
1593 void ShenandoahHeap::op_traversal() {
1594 traversal_gc()->concurrent_traversal_collection();
1595 }
1596
1597 void ShenandoahHeap::op_final_traversal() {
1598 traversal_gc()->final_traversal_collection();
1599 }
1600
2537 "concurrent evacuation");
2538
2539 try_inject_alloc_failure();
2540 op_conc_evac();
2541 }
2542
2543 void ShenandoahHeap::entry_updaterefs() {
2544 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2545
2546 static const char* msg = "Concurrent update references";
2547 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2548 EventMark em("%s", msg);
2549
2550 ShenandoahWorkerScope scope(workers(),
2551 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2552 "concurrent reference update");
2553
2554 try_inject_alloc_failure();
2555 op_updaterefs();
2556 }
2557 void ShenandoahHeap::entry_cleanup() {
2558 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2559
2560 static const char* msg = "Concurrent cleanup";
2561 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2562 EventMark em("%s", msg);
2563
2564 // This phase does not use workers, no need for setup
2565
2566 try_inject_alloc_failure();
2567 op_cleanup();
2568 }
2569
2570 void ShenandoahHeap::entry_reset() {
2571 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2572
2573 static const char* msg = "Concurrent reset";
2574 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2575 EventMark em("%s", msg);
2576
|
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26 #include "memory/universe.hpp"
27
28 #include "gc/shared/gcArguments.hpp"
29 #include "gc/shared/gcTimer.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/memAllocator.hpp"
32 #include "gc/shared/parallelCleaning.hpp"
33 #include "gc/shared/plab.hpp"
34
35 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
37 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
38 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
39 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
40 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
41 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
42 #include "gc/shenandoah/shenandoahControlThread.hpp"
43 #include "gc/shenandoah/shenandoahFreeSet.hpp"
44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
45 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
46 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
47 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
48 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
49 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
50 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
51 #include "gc/shenandoah/shenandoahMetrics.hpp"
52 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
53 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
54 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
55 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
56 #include "gc/shenandoah/shenandoahStringDedup.hpp"
57 #include "gc/shenandoah/shenandoahTaskqueue.hpp"
58 #include "gc/shenandoah/shenandoahUtils.hpp"
59 #include "gc/shenandoah/shenandoahVerifier.hpp"
60 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
61 #include "gc/shenandoah/shenandoahVMOperations.hpp"
1055
1056 public:
1057 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1058 AbstractGangTask("Shenandoah evacuate and update roots"),
1059 _rp(rp) {}
1060
1061 void work(uint worker_id) {
1062 ShenandoahParallelWorkerSession worker_session(worker_id);
1063 ShenandoahEvacOOMScope oom_evac_scope;
1064 ShenandoahEvacuateUpdateRootsClosure cl;
1065 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1066 _rp->roots_do(worker_id, &cl);
1067 }
1068 };
1069
1070 void ShenandoahHeap::evacuate_and_update_roots() {
1071 #if COMPILER2_OR_JVMCI
1072 DerivedPointerTable::clear();
1073 #endif
1074 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
1075 {
1076 // Include concurrent roots if current cycle can not process those roots concurrently
1077 bool include_concurrent_roots = !ShenandoahConcurrentRoots::should_do_concurrent_roots();
1078
1079 ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac, include_concurrent_roots);
1080 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1081 workers()->run_task(&roots_task);
1082 }
1083
1084 #if COMPILER2_OR_JVMCI
1085 DerivedPointerTable::update_pointers();
1086 #endif
1087 }
1088
1089 // Returns size in bytes
1090 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1091 if (ShenandoahElasticTLAB) {
1092 // With Elastic TLABs, return the max allowed size, and let the allocation path
1093 // figure out the safe size for current allocation.
1094 return ShenandoahHeapRegion::max_tlab_size_bytes();
1095 } else {
1096 return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes());
1097 }
1098 }
1099
1503 // If collection set has candidates, start evacuation.
1504 // Otherwise, bypass the rest of the cycle.
1505 if (!collection_set()->is_empty()) {
1506 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1507
1508 if (ShenandoahVerify) {
1509 verifier()->verify_before_evacuation();
1510 }
1511
1512 set_evacuation_in_progress(true);
1513 // From here on, we need to update references.
1514 set_has_forwarded_objects(true);
1515
1516 evacuate_and_update_roots();
1517
1518 if (ShenandoahPacing) {
1519 pacer()->setup_for_evac();
1520 }
1521
1522 if (ShenandoahVerify) {
1523 if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1524 ShenandoahRootVerifier::RootTypes types = ShenandoahRootVerifier::combine(ShenandoahRootVerifier::JNIHandleRoots, ShenandoahRootVerifier::CLDGRoots);
1525 verifier()->verify_roots_no_forwarded_except(types);
1526 } else {
1527 verifier()->verify_roots_no_forwarded();
1528 }
1529 verifier()->verify_during_evacuation();
1530 }
1531 } else {
1532 if (ShenandoahVerify) {
1533 verifier()->verify_after_concmark();
1534 }
1535
1536 if (VerifyAfterGC) {
1537 Universe::verify();
1538 }
1539 }
1540
1541 } else {
1542 concurrent_mark()->cancel();
1543 stop_concurrent_marking();
1544
1545 if (process_references()) {
1546 // Abandon reference processing right away: pre-cleaning must have failed.
1547 ReferenceProcessor *rp = ref_processor();
1548 rp->disable_discovery();
1569 }
1570
1571 void ShenandoahHeap::op_conc_evac() {
1572 ShenandoahEvacuationTask task(this, _collection_set, true);
1573 workers()->run_task(&task);
1574 }
1575
1576 void ShenandoahHeap::op_stw_evac() {
1577 ShenandoahEvacuationTask task(this, _collection_set, false);
1578 workers()->run_task(&task);
1579 }
1580
1581 void ShenandoahHeap::op_updaterefs() {
1582 update_heap_references(true);
1583 }
1584
1585 void ShenandoahHeap::op_cleanup() {
1586 free_set()->recycle_trash();
1587 }
1588
1589 class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
1590 private:
1591 SuspendibleThreadSetJoiner _sts_joiner;
1592 ShenandoahJNIHandleRoots<true /*concurrent*/> _jni_roots;
1593 ShenandoahClassLoaderDataRoots<true /* concurrent */> _cld_roots;
1594 public:
1595 ShenandoahConcurrentRootsEvacUpdateTask() :
1596 AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") {
1597 }
1598
1599 void work(uint worker_id) {
1600 ShenandoahEvacOOMScope oom;
1601 ShenandoahEvacuateUpdateRootsClosure cl;
1602 CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1603
1604 _jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
1605 _cld_roots.cld_do(&clds);
1606 }
1607 };
1608
1609 void ShenandoahHeap::op_concurrent_roots() {
1610 if (is_evacuation_in_progress() &&
1611 ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
1612 ShenandoahConcurrentRootsEvacUpdateTask task;
1613 workers()->run_task(&task);
1614 }
1615 }
1616
1617 void ShenandoahHeap::op_reset() {
1618 reset_mark_bitmap();
1619 }
1620
1621 void ShenandoahHeap::op_preclean() {
1622 concurrent_mark()->preclean_weak_refs();
1623 }
1624
1625 void ShenandoahHeap::op_init_traversal() {
1626 traversal_gc()->init_traversal_collection();
1627 }
1628
1629 void ShenandoahHeap::op_traversal() {
1630 traversal_gc()->concurrent_traversal_collection();
1631 }
1632
1633 void ShenandoahHeap::op_final_traversal() {
1634 traversal_gc()->final_traversal_collection();
1635 }
1636
2573 "concurrent evacuation");
2574
2575 try_inject_alloc_failure();
2576 op_conc_evac();
2577 }
2578
2579 void ShenandoahHeap::entry_updaterefs() {
2580 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_update_refs);
2581
2582 static const char* msg = "Concurrent update references";
2583 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2584 EventMark em("%s", msg);
2585
2586 ShenandoahWorkerScope scope(workers(),
2587 ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
2588 "concurrent reference update");
2589
2590 try_inject_alloc_failure();
2591 op_updaterefs();
2592 }
2593
2594 void ShenandoahHeap::entry_concurrent_roots() {
2595 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
2596
2597 static const char* msg = "Concurrent roots processing";
2598 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2599 EventMark em("%s", msg);
2600
2601 ShenandoahWorkerScope scope(workers(),
2602 ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
2603 "concurrent root processing");
2604
2605 try_inject_alloc_failure();
2606 op_concurrent_roots();
2607 }
2608
2609 void ShenandoahHeap::entry_cleanup() {
2610 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
2611
2612 static const char* msg = "Concurrent cleanup";
2613 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2614 EventMark em("%s", msg);
2615
2616 // This phase does not use workers, no need for setup
2617
2618 try_inject_alloc_failure();
2619 op_cleanup();
2620 }
2621
2622 void ShenandoahHeap::entry_reset() {
2623 ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_reset);
2624
2625 static const char* msg = "Concurrent reset";
2626 GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
2627 EventMark em("%s", msg);
2628
|