7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26
27 #include "gc/shared/gcTimer.hpp"
28 #include "gc/shared/gcTraceTime.inline.hpp"
29 #include "gc/shared/memAllocator.hpp"
30 #include "gc/shared/parallelCleaning.hpp"
31 #include "gc/shared/plab.hpp"
32
33 #include "gc/shenandoah/brooksPointer.hpp"
34 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
35 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
36 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
37 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
38 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
39 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
40 #include "gc/shenandoah/shenandoahControlThread.hpp"
41 #include "gc/shenandoah/shenandoahFreeSet.hpp"
42 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
43 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
44 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
45 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
46 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
47 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
48 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
49 #include "gc/shenandoah/shenandoahMetrics.hpp"
861 do_oop_work(p);
862 }
863 };
864
865 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
866 private:
867 ShenandoahHeap* const _heap;
868 Thread* const _thread;
869 public:
870 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
871 _heap(heap), _thread(Thread::current()) {}
872
873 void do_object(oop p) {
874 shenandoah_assert_marked(NULL, p);
875 if (oopDesc::equals_raw(p, ShenandoahBarrierSet::resolve_forwarded_not_null(p))) {
876 _heap->evacuate_object(p, _thread);
877 }
878 }
879 };
880
881 class ShenandoahEvacuationTask : public AbstractGangTask {
882 private:
883 ShenandoahHeap* const _sh;
884 ShenandoahCollectionSet* const _cs;
885 bool _concurrent;
886 public:
887 ShenandoahEvacuationTask(ShenandoahHeap* sh,
888 ShenandoahCollectionSet* cs,
889 bool concurrent) :
890 AbstractGangTask("Parallel Evacuation Task"),
891 _sh(sh),
892 _cs(cs),
893 _concurrent(concurrent)
894 {}
895
896 void work(uint worker_id) {
897 if (_concurrent) {
898 ShenandoahConcurrentWorkerSession worker_session(worker_id);
899 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
900 ShenandoahEvacOOMScope oom_evac_scope;
901 do_work();
902 } else {
903 ShenandoahParallelWorkerSession worker_session(worker_id);
904 ShenandoahEvacOOMScope oom_evac_scope;
905 do_work();
906 }
907 }
908
909 private:
910 void do_work() {
911 ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
912 ShenandoahHeapRegion* r;
913 while ((r =_cs->claim_next()) != NULL) {
914 assert(r->has_live(), "all-garbage regions are reclaimed early");
915 _sh->marked_object_iterate(r, &cl);
916
917 if (ShenandoahPacing) {
918 _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
919 }
920
921 if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
922 break;
923 }
924 }
925 }
926 };
927
928 void ShenandoahHeap::trash_cset_regions() {
929 ShenandoahHeapLocker locker(lock());
930
1426 {
1427 ShenandoahHeapLocker locker(lock());
1428 _collection_set->clear();
1429 _free_set->clear();
1430
1431 heuristics()->choose_collection_set(_collection_set);
1432
1433 _free_set->rebuild();
1434 }
1435 }
1436
1437 // If collection set has candidates, start evacuation.
1438 // Otherwise, bypass the rest of the cycle.
1439 if (!collection_set()->is_empty()) {
1440 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1441
1442 if (ShenandoahVerify) {
1443 verifier()->verify_before_evacuation();
1444 }
1445
1446 set_evacuation_in_progress(true);
1447 // From here on, we need to update references.
1448 set_has_forwarded_objects(true);
1449
1450 evacuate_and_update_roots();
1451
1452 if (ShenandoahPacing) {
1453 pacer()->setup_for_evac();
1454 }
1455 } else {
1456 if (ShenandoahVerify) {
1457 verifier()->verify_after_concmark();
1458 }
1459
1460 if (VerifyAfterGC) {
1461 Universe::verify();
1462 }
1463 }
1464
1465 } else {
1466 concurrent_mark()->cancel();
1467 stop_concurrent_marking();
1468
1469 if (process_references()) {
1470 // Abandon reference processing right away: pre-cleaning must have failed.
1471 ReferenceProcessor *rp = ref_processor();
1472 rp->disable_discovery();
1473 rp->abandon_partial_discovery();
1474 rp->verify_no_references_recorded();
1475 }
1476 }
1477 }
1478
1479 void ShenandoahHeap::op_final_evac() {
1480 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1481
1482 set_evacuation_in_progress(false);
1483
1484 retire_and_reset_gclabs();
1485
1486 if (ShenandoahVerify) {
1487 verifier()->verify_after_evacuation();
1488 }
1489
1490 if (VerifyAfterGC) {
1491 Universe::verify();
1492 }
1493 }
1494
1495 void ShenandoahHeap::op_conc_evac() {
1496 ShenandoahEvacuationTask task(this, _collection_set, true);
1497 workers()->run_task(&task);
1498 }
1499
1500 void ShenandoahHeap::op_stw_evac() {
1501 ShenandoahEvacuationTask task(this, _collection_set, false);
1502 workers()->run_task(&task);
1880
1881 if (ShenandoahStringDedup::is_enabled()) {
1882 ShenandoahGCPhase phase(full_gc ?
1883 ShenandoahPhaseTimings::full_gc_purge_string_dedup :
1884 ShenandoahPhaseTimings::purge_string_dedup);
1885 ShenandoahStringDedup::parallel_cleanup();
1886 }
1887
1888 {
1889 ShenandoahGCPhase phase(full_gc ?
1890 ShenandoahPhaseTimings::full_gc_purge_cldg :
1891 ShenandoahPhaseTimings::purge_cldg);
1892 ClassLoaderDataGraph::purge();
1893 }
1894 }
1895
1896 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1897 set_gc_state_mask(HAS_FORWARDED, cond);
1898 }
1899
1900 void ShenandoahHeap::set_process_references(bool pr) {
1901 _process_references.set_cond(pr);
1902 }
1903
1904 void ShenandoahHeap::set_unload_classes(bool uc) {
1905 _unload_classes.set_cond(uc);
1906 }
1907
1908 bool ShenandoahHeap::process_references() const {
1909 return _process_references.is_set();
1910 }
1911
1912 bool ShenandoahHeap::unload_classes() const {
1913 return _unload_classes.is_set();
1914 }
1915
1916 address ShenandoahHeap::in_cset_fast_test_addr() {
1917 ShenandoahHeap* heap = ShenandoahHeap::heap();
1918 assert(heap->collection_set() != NULL, "Sanity");
1919 return (address) heap->collection_set()->biased_map_address();
2047 if (ShenandoahPacing) {
2048 _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
2049 }
2050 if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2051 return;
2052 }
2053 r = _regions->next();
2054 }
2055 }
2056 };
2057
2058 void ShenandoahHeap::update_heap_references(bool concurrent) {
2059 ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2060 workers()->run_task(&task);
2061 }
2062
2063 void ShenandoahHeap::op_init_updaterefs() {
2064 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2065
2066 set_evacuation_in_progress(false);
2067
2068 retire_and_reset_gclabs();
2069
2070 if (ShenandoahVerify) {
2071 verifier()->verify_before_updaterefs();
2072 }
2073
2074 set_update_refs_in_progress(true);
2075 make_parsable(true);
2076 for (uint i = 0; i < num_regions(); i++) {
2077 ShenandoahHeapRegion* r = get_region(i);
2078 r->set_concurrent_iteration_safe_limit(r->top());
2079 }
2080
2081 // Reset iterator.
2082 _update_refs_iterator.reset();
2083
2084 if (ShenandoahPacing) {
2085 pacer()->setup_for_updaterefs();
2086 }
|
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26
27 #include "classfile/stringTable.hpp"
28
29 #include "gc/shared/gcTimer.hpp"
30 #include "gc/shared/gcTraceTime.inline.hpp"
31 #include "gc/shared/memAllocator.hpp"
32 #include "gc/shared/oopStorage.hpp"
33 #include "gc/shared/oopStorageParState.inline.hpp"
34 #include "gc/shared/parallelCleaning.hpp"
35 #include "gc/shared/plab.hpp"
36
37 #include "gc/shenandoah/brooksPointer.hpp"
38 #include "gc/shenandoah/shenandoahAllocTracker.hpp"
39 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
40 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
41 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
42 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
43 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
44 #include "gc/shenandoah/shenandoahControlThread.hpp"
45 #include "gc/shenandoah/shenandoahFreeSet.hpp"
46 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
47 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
48 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
49 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
50 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
51 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
52 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
53 #include "gc/shenandoah/shenandoahMetrics.hpp"
865 do_oop_work(p);
866 }
867 };
868
869 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
870 private:
871 ShenandoahHeap* const _heap;
872 Thread* const _thread;
873 public:
874 ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
875 _heap(heap), _thread(Thread::current()) {}
876
877 void do_object(oop p) {
878 shenandoah_assert_marked(NULL, p);
879 if (oopDesc::equals_raw(p, ShenandoahBarrierSet::resolve_forwarded_not_null(p))) {
880 _heap->evacuate_object(p, _thread);
881 }
882 }
883 };
884
885 class ShenandoahUnlinkAndUpdateStringTableClosure : public OopClosure {
886 private:
887 ShenandoahHeap* const _heap;
888 ShenandoahMarkingContext* const _ctx;
889 Thread* const _thread;
890
891 template <typename T>
892 inline void do_oop_work(T* p) {
893 T o = RawAccess<>::oop_load(p);
894 if (!CompressedOops::is_null(o)) {
895 oop obj = CompressedOops::decode_not_null(o);
896 if (_ctx->is_marked(obj)) {
897 if (_heap->in_collection_set(obj)) {
898 oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
899 if (oopDesc::equals_raw(resolved, obj)) {
900 resolved = _heap->evacuate_object(obj, _thread);
901 }
902 RawAccess<IS_NOT_NULL>::oop_store(p, resolved);
903 }
904 } else {
905 RawAccess<>::oop_store(p, (oop) NULL);
906 }
907 }
908 }
909 public:
910 ShenandoahUnlinkAndUpdateStringTableClosure() :
911 _heap(ShenandoahHeap::heap()),
912 _ctx(ShenandoahHeap::heap()->complete_marking_context()),
913 _thread(Thread::current()) {}
914 virtual void do_oop(oop* p) { do_oop_work(p); }
915 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
916 };
917
918 class ShenandoahEvacuationTask : public AbstractGangTask {
919 private:
920 ShenandoahHeap* const _sh;
921 ShenandoahCollectionSet* const _cs;
922 OopStorage::ParState<true /* concurrent */, false /* is_const */> _string_table;
923 bool _concurrent;
924 public:
925 ShenandoahEvacuationTask(ShenandoahHeap* sh,
926 ShenandoahCollectionSet* cs,
927 bool concurrent) :
928 AbstractGangTask("Parallel Evacuation Task"),
929 _sh(sh),
930 _cs(cs),
931 _string_table(StringTable::weak_storage()),
932 _concurrent(concurrent)
933 {}
934
935 void work(uint worker_id) {
936 if (_concurrent) {
937 ShenandoahConcurrentWorkerSession worker_session(worker_id);
938 ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers);
939 ShenandoahEvacOOMScope oom_evac_scope;
940 do_work();
941 } else {
942 ShenandoahParallelWorkerSession worker_session(worker_id);
943 ShenandoahEvacOOMScope oom_evac_scope;
944 do_work();
945 }
946 }
947
948 private:
949 void do_work() {
950 ShenandoahUnlinkAndUpdateStringTableClosure stcl;
951 _string_table.oops_do(&stcl);
952
953 ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
954 ShenandoahHeapRegion* r;
955 while ((r =_cs->claim_next()) != NULL) {
956 assert(r->has_live(), "all-garbage regions are reclaimed early");
957 _sh->marked_object_iterate(r, &cl);
958
959 if (ShenandoahPacing) {
960 _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
961 }
962
963 if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
964 break;
965 }
966 }
967 }
968 };
969
970 void ShenandoahHeap::trash_cset_regions() {
971 ShenandoahHeapLocker locker(lock());
972
1468 {
1469 ShenandoahHeapLocker locker(lock());
1470 _collection_set->clear();
1471 _free_set->clear();
1472
1473 heuristics()->choose_collection_set(_collection_set);
1474
1475 _free_set->rebuild();
1476 }
1477 }
1478
1479 // If collection set has candidates, start evacuation.
1480 // Otherwise, bypass the rest of the cycle.
1481 if (!collection_set()->is_empty()) {
1482 ShenandoahGCPhase init_evac(ShenandoahPhaseTimings::init_evac);
1483
1484 if (ShenandoahVerify) {
1485 verifier()->verify_before_evacuation();
1486 }
1487
1488 set_refcleaning_in_progress(true);
1489 set_evacuation_in_progress(true);
1490 // From here on, we need to update references.
1491 set_has_forwarded_objects(true);
1492
1493 evacuate_and_update_roots();
1494
1495 if (ShenandoahPacing) {
1496 pacer()->setup_for_evac();
1497 }
1498 } else {
1499 if (ShenandoahVerify) {
1500 verifier()->verify_after_concmark();
1501 }
1502
1503 if (VerifyAfterGC) {
1504 Universe::verify();
1505 }
1506 }
1507
1508 } else {
1509 concurrent_mark()->cancel();
1510 stop_concurrent_marking();
1511
1512 if (process_references()) {
1513 // Abandon reference processing right away: pre-cleaning must have failed.
1514 ReferenceProcessor *rp = ref_processor();
1515 rp->disable_discovery();
1516 rp->abandon_partial_discovery();
1517 rp->verify_no_references_recorded();
1518 }
1519 }
1520 }
1521
1522 void ShenandoahHeap::op_final_evac() {
1523 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
1524
1525 set_evacuation_in_progress(false);
1526 set_refcleaning_in_progress(false);
1527
1528 retire_and_reset_gclabs();
1529
1530 if (ShenandoahVerify) {
1531 verifier()->verify_after_evacuation();
1532 }
1533
1534 if (VerifyAfterGC) {
1535 Universe::verify();
1536 }
1537 }
1538
1539 void ShenandoahHeap::op_conc_evac() {
1540 ShenandoahEvacuationTask task(this, _collection_set, true);
1541 workers()->run_task(&task);
1542 }
1543
1544 void ShenandoahHeap::op_stw_evac() {
1545 ShenandoahEvacuationTask task(this, _collection_set, false);
1546 workers()->run_task(&task);
1924
1925 if (ShenandoahStringDedup::is_enabled()) {
1926 ShenandoahGCPhase phase(full_gc ?
1927 ShenandoahPhaseTimings::full_gc_purge_string_dedup :
1928 ShenandoahPhaseTimings::purge_string_dedup);
1929 ShenandoahStringDedup::parallel_cleanup();
1930 }
1931
1932 {
1933 ShenandoahGCPhase phase(full_gc ?
1934 ShenandoahPhaseTimings::full_gc_purge_cldg :
1935 ShenandoahPhaseTimings::purge_cldg);
1936 ClassLoaderDataGraph::purge();
1937 }
1938 }
1939
1940 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
1941 set_gc_state_mask(HAS_FORWARDED, cond);
1942 }
1943
1944 void ShenandoahHeap::set_refcleaning_in_progress(bool in_progress) {
1945 set_gc_state_mask(REFCLEANING, in_progress);
1946 }
1947
1948 void ShenandoahHeap::set_process_references(bool pr) {
1949 _process_references.set_cond(pr);
1950 }
1951
1952 void ShenandoahHeap::set_unload_classes(bool uc) {
1953 _unload_classes.set_cond(uc);
1954 }
1955
1956 bool ShenandoahHeap::process_references() const {
1957 return _process_references.is_set();
1958 }
1959
1960 bool ShenandoahHeap::unload_classes() const {
1961 return _unload_classes.is_set();
1962 }
1963
1964 address ShenandoahHeap::in_cset_fast_test_addr() {
1965 ShenandoahHeap* heap = ShenandoahHeap::heap();
1966 assert(heap->collection_set() != NULL, "Sanity");
1967 return (address) heap->collection_set()->biased_map_address();
2095 if (ShenandoahPacing) {
2096 _heap->pacer()->report_updaterefs(pointer_delta(top_at_start_ur, r->bottom()));
2097 }
2098 if (_heap->check_cancelled_gc_and_yield(_concurrent)) {
2099 return;
2100 }
2101 r = _regions->next();
2102 }
2103 }
2104 };
2105
2106 void ShenandoahHeap::update_heap_references(bool concurrent) {
2107 ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(&_update_refs_iterator, concurrent);
2108 workers()->run_task(&task);
2109 }
2110
2111 void ShenandoahHeap::op_init_updaterefs() {
2112 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2113
2114 set_evacuation_in_progress(false);
2115 set_refcleaning_in_progress(false);
2116
2117 retire_and_reset_gclabs();
2118
2119 if (ShenandoahVerify) {
2120 verifier()->verify_before_updaterefs();
2121 }
2122
2123 set_update_refs_in_progress(true);
2124 make_parsable(true);
2125 for (uint i = 0; i < num_regions(); i++) {
2126 ShenandoahHeapRegion* r = get_region(i);
2127 r->set_concurrent_iteration_safe_limit(r->top());
2128 }
2129
2130 // Reset iterator.
2131 _update_refs_iterator.reset();
2132
2133 if (ShenandoahPacing) {
2134 pacer()->setup_for_updaterefs();
2135 }
|