25 #include "precompiled.hpp"
26 #include "gc/g1/g1BarrierSet.hpp"
27 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
28 #include "gc/g1/g1CardTable.inline.hpp"
29 #include "gc/g1/g1CardTableEntryClosure.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1ConcurrentRefine.hpp"
32 #include "gc/g1/g1DirtyCardQueue.hpp"
33 #include "gc/g1/g1FromCardCache.hpp"
34 #include "gc/g1/g1GCPhaseTimes.hpp"
35 #include "gc/g1/g1HotCardCache.hpp"
36 #include "gc/g1/g1OopClosures.inline.hpp"
37 #include "gc/g1/g1RootClosures.hpp"
38 #include "gc/g1/g1RemSet.hpp"
39 #include "gc/g1/g1SharedDirtyCardQueue.hpp"
40 #include "gc/g1/heapRegion.inline.hpp"
41 #include "gc/g1/heapRegionManager.inline.hpp"
42 #include "gc/g1/heapRegionRemSet.inline.hpp"
43 #include "gc/g1/sparsePRT.hpp"
44 #include "gc/shared/gcTraceTime.inline.hpp"
45 #include "gc/shared/suspendibleThreadSet.hpp"
46 #include "jfr/jfrEvents.hpp"
47 #include "memory/iterator.hpp"
48 #include "memory/resourceArea.hpp"
49 #include "oops/access.inline.hpp"
50 #include "oops/oop.inline.hpp"
51 #include "runtime/os.hpp"
52 #include "utilities/align.hpp"
53 #include "utilities/globalDefinitions.hpp"
54 #include "utilities/stack.inline.hpp"
55 #include "utilities/ticks.hpp"
56
57 // Collects information about the overall heap root scan progress during an evacuation.
58 //
59 // Scanning the remembered sets works by first merging all sources of cards to be
60 // scanned (log buffers, hcc, remembered sets) into a single data structure to remove
61 // duplicates and simplify work distribution.
62 //
63 // During the following card scanning we not only scan this combined set of cards, but
64 // also remember that these were completely scanned. The following evacuation passes
1043 return false;
1044 }
1045
1046 size_t merged_sparse() const { return _cl.merged_sparse(); }
1047 size_t merged_fine() const { return _cl.merged_fine(); }
1048 size_t merged_coarse() const { return _cl.merged_coarse(); }
1049 };
1050
1051 // Visitor for the log buffer entries to merge them into the card table.
1052 class G1MergeLogBufferCardsClosure : public G1CardTableEntryClosure {
1053 G1RemSetScanState* _scan_state;
1054 G1CardTable* _ct;
1055
1056 size_t _cards_dirty;
1057 size_t _cards_skipped;
1058 public:
1059 G1MergeLogBufferCardsClosure(G1CollectedHeap* g1h, G1RemSetScanState* scan_state) :
1060 _scan_state(scan_state), _ct(g1h->card_table()), _cards_dirty(0), _cards_skipped(0)
1061 {}
1062
1063 bool do_card_ptr(CardValue* card_ptr, uint worker_i) {
1064 // The only time we care about recording cards that
1065 // contain references that point into the collection set
1066 // is during RSet updating within an evacuation pause.
1067 // In this case worker_id should be the id of a GC worker thread.
1068 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
1069
1070 uint const region_idx = _ct->region_idx_for(card_ptr);
1071
1072 // The second clause must come after - the log buffers might contain cards to uncommited
1073 // regions.
1074 // This code may count duplicate entries in the log buffers (even if rare) multiple
1075 // times.
1076 if (_scan_state->contains_cards_to_process(region_idx) && (*card_ptr == G1CardTable::dirty_card_val())) {
1077 _scan_state->add_dirty_region(region_idx);
1078 _scan_state->set_chunk_dirty(_ct->index_for_cardvalue(card_ptr));
1079 _cards_dirty++;
1080 } else {
1081 // We may have had dirty cards in the (initial) collection set (or the
1082 // young regions which are always in the initial collection set). We do
1083 // not fix their cards here: we already added these regions to the set of
1084 // regions to clear the card table at the end during the prepare() phase.
1085 _cards_skipped++;
1086 }
1087 return true;
1088 }
1089
1090 size_t cards_dirty() const { return _cards_dirty; }
1091 size_t cards_skipped() const { return _cards_skipped; }
1092 };
1093
1094 HeapRegionClaimer _hr_claimer;
1095 G1RemSetScanState* _scan_state;
1096 bool _initial_evacuation;
1097
1098 volatile bool _fast_reclaim_handled;
1099
1100 public:
1101 G1MergeHeapRootsTask(G1RemSetScanState* scan_state, uint num_workers, bool initial_evacuation) :
1102 AbstractGangTask("G1 Merge Heap Roots"),
1103 _hr_claimer(num_workers),
1104 _scan_state(scan_state),
1105 _initial_evacuation(initial_evacuation),
1106 _fast_reclaim_handled(false) { }
1107
1108 virtual void work(uint worker_id) {
1109 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1110 G1GCPhaseTimes* p = g1h->phase_times();
1111
1112 G1GCPhaseTimes::GCParPhases merge_remset_phase = _initial_evacuation ?
1113 G1GCPhaseTimes::MergeRS :
1114 G1GCPhaseTimes::OptMergeRS;
1115
1116 // We schedule flushing the remembered sets of humongous fast reclaim candidates
1117 // onto the card table first to allow the remaining parallelized tasks hide it.
1118 if (_initial_evacuation &&
1119 p->fast_reclaim_humongous_candidates() > 0 &&
1120 !_fast_reclaim_handled &&
1121 !Atomic::cmpxchg(true, &_fast_reclaim_handled, false)) {
1122
1123 G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeER, worker_id);
1124
1125 G1FlushHumongousCandidateRemSets cl(_scan_state);
1126 g1h->heap_region_iterate(&cl);
1141 p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
1142 }
1143
1144 // Apply closure to log entries in the HCC.
1145 if (_initial_evacuation && G1HotCardCache::default_use_cache()) {
1146 assert(merge_remset_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
1147 G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeHCC, worker_id);
1148 G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
1149 g1h->iterate_hcc_closure(&cl, worker_id);
1150
1151 p->record_thread_work_item(G1GCPhaseTimes::MergeHCC, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeHCCDirtyCards);
1152 p->record_thread_work_item(G1GCPhaseTimes::MergeHCC, worker_id, cl.cards_skipped(), G1GCPhaseTimes::MergeHCCSkippedCards);
1153 }
1154
1155 // Now apply the closure to all remaining log entries.
1156 if (_initial_evacuation) {
1157 assert(merge_remset_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
1158 G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeLB, worker_id);
1159
1160 G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
1161 g1h->iterate_dirty_card_closure(&cl, worker_id);
1162
1163 p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeLBDirtyCards);
1164 p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_skipped(), G1GCPhaseTimes::MergeLBSkippedCards);
1165 }
1166 }
1167 };
1168
1169 void G1RemSet::print_merge_heap_roots_stats() {
1170 size_t num_visited_cards = _scan_state->num_visited_cards();
1171
1172 size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
1173
1174 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1175 size_t total_old_region_cards =
1176 (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
1177
1178 log_debug(gc,remset)("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
1179 num_visited_cards,
1180 total_dirty_region_cards,
1181 percent_of(num_visited_cards, total_dirty_region_cards),
|
25 #include "precompiled.hpp"
26 #include "gc/g1/g1BarrierSet.hpp"
27 #include "gc/g1/g1BlockOffsetTable.inline.hpp"
28 #include "gc/g1/g1CardTable.inline.hpp"
29 #include "gc/g1/g1CardTableEntryClosure.hpp"
30 #include "gc/g1/g1CollectedHeap.inline.hpp"
31 #include "gc/g1/g1ConcurrentRefine.hpp"
32 #include "gc/g1/g1DirtyCardQueue.hpp"
33 #include "gc/g1/g1FromCardCache.hpp"
34 #include "gc/g1/g1GCPhaseTimes.hpp"
35 #include "gc/g1/g1HotCardCache.hpp"
36 #include "gc/g1/g1OopClosures.inline.hpp"
37 #include "gc/g1/g1RootClosures.hpp"
38 #include "gc/g1/g1RemSet.hpp"
39 #include "gc/g1/g1SharedDirtyCardQueue.hpp"
40 #include "gc/g1/heapRegion.inline.hpp"
41 #include "gc/g1/heapRegionManager.inline.hpp"
42 #include "gc/g1/heapRegionRemSet.inline.hpp"
43 #include "gc/g1/sparsePRT.hpp"
44 #include "gc/shared/gcTraceTime.inline.hpp"
45 #include "gc/shared/ptrQueue.hpp"
46 #include "gc/shared/suspendibleThreadSet.hpp"
47 #include "jfr/jfrEvents.hpp"
48 #include "memory/iterator.hpp"
49 #include "memory/resourceArea.hpp"
50 #include "oops/access.inline.hpp"
51 #include "oops/oop.inline.hpp"
52 #include "runtime/os.hpp"
53 #include "utilities/align.hpp"
54 #include "utilities/globalDefinitions.hpp"
55 #include "utilities/stack.inline.hpp"
56 #include "utilities/ticks.hpp"
57
58 // Collects information about the overall heap root scan progress during an evacuation.
59 //
60 // Scanning the remembered sets works by first merging all sources of cards to be
61 // scanned (log buffers, hcc, remembered sets) into a single data structure to remove
62 // duplicates and simplify work distribution.
63 //
64 // During the following card scanning we not only scan this combined set of cards, but
65 // also remember that these were completely scanned. The following evacuation passes
1044 return false;
1045 }
1046
1047 size_t merged_sparse() const { return _cl.merged_sparse(); }
1048 size_t merged_fine() const { return _cl.merged_fine(); }
1049 size_t merged_coarse() const { return _cl.merged_coarse(); }
1050 };
1051
1052 // Visitor for the log buffer entries to merge them into the card table.
1053 class G1MergeLogBufferCardsClosure : public G1CardTableEntryClosure {
1054 G1RemSetScanState* _scan_state;
1055 G1CardTable* _ct;
1056
1057 size_t _cards_dirty;
1058 size_t _cards_skipped;
1059 public:
1060 G1MergeLogBufferCardsClosure(G1CollectedHeap* g1h, G1RemSetScanState* scan_state) :
1061 _scan_state(scan_state), _ct(g1h->card_table()), _cards_dirty(0), _cards_skipped(0)
1062 {}
1063
1064 void do_card_ptr(CardValue* card_ptr, uint worker_i) {
1065 // The only time we care about recording cards that
1066 // contain references that point into the collection set
1067 // is during RSet updating within an evacuation pause.
1068 // In this case worker_id should be the id of a GC worker thread.
1069 assert(SafepointSynchronize::is_at_safepoint(), "not during an evacuation pause");
1070
1071 uint const region_idx = _ct->region_idx_for(card_ptr);
1072
1073 // The second clause must come after - the log buffers might contain cards to uncommited
1074 // regions.
1075 // This code may count duplicate entries in the log buffers (even if rare) multiple
1076 // times.
1077 if (_scan_state->contains_cards_to_process(region_idx) && (*card_ptr == G1CardTable::dirty_card_val())) {
1078 _scan_state->add_dirty_region(region_idx);
1079 _scan_state->set_chunk_dirty(_ct->index_for_cardvalue(card_ptr));
1080 _cards_dirty++;
1081 } else {
1082 // We may have had dirty cards in the (initial) collection set (or the
1083 // young regions which are always in the initial collection set). We do
1084 // not fix their cards here: we already added these regions to the set of
1085 // regions to clear the card table at the end during the prepare() phase.
1086 _cards_skipped++;
1087 }
1088 }
1089
1090 size_t cards_dirty() const { return _cards_dirty; }
1091 size_t cards_skipped() const { return _cards_skipped; }
1092 };
1093
1094 HeapRegionClaimer _hr_claimer;
1095 G1RemSetScanState* _scan_state;
1096 BufferNode::Stack _dirty_card_buffers;
1097 bool _initial_evacuation;
1098
1099 volatile bool _fast_reclaim_handled;
1100
1101 void apply_closure_to_dirty_card_buffers(G1MergeLogBufferCardsClosure* cl, uint worker_id) {
1102 G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
1103 size_t buffer_size = dcqs.buffer_size();
1104 while (BufferNode* node = _dirty_card_buffers.pop()) {
1105 cl->apply_to_buffer(node, buffer_size, worker_id);
1106 dcqs.deallocate_buffer(node);
1107 }
1108 }
1109
1110 public:
1111 G1MergeHeapRootsTask(G1RemSetScanState* scan_state, uint num_workers, bool initial_evacuation) :
1112 AbstractGangTask("G1 Merge Heap Roots"),
1113 _hr_claimer(num_workers),
1114 _scan_state(scan_state),
1115 _dirty_card_buffers(),
1116 _initial_evacuation(initial_evacuation),
1117 _fast_reclaim_handled(false)
1118 {
1119 if (initial_evacuation) {
1120 G1DirtyCardQueueSet& dcqs = G1BarrierSet::dirty_card_queue_set();
1121 G1BufferNodeList buffers = dcqs.take_all_completed_buffers();
1122 if (buffers._entry_count != 0) {
1123 _dirty_card_buffers.prepend(*buffers._head, *buffers._tail);
1124 }
1125 }
1126 }
1127
1128 virtual void work(uint worker_id) {
1129 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1130 G1GCPhaseTimes* p = g1h->phase_times();
1131
1132 G1GCPhaseTimes::GCParPhases merge_remset_phase = _initial_evacuation ?
1133 G1GCPhaseTimes::MergeRS :
1134 G1GCPhaseTimes::OptMergeRS;
1135
1136 // We schedule flushing the remembered sets of humongous fast reclaim candidates
1137 // onto the card table first to allow the remaining parallelized tasks hide it.
1138 if (_initial_evacuation &&
1139 p->fast_reclaim_humongous_candidates() > 0 &&
1140 !_fast_reclaim_handled &&
1141 !Atomic::cmpxchg(true, &_fast_reclaim_handled, false)) {
1142
1143 G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeER, worker_id);
1144
1145 G1FlushHumongousCandidateRemSets cl(_scan_state);
1146 g1h->heap_region_iterate(&cl);
1161 p->record_or_add_thread_work_item(merge_remset_phase, worker_id, cl.merged_coarse(), G1GCPhaseTimes::MergeRSMergedCoarse);
1162 }
1163
1164 // Apply closure to log entries in the HCC.
1165 if (_initial_evacuation && G1HotCardCache::default_use_cache()) {
1166 assert(merge_remset_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
1167 G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeHCC, worker_id);
1168 G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
1169 g1h->iterate_hcc_closure(&cl, worker_id);
1170
1171 p->record_thread_work_item(G1GCPhaseTimes::MergeHCC, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeHCCDirtyCards);
1172 p->record_thread_work_item(G1GCPhaseTimes::MergeHCC, worker_id, cl.cards_skipped(), G1GCPhaseTimes::MergeHCCSkippedCards);
1173 }
1174
1175 // Now apply the closure to all remaining log entries.
1176 if (_initial_evacuation) {
1177 assert(merge_remset_phase == G1GCPhaseTimes::MergeRS, "Wrong merge phase");
1178 G1GCParPhaseTimesTracker x(p, G1GCPhaseTimes::MergeLB, worker_id);
1179
1180 G1MergeLogBufferCardsClosure cl(g1h, _scan_state);
1181 apply_closure_to_dirty_card_buffers(&cl, worker_id);
1182
1183 p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_dirty(), G1GCPhaseTimes::MergeLBDirtyCards);
1184 p->record_thread_work_item(G1GCPhaseTimes::MergeLB, worker_id, cl.cards_skipped(), G1GCPhaseTimes::MergeLBSkippedCards);
1185 }
1186 }
1187 };
1188
1189 void G1RemSet::print_merge_heap_roots_stats() {
1190 size_t num_visited_cards = _scan_state->num_visited_cards();
1191
1192 size_t total_dirty_region_cards = _scan_state->num_cards_in_dirty_regions();
1193
1194 G1CollectedHeap* g1h = G1CollectedHeap::heap();
1195 size_t total_old_region_cards =
1196 (g1h->num_regions() - (g1h->num_free_regions() - g1h->collection_set()->cur_length())) * HeapRegion::CardsPerRegion;
1197
1198 log_debug(gc,remset)("Visited cards " SIZE_FORMAT " Total dirty " SIZE_FORMAT " (%.2lf%%) Total old " SIZE_FORMAT " (%.2lf%%)",
1199 num_visited_cards,
1200 total_dirty_region_cards,
1201 percent_of(num_visited_cards, total_dirty_region_cards),
|