1 /*
2 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/g1/g1BufferNodeList.hpp"
27 #include "gc/g1/g1CardTableEntryClosure.hpp"
28 #include "gc/g1/g1CollectedHeap.inline.hpp"
29 #include "gc/g1/g1DirtyCardQueue.hpp"
30 #include "gc/g1/g1FreeIdSet.hpp"
31 #include "gc/g1/g1RedirtyCardsQueue.hpp"
32 #include "gc/g1/g1RemSet.hpp"
33 #include "gc/g1/g1ThreadLocalData.hpp"
34 #include "gc/g1/heapRegionRemSet.hpp"
35 #include "gc/shared/suspendibleThreadSet.hpp"
36 #include "gc/shared/workgroup.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/flags/flagSetting.hpp"
39 #include "runtime/mutexLocker.hpp"
40 #include "runtime/os.hpp"
41 #include "runtime/safepoint.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "runtime/threadSMR.hpp"
44
45 G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) :
46 // Dirty card queues are always active, so we create them with their
47 // active field set to true.
48 PtrQueue(qset, true /* active */)
49 { }
50
51 G1DirtyCardQueue::~G1DirtyCardQueue() {
52 flush();
53 }
54
55 void G1DirtyCardQueue::handle_completed_buffer() {
56 assert(_buf != NULL, "precondition");
57 BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
58 G1DirtyCardQueueSet* dcqs = dirty_card_qset();
59 if (dcqs->process_or_enqueue_completed_buffer(node)) {
60 reset(); // Buffer fully processed, reset index.
61 } else {
62 allocate_buffer(); // Buffer enqueued, get a new one.
63 }
64 }
65
66 // Assumed to be zero by concurrent threads.
67 static uint par_ids_start() { return 0; }
68
69 G1DirtyCardQueueSet::G1DirtyCardQueueSet(Monitor* cbl_mon,
70 BufferNode::Allocator* allocator) :
71 PtrQueueSet(allocator),
72 _cbl_mon(cbl_mon),
73 _completed_buffers_head(NULL),
74 _completed_buffers_tail(NULL),
75 _num_cards(0),
76 _process_cards_threshold(ProcessCardsThresholdNever),
77 _process_completed_buffers(false),
78 _max_cards(MaxCardsUnlimited),
79 _max_cards_padding(0),
80 _free_ids(par_ids_start(), num_par_ids()),
81 _mutator_refined_cards_counters(NEW_C_HEAP_ARRAY(size_t, num_par_ids(), mtGC))
82 {
83 ::memset(_mutator_refined_cards_counters, 0, num_par_ids() * sizeof(size_t));
84 _all_active = true;
85 }
86
87 G1DirtyCardQueueSet::~G1DirtyCardQueueSet() {
88 abandon_completed_buffers();
89 FREE_C_HEAP_ARRAY(size_t, _mutator_refined_cards_counters);
90 }
91
92 // Determines how many mutator threads can process the buffers in parallel.
93 uint G1DirtyCardQueueSet::num_par_ids() {
94 return (uint)os::initial_active_processor_count();
95 }
96
97 size_t G1DirtyCardQueueSet::total_mutator_refined_cards() const {
98 size_t sum = 0;
99 for (uint i = 0; i < num_par_ids(); ++i) {
100 sum += _mutator_refined_cards_counters[i];
101 }
102 return sum;
103 }
104
105 void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) {
106 G1ThreadLocalData::dirty_card_queue(t).handle_zero_index();
107 }
108
109 void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) {
110 MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag);
111 cbn->set_next(NULL);
112 if (_completed_buffers_tail == NULL) {
113 assert(_completed_buffers_head == NULL, "Well-formedness");
114 _completed_buffers_head = cbn;
115 _completed_buffers_tail = cbn;
116 } else {
117 _completed_buffers_tail->set_next(cbn);
118 _completed_buffers_tail = cbn;
119 }
120 _num_cards += buffer_size() - cbn->index();
121
122 if (!process_completed_buffers() &&
123 (num_cards() > process_cards_threshold())) {
124 set_process_completed_buffers(true);
125 ml.notify_all();
126 }
127 verify_num_cards();
128 }
129
130 BufferNode* G1DirtyCardQueueSet::get_completed_buffer(size_t stop_at) {
131 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
132
133 if (num_cards() <= stop_at) {
134 return NULL;
135 }
136
137 assert(num_cards() > 0, "invariant");
138 assert(_completed_buffers_head != NULL, "invariant");
139 assert(_completed_buffers_tail != NULL, "invariant");
140
141 BufferNode* bn = _completed_buffers_head;
142 _num_cards -= buffer_size() - bn->index();
143 _completed_buffers_head = bn->next();
144 if (_completed_buffers_head == NULL) {
145 assert(num_cards() == 0, "invariant");
146 _completed_buffers_tail = NULL;
147 set_process_completed_buffers(false);
148 }
149 verify_num_cards();
150 bn->set_next(NULL);
151 return bn;
152 }
153
154 #ifdef ASSERT
155 void G1DirtyCardQueueSet::verify_num_cards() const {
156 size_t actual = 0;
157 BufferNode* cur = _completed_buffers_head;
158 while (cur != NULL) {
159 actual += buffer_size() - cur->index();
160 cur = cur->next();
161 }
162 assert(actual == _num_cards,
163 "Num entries in completed buffers should be " SIZE_FORMAT " but are " SIZE_FORMAT,
164 _num_cards, actual);
165 }
166 #endif
167
168 void G1DirtyCardQueueSet::abandon_completed_buffers() {
169 BufferNode* buffers_to_delete = NULL;
170 {
171 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
172 buffers_to_delete = _completed_buffers_head;
173 _completed_buffers_head = NULL;
174 _completed_buffers_tail = NULL;
175 _num_cards = 0;
176 set_process_completed_buffers(false);
177 }
178 while (buffers_to_delete != NULL) {
179 BufferNode* bn = buffers_to_delete;
180 buffers_to_delete = bn->next();
181 bn->set_next(NULL);
182 deallocate_buffer(bn);
183 }
184 }
185
186 void G1DirtyCardQueueSet::notify_if_necessary() {
187 MonitorLocker ml(_cbl_mon, Mutex::_no_safepoint_check_flag);
188 if (num_cards() > process_cards_threshold()) {
189 set_process_completed_buffers(true);
190 ml.notify_all();
191 }
192 }
193
194 // Merge lists of buffers. Notify the processing threads.
195 // The source queue is emptied as a result. The queues
196 // must share the monitor.
197 void G1DirtyCardQueueSet::merge_bufferlists(G1RedirtyCardsQueueSet* src) {
198 assert(allocator() == src->allocator(), "precondition");
199 const G1BufferNodeList from = src->take_all_completed_buffers();
200 if (from._head == NULL) return;
201
202 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
203 if (_completed_buffers_tail == NULL) {
204 assert(_completed_buffers_head == NULL, "Well-formedness");
205 _completed_buffers_head = from._head;
206 _completed_buffers_tail = from._tail;
207 } else {
208 assert(_completed_buffers_head != NULL, "Well formedness");
209 _completed_buffers_tail->set_next(from._head);
210 _completed_buffers_tail = from._tail;
211 }
212 _num_cards += from._entry_count;
213
214 assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL ||
215 _completed_buffers_head != NULL && _completed_buffers_tail != NULL,
216 "Sanity");
217 verify_num_cards();
218 }
219
220 G1BufferNodeList G1DirtyCardQueueSet::take_all_completed_buffers() {
221 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
222 G1BufferNodeList result(_completed_buffers_head, _completed_buffers_tail, _num_cards);
223 _completed_buffers_head = NULL;
224 _completed_buffers_tail = NULL;
225 _num_cards = 0;
226 return result;
227 }
228
229 bool G1DirtyCardQueueSet::refine_buffer(BufferNode* node,
230 uint worker_id,
231 size_t* total_refined_cards) {
232 G1RemSet* rem_set = G1CollectedHeap::heap()->rem_set();
233 size_t size = buffer_size();
234 void** buffer = BufferNode::make_buffer_from_node(node);
235 size_t i = node->index();
236 assert(i <= size, "invariant");
237 for ( ; (i < size) && !SuspendibleThreadSet::should_yield(); ++i) {
238 CardTable::CardValue* cp = static_cast<CardTable::CardValue*>(buffer[i]);
239 rem_set->refine_card_concurrently(cp, worker_id);
240 }
241 *total_refined_cards += (i - node->index());
242 node->set_index(i);
243 return i == size;
244 }
245
246 #ifndef ASSERT
247 #define assert_fully_consumed(node, buffer_size)
248 #else
249 #define assert_fully_consumed(node, buffer_size) \
250 do { \
251 size_t _afc_index = (node)->index(); \
252 size_t _afc_size = (buffer_size); \
253 assert(_afc_index == _afc_size, \
254 "Buffer was not fully consumed as claimed: index: " \
255 SIZE_FORMAT ", size: " SIZE_FORMAT, \
256 _afc_index, _afc_size); \
257 } while (0)
258 #endif // ASSERT
259
260 bool G1DirtyCardQueueSet::process_or_enqueue_completed_buffer(BufferNode* node) {
261 if (Thread::current()->is_Java_thread()) {
262 // If the number of buffers exceeds the limit, make this Java
263 // thread do the processing itself. We don't lock to access
264 // buffer count or padding; it is fine to be imprecise here. The
265 // add of padding could overflow, which is treated as unlimited.
266 size_t limit = max_cards() + max_cards_padding();
267 if ((num_cards() > limit) && (limit >= max_cards())) {
268 if (mut_process_buffer(node)) {
269 return true;
270 }
271 }
272 }
273 enqueue_completed_buffer(node);
274 return false;
275 }
276
277 bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) {
278 uint worker_id = _free_ids.claim_par_id(); // temporarily claim an id
279 uint counter_index = worker_id - par_ids_start();
280 size_t* counter = &_mutator_refined_cards_counters[counter_index];
281 bool result = refine_buffer(node, worker_id, counter);
282 _free_ids.release_par_id(worker_id); // release the id
283
284 if (result) {
285 assert_fully_consumed(node, buffer_size());
286 }
287 return result;
288 }
289
290 bool G1DirtyCardQueueSet::refine_completed_buffer_concurrently(uint worker_id,
291 size_t stop_at,
292 size_t* total_refined_cards) {
293 BufferNode* node = get_completed_buffer(stop_at);
294 if (node == NULL) {
295 return false;
296 } else if (refine_buffer(node, worker_id, total_refined_cards)) {
297 assert_fully_consumed(node, buffer_size());
298 // Done with fully processed buffer.
299 deallocate_buffer(node);
300 return true;
301 } else {
302 // Return partially processed buffer to the queue.
303 enqueue_completed_buffer(node);
304 return true;
305 }
306 }
307
308 void G1DirtyCardQueueSet::abandon_logs() {
309 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
310 abandon_completed_buffers();
311
312 // Since abandon is done only at safepoints, we can safely manipulate
313 // these queues.
314 struct AbandonThreadLogClosure : public ThreadClosure {
315 virtual void do_thread(Thread* t) {
316 G1ThreadLocalData::dirty_card_queue(t).reset();
317 }
318 } closure;
319 Threads::threads_do(&closure);
320
321 G1BarrierSet::shared_dirty_card_queue().reset();
322 }
323
324 void G1DirtyCardQueueSet::concatenate_logs() {
325 // Iterate over all the threads, if we find a partial log add it to
326 // the global list of logs. Temporarily turn off the limit on the number
327 // of outstanding buffers.
328 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
329 size_t old_limit = max_cards();
330 set_max_cards(MaxCardsUnlimited);
331
332 struct ConcatenateThreadLogClosure : public ThreadClosure {
333 virtual void do_thread(Thread* t) {
334 G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t);
335 if (!dcq.is_empty()) {
336 dcq.flush();
337 }
338 }
339 } closure;
340 Threads::threads_do(&closure);
341
342 G1BarrierSet::shared_dirty_card_queue().flush();
343 set_max_cards(old_limit);
344 }