1 /*
2 * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/g1/g1BufferNodeList.hpp"
27 #include "gc/g1/g1CardTableEntryClosure.hpp"
28 #include "gc/g1/g1CollectedHeap.inline.hpp"
29 #include "gc/g1/g1DirtyCardQueue.hpp"
30 #include "gc/g1/g1FreeIdSet.hpp"
31 #include "gc/g1/g1RedirtyCardsQueue.hpp"
32 #include "gc/g1/g1RemSet.hpp"
33 #include "gc/g1/g1ThreadLocalData.hpp"
34 #include "gc/g1/heapRegionRemSet.hpp"
35 #include "gc/shared/suspendibleThreadSet.hpp"
36 #include "gc/shared/workgroup.hpp"
37 #include "runtime/atomic.hpp"
38 #include "runtime/flags/flagSetting.hpp"
39 #include "runtime/mutexLocker.hpp"
40 #include "runtime/safepoint.hpp"
41 #include "runtime/thread.inline.hpp"
42 #include "runtime/threadSMR.hpp"
43
44 // Closure used for updating remembered sets and recording references that
45 // point into the collection set while the mutator is running.
46 // Assumed to be only executed concurrently with the mutator. Yields via
47 // SuspendibleThreadSet after every card.
48 class G1RefineCardConcurrentlyClosure: public G1CardTableEntryClosure {
49 public:
50 bool do_card_ptr(CardValue* card_ptr, uint worker_i) {
51 G1CollectedHeap::heap()->rem_set()->refine_card_concurrently(card_ptr, worker_i);
52
53 if (SuspendibleThreadSet::should_yield()) {
54 // Caller will actually yield.
55 return false;
56 }
57 // Otherwise, we finished successfully; return true.
58 return true;
59 }
60 };
61
62 G1DirtyCardQueue::G1DirtyCardQueue(G1DirtyCardQueueSet* qset) :
63 // Dirty card queues are always active, so we create them with their
64 // active field set to true.
65 PtrQueue(qset, true /* active */)
66 { }
67
68 G1DirtyCardQueue::~G1DirtyCardQueue() {
69 flush();
70 }
71
72 void G1DirtyCardQueue::handle_completed_buffer() {
73 assert(_buf != NULL, "precondition");
74 BufferNode* node = BufferNode::make_node_from_buffer(_buf, index());
75 G1DirtyCardQueueSet* dcqs = dirty_card_qset();
76 if (dcqs->process_or_enqueue_completed_buffer(node)) {
77 reset(); // Buffer fully processed, reset index.
78 } else {
79 allocate_buffer(); // Buffer enqueued, get a new one.
80 }
81 }
82
83 G1DirtyCardQueueSet::G1DirtyCardQueueSet(bool notify_when_complete) :
84 PtrQueueSet(),
85 _cbl_mon(NULL),
86 _completed_buffers_head(NULL),
87 _completed_buffers_tail(NULL),
88 _num_entries_in_completed_buffers(0),
89 _process_completed_buffers_threshold(ProcessCompletedBuffersThresholdNever),
90 _process_completed_buffers(false),
91 _notify_when_complete(notify_when_complete),
92 _max_completed_buffers(MaxCompletedBuffersUnlimited),
93 _completed_buffers_padding(0),
94 _free_ids(0, num_par_ids()),
95 _processed_buffers_mut(0),
96 _processed_buffers_rs_thread(0)
97 {
98 _all_active = true;
99 }
100
101 G1DirtyCardQueueSet::~G1DirtyCardQueueSet() {
102 abandon_completed_buffers();
103 }
104
105 // Determines how many mutator threads can process the buffers in parallel.
106 uint G1DirtyCardQueueSet::num_par_ids() {
107 return (uint)os::initial_active_processor_count();
108 }
109
110 void G1DirtyCardQueueSet::initialize(Monitor* cbl_mon,
111 BufferNode::Allocator* allocator) {
112 PtrQueueSet::initialize(allocator);
113 assert(_cbl_mon == NULL, "Init order issue?");
114 _cbl_mon = cbl_mon;
115 }
116
117 void G1DirtyCardQueueSet::handle_zero_index_for_thread(Thread* t) {
118 G1ThreadLocalData::dirty_card_queue(t).handle_zero_index();
119 }
120
121 void G1DirtyCardQueueSet::enqueue_completed_buffer(BufferNode* cbn) {
122 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
123 cbn->set_next(NULL);
124 if (_completed_buffers_tail == NULL) {
125 assert(_completed_buffers_head == NULL, "Well-formedness");
126 _completed_buffers_head = cbn;
127 _completed_buffers_tail = cbn;
128 } else {
129 _completed_buffers_tail->set_next(cbn);
130 _completed_buffers_tail = cbn;
131 }
132 _num_entries_in_completed_buffers += buffer_size() - cbn->index();
133
134 if (!process_completed_buffers() &&
135 (num_completed_buffers() > process_completed_buffers_threshold())) {
136 set_process_completed_buffers(true);
137 if (_notify_when_complete) {
138 _cbl_mon->notify_all();
139 }
140 }
141 verify_num_entries_in_completed_buffers();
142 }
143
144 BufferNode* G1DirtyCardQueueSet::get_completed_buffer(size_t stop_at) {
145 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
146
147 if (num_completed_buffers() <= stop_at) {
148 return NULL;
149 }
150
151 assert(num_completed_buffers() > 0, "invariant");
152 assert(_completed_buffers_head != NULL, "invariant");
153 assert(_completed_buffers_tail != NULL, "invariant");
154
155 BufferNode* bn = _completed_buffers_head;
156 _num_entries_in_completed_buffers -= buffer_size() - bn->index();
157 _completed_buffers_head = bn->next();
158 if (_completed_buffers_head == NULL) {
159 assert(num_completed_buffers() == 0, "invariant");
160 _completed_buffers_tail = NULL;
161 set_process_completed_buffers(false);
162 }
163 verify_num_entries_in_completed_buffers();
164 bn->set_next(NULL);
165 return bn;
166 }
167
168 #ifdef ASSERT
169 void G1DirtyCardQueueSet::verify_num_entries_in_completed_buffers() const {
170 size_t actual = 0;
171 BufferNode* cur = _completed_buffers_head;
172 while (cur != NULL) {
173 actual += buffer_size() - cur->index();
174 cur = cur->next();
175 }
176 assert(actual == _num_entries_in_completed_buffers,
177 "Num entries in completed buffers should be " SIZE_FORMAT " but are " SIZE_FORMAT,
178 _num_entries_in_completed_buffers, actual);
179 }
180 #endif
181
182 void G1DirtyCardQueueSet::abandon_completed_buffers() {
183 BufferNode* buffers_to_delete = NULL;
184 {
185 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
186 buffers_to_delete = _completed_buffers_head;
187 _completed_buffers_head = NULL;
188 _completed_buffers_tail = NULL;
189 _num_entries_in_completed_buffers = 0;
190 set_process_completed_buffers(false);
191 }
192 while (buffers_to_delete != NULL) {
193 BufferNode* bn = buffers_to_delete;
194 buffers_to_delete = bn->next();
195 bn->set_next(NULL);
196 deallocate_buffer(bn);
197 }
198 }
199
200 void G1DirtyCardQueueSet::notify_if_necessary() {
201 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
202 if (num_completed_buffers() > process_completed_buffers_threshold()) {
203 set_process_completed_buffers(true);
204 if (_notify_when_complete)
205 _cbl_mon->notify();
206 }
207 }
208
209 // Merge lists of buffers. Notify the processing threads.
210 // The source queue is emptied as a result. The queues
211 // must share the monitor.
212 void G1DirtyCardQueueSet::merge_bufferlists(G1RedirtyCardsQueueSet* src) {
213 assert(allocator() == src->allocator(), "precondition");
214 const G1BufferNodeList from = src->take_all_completed_buffers();
215 if (from._head == NULL) return;
216
217 MutexLocker x(_cbl_mon, Mutex::_no_safepoint_check_flag);
218 if (_completed_buffers_tail == NULL) {
219 assert(_completed_buffers_head == NULL, "Well-formedness");
220 _completed_buffers_head = from._head;
221 _completed_buffers_tail = from._tail;
222 } else {
223 assert(_completed_buffers_head != NULL, "Well formedness");
224 _completed_buffers_tail->set_next(from._head);
225 _completed_buffers_tail = from._tail;
226 }
227 _num_entries_in_completed_buffers += from._entry_count;
228
229 assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL ||
230 _completed_buffers_head != NULL && _completed_buffers_tail != NULL,
231 "Sanity");
232 verify_num_entries_in_completed_buffers();
233 }
234
235 bool G1DirtyCardQueueSet::apply_closure_to_buffer(G1CardTableEntryClosure* cl,
236 BufferNode* node,
237 uint worker_i) {
238 if (cl == NULL) return true;
239 bool result = true;
240 void** buf = BufferNode::make_buffer_from_node(node);
241 size_t i = node->index();
242 size_t limit = buffer_size();
243 for ( ; i < limit; ++i) {
244 CardTable::CardValue* card_ptr = static_cast<CardTable::CardValue*>(buf[i]);
245 assert(card_ptr != NULL, "invariant");
246 if (!cl->do_card_ptr(card_ptr, worker_i)) {
247 result = false; // Incomplete processing.
248 break;
249 }
250 }
251 assert(i <= buffer_size(), "invariant");
252 node->set_index(i);
253 return result;
254 }
255
256 #ifndef ASSERT
257 #define assert_fully_consumed(node, buffer_size)
258 #else
259 #define assert_fully_consumed(node, buffer_size) \
260 do { \
261 size_t _afc_index = (node)->index(); \
262 size_t _afc_size = (buffer_size); \
263 assert(_afc_index == _afc_size, \
264 "Buffer was not fully consumed as claimed: index: " \
265 SIZE_FORMAT ", size: " SIZE_FORMAT, \
266 _afc_index, _afc_size); \
267 } while (0)
268 #endif // ASSERT
269
270 bool G1DirtyCardQueueSet::process_or_enqueue_completed_buffer(BufferNode* node) {
271 if (Thread::current()->is_Java_thread()) {
272 // If the number of buffers exceeds the limit, make this Java
273 // thread do the processing itself. We don't lock to access
274 // buffer count or padding; it is fine to be imprecise here. The
275 // add of padding could overflow, which is treated as unlimited.
276 size_t max_buffers = max_completed_buffers();
277 size_t limit = max_buffers + completed_buffers_padding();
278 if ((num_completed_buffers() > limit) && (limit >= max_buffers)) {
279 if (mut_process_buffer(node)) {
280 return true;
281 }
282 }
283 }
284 enqueue_completed_buffer(node);
285 return false;
286 }
287
288 bool G1DirtyCardQueueSet::mut_process_buffer(BufferNode* node) {
289 uint worker_id = _free_ids.claim_par_id(); // temporarily claim an id
290 G1RefineCardConcurrentlyClosure cl;
291 bool result = apply_closure_to_buffer(&cl, node, worker_id);
292 _free_ids.release_par_id(worker_id); // release the id
293
294 if (result) {
295 assert_fully_consumed(node, buffer_size());
296 Atomic::inc(&_processed_buffers_mut);
297 }
298 return result;
299 }
300
301 bool G1DirtyCardQueueSet::refine_completed_buffer_concurrently(uint worker_i, size_t stop_at) {
302 G1RefineCardConcurrentlyClosure cl;
303 return apply_closure_to_completed_buffer(&cl, worker_i, stop_at, false);
304 }
305
306 bool G1DirtyCardQueueSet::apply_closure_during_gc(G1CardTableEntryClosure* cl, uint worker_i) {
307 assert_at_safepoint();
308 return apply_closure_to_completed_buffer(cl, worker_i, 0, true);
309 }
310
311 bool G1DirtyCardQueueSet::apply_closure_to_completed_buffer(G1CardTableEntryClosure* cl,
312 uint worker_i,
313 size_t stop_at,
314 bool during_pause) {
315 assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause");
316 BufferNode* nd = get_completed_buffer(stop_at);
317 if (nd == NULL) {
318 return false;
319 } else {
320 if (apply_closure_to_buffer(cl, nd, worker_i)) {
321 assert_fully_consumed(nd, buffer_size());
322 // Done with fully processed buffer.
323 deallocate_buffer(nd);
324 Atomic::inc(&_processed_buffers_rs_thread);
325 } else {
326 // Return partially processed buffer to the queue.
327 guarantee(!during_pause, "Should never stop early");
328 enqueue_completed_buffer(nd);
329 }
330 return true;
331 }
332 }
333
334 void G1DirtyCardQueueSet::abandon_logs() {
335 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
336 abandon_completed_buffers();
337
338 // Since abandon is done only at safepoints, we can safely manipulate
339 // these queues.
340 struct AbandonThreadLogClosure : public ThreadClosure {
341 virtual void do_thread(Thread* t) {
342 G1ThreadLocalData::dirty_card_queue(t).reset();
343 }
344 } closure;
345 Threads::threads_do(&closure);
346
347 G1BarrierSet::shared_dirty_card_queue().reset();
348 }
349
350 void G1DirtyCardQueueSet::concatenate_logs() {
351 // Iterate over all the threads, if we find a partial log add it to
352 // the global list of logs. Temporarily turn off the limit on the number
353 // of outstanding buffers.
354 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
355 size_t old_limit = max_completed_buffers();
356 set_max_completed_buffers(MaxCompletedBuffersUnlimited);
357
358 struct ConcatenateThreadLogClosure : public ThreadClosure {
359 virtual void do_thread(Thread* t) {
360 G1DirtyCardQueue& dcq = G1ThreadLocalData::dirty_card_queue(t);
361 if (!dcq.is_empty()) {
362 dcq.flush();
363 }
364 }
365 } closure;
366 Threads::threads_do(&closure);
367
368 G1BarrierSet::shared_dirty_card_queue().flush();
369 set_max_completed_buffers(old_limit);
370 }