1 /* 2 * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "jvm.h" 27 #include "gc/shared/satbMarkQueue.hpp" 28 #include "gc/shared/collectedHeap.hpp" 29 #include "logging/log.hpp" 30 #include "memory/allocation.inline.hpp" 31 #include "oops/oop.inline.hpp" 32 #include "runtime/mutexLocker.hpp" 33 #include "runtime/safepoint.hpp" 34 #include "runtime/thread.hpp" 35 #include "runtime/threadSMR.hpp" 36 #include "runtime/vmThread.hpp" 37 38 SATBMarkQueue::SATBMarkQueue(SATBMarkQueueSet* qset, bool permanent) : 39 // SATB queues are only active during marking cycles. We create 40 // them with their active field set to false. If a thread is 41 // created during a cycle and its SATB queue needs to be activated 42 // before the thread starts running, we'll need to set its active 43 // field to true. This is done in G1SBarrierSet::on_thread_attach(). 44 PtrQueue(qset, permanent, false /* active */) 45 { } 46 47 void SATBMarkQueue::flush() { 48 // Filter now to possibly save work later. If filtering empties the 49 // buffer then flush_impl can deallocate the buffer. 50 filter(); 51 flush_impl(); 52 } 53 54 // This method removes entries from a SATB buffer that will not be 55 // useful to the concurrent marking threads. Entries are retained if 56 // they require marking and are not already marked. Retained entries 57 // are compacted toward the top of the buffer. 58 void SATBMarkQueue::filter() { 59 CollectedHeap* heap = Universe::heap(); 60 void** buf = _buf; 61 62 if (buf == NULL) { 63 // nothing to do 64 return; 65 } 66 67 // Two-fingered compaction toward the end. 68 void** src = &buf[index()]; 69 void** dst = &buf[capacity()]; 70 assert(src <= dst, "invariant"); 71 for ( ; src < dst; ++src) { 72 // Search low to high for an entry to keep. 73 void* entry = *src; 74 if (heap->retain_satb_entry(entry)) { 75 // Found keeper. Search high to low for an entry to discard. 76 while (src < --dst) { 77 if (!heap->retain_satb_entry(*dst)) { 78 *dst = entry; // Replace discard with keeper. 79 break; 80 } 81 } 82 // If discard search failed (src == dst), the outer loop will also end. 83 } 84 } 85 // dst points to the lowest retained entry, or the end of the buffer 86 // if all the entries were filtered out. 87 set_index(dst - buf); 88 } 89 90 // This method will first apply the above filtering to the buffer. If 91 // post-filtering a large enough chunk of the buffer has been cleared 92 // we can re-use the buffer (instead of enqueueing it) and we can just 93 // allow the mutator to carry on executing using the same buffer 94 // instead of replacing it. 95 96 bool SATBMarkQueue::should_enqueue_buffer() { 97 assert(_lock == NULL || _lock->owned_by_self(), 98 "we should have taken the lock before calling this"); 99 100 // If G1SATBBufferEnqueueingThresholdPercent == 0 we could skip filtering. 101 102 // This method should only be called if there is a non-NULL buffer 103 // that is full. 104 assert(index() == 0, "pre-condition"); 105 assert(_buf != NULL, "pre-condition"); 106 107 filter(); 108 109 size_t cap = capacity(); 110 size_t percent_used = ((cap - index()) * 100) / cap; 111 bool should_enqueue = percent_used > G1SATBBufferEnqueueingThresholdPercent; 112 return should_enqueue; 113 } 114 115 void SATBMarkQueue::apply_closure_and_empty(SATBBufferClosure* cl) { 116 assert(SafepointSynchronize::is_at_safepoint(), 117 "SATB queues must only be processed at safepoints"); 118 if (_buf != NULL) { 119 cl->do_buffer(&_buf[index()], size()); 120 reset(); 121 } 122 } 123 124 #ifndef PRODUCT 125 // Helpful for debugging 126 127 static void print_satb_buffer(const char* name, 128 void** buf, 129 size_t index, 130 size_t capacity) { 131 tty->print_cr(" SATB BUFFER [%s] buf: " PTR_FORMAT " index: " SIZE_FORMAT 132 " capacity: " SIZE_FORMAT, 133 name, p2i(buf), index, capacity); 134 } 135 136 void SATBMarkQueue::print(const char* name) { 137 print_satb_buffer(name, _buf, index(), capacity()); 138 } 139 140 #endif // PRODUCT 141 142 SATBMarkQueueSet::SATBMarkQueueSet() : 143 PtrQueueSet(), 144 _shared_satb_queue(this, true /* permanent */) { } 145 146 void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock, 147 int process_completed_threshold, 148 Mutex* lock) { 149 PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, -1); 150 _shared_satb_queue.set_lock(lock); 151 } 152 153 void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) { 154 Universe::heap()->satb_mark_queue(t)->handle_zero_index(); 155 } 156 157 #ifdef ASSERT 158 void SATBMarkQueueSet::dump_active_states(bool expected_active) { 159 log_error(gc, verify)("Expected SATB active state: %s", expected_active ? "ACTIVE" : "INACTIVE"); 160 log_error(gc, verify)("Actual SATB active states:"); 161 log_error(gc, verify)(" Queue set: %s", is_active() ? "ACTIVE" : "INACTIVE"); 162 CollectedHeap* heap = Universe::heap(); 163 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { 164 log_error(gc, verify)(" Thread \"%s\" queue: %s", t->name(), heap->satb_mark_queue(t)->is_active() ? "ACTIVE" : "INACTIVE"); 165 } 166 log_error(gc, verify)(" Shared queue: %s", shared_satb_queue()->is_active() ? "ACTIVE" : "INACTIVE"); 167 } 168 169 void SATBMarkQueueSet::verify_active_states(bool expected_active) { 170 // Verify queue set state 171 if (is_active() != expected_active) { 172 dump_active_states(expected_active); 173 guarantee(false, "SATB queue set has an unexpected active state"); 174 } 175 176 // Verify thread queue states 177 CollectedHeap* heap = Universe::heap(); 178 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { 179 if (heap->satb_mark_queue(t)->is_active() != expected_active) { 180 dump_active_states(expected_active); 181 guarantee(false, "Thread SATB queue has an unexpected active state"); 182 } 183 } 184 185 // Verify shared queue state 186 if (shared_satb_queue()->is_active() != expected_active) { 187 dump_active_states(expected_active); 188 guarantee(false, "Shared SATB queue has an unexpected active state"); 189 } 190 } 191 #endif // ASSERT 192 193 void SATBMarkQueueSet::set_active_all_threads(bool active, bool expected_active) { 194 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 195 #ifdef ASSERT 196 verify_active_states(expected_active); 197 #endif // ASSERT 198 _all_active = active; 199 CollectedHeap* heap = Universe::heap(); 200 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { 201 heap->satb_mark_queue(t)->set_active(active); 202 } 203 shared_satb_queue()->set_active(active); 204 } 205 206 void SATBMarkQueueSet::filter_thread_buffers() { 207 CollectedHeap* heap = Universe::heap(); 208 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { 209 heap->satb_mark_queue(t)->filter(); 210 } 211 shared_satb_queue()->filter(); 212 } 213 214 bool SATBMarkQueueSet::apply_closure_to_completed_buffer(SATBBufferClosure* cl) { 215 BufferNode* nd = NULL; 216 { 217 MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); 218 if (_completed_buffers_head != NULL) { 219 nd = _completed_buffers_head; 220 _completed_buffers_head = nd->next(); 221 if (_completed_buffers_head == NULL) _completed_buffers_tail = NULL; 222 _n_completed_buffers--; 223 if (_n_completed_buffers == 0) _process_completed = false; 224 } 225 } 226 if (nd != NULL) { 227 void **buf = BufferNode::make_buffer_from_node(nd); 228 size_t index = nd->index(); 229 size_t size = buffer_size(); 230 assert(index <= size, "invariant"); 231 cl->do_buffer(buf + index, size - index); 232 deallocate_buffer(nd); 233 return true; 234 } else { 235 return false; 236 } 237 } 238 239 #ifndef PRODUCT 240 // Helpful for debugging 241 242 #define SATB_PRINTER_BUFFER_SIZE 256 243 244 void SATBMarkQueueSet::print_all(const char* msg) { 245 char buffer[SATB_PRINTER_BUFFER_SIZE]; 246 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 247 248 tty->cr(); 249 tty->print_cr("SATB BUFFERS [%s]", msg); 250 251 BufferNode* nd = _completed_buffers_head; 252 int i = 0; 253 while (nd != NULL) { 254 void** buf = BufferNode::make_buffer_from_node(nd); 255 jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Enqueued: %d", i); 256 print_satb_buffer(buffer, buf, nd->index(), buffer_size()); 257 nd = nd->next(); 258 i += 1; 259 } 260 261 CollectedHeap* heap = Universe::heap(); 262 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { 263 jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Thread: %s", t->name()); 264 heap->satb_mark_queue(t)->print(buffer); 265 } 266 267 shared_satb_queue()->print("Shared"); 268 269 tty->cr(); 270 } 271 #endif // PRODUCT 272 273 void SATBMarkQueueSet::abandon_partial_marking() { 274 BufferNode* buffers_to_delete = NULL; 275 { 276 MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); 277 while (_completed_buffers_head != NULL) { 278 BufferNode* nd = _completed_buffers_head; 279 _completed_buffers_head = nd->next(); 280 nd->set_next(buffers_to_delete); 281 buffers_to_delete = nd; 282 } 283 _completed_buffers_tail = NULL; 284 _n_completed_buffers = 0; 285 DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked()); 286 } 287 while (buffers_to_delete != NULL) { 288 BufferNode* nd = buffers_to_delete; 289 buffers_to_delete = nd->next(); 290 deallocate_buffer(nd); 291 } 292 assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); 293 // So we can safely manipulate these queues. 294 CollectedHeap* heap = Universe::heap(); 295 for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { 296 heap->satb_mark_queue(t)->reset(); 297 } 298 shared_satb_queue()->reset(); 299 }