1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 26 #include "gc/shared/gcTraceTime.inline.hpp" 27 #include "gc/shared/workgroup.hpp" 28 #include "gc/shared/taskqueue.inline.hpp" 29 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 30 #include "gc/shenandoah/shenandoahCollectionSet.hpp" 31 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 32 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp" 33 #include "gc/shenandoah/shenandoahFreeSet.hpp" 34 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 35 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" 36 #include "gc/shenandoah/shenandoahHeap.hpp" 37 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 38 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 39 #include "gc/shenandoah/shenandoahPartialGC.hpp" 40 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 41 #include "gc/shenandoah/shenandoahStringDedup.hpp" 42 #include "gc/shenandoah/shenandoahTaskqueue.hpp" 43 #include "gc/shenandoah/shenandoahUtils.hpp" 44 #include "gc/shenandoah/shenandoahVerifier.hpp" 45 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 47 48 #include "memory/iterator.hpp" 49 #include "runtime/safepoint.hpp" 50 51 class ShenandoahPartialEvacuateUpdateRootsClosure : public OopClosure { 52 ShenandoahPartialGC* _partial_gc; 53 Thread* _thread; 54 ShenandoahObjToScanQueue* _queue; 55 private: 56 template <class T> 57 void do_oop_work(T* p) { _partial_gc->process_oop<T, false>(p, _thread, _queue); } 58 public: 59 ShenandoahPartialEvacuateUpdateRootsClosure(ShenandoahObjToScanQueue* q) : 60 _partial_gc(ShenandoahHeap::heap()->partial_gc()), 61 _thread(Thread::current()), _queue(q) {} 62 void do_oop(oop* p) { 63 assert(! ShenandoahHeap::heap()->is_in_reserved(p), "sanity"); 64 do_oop_work(p); 65 } 66 void do_oop(narrowOop* p) { do_oop_work(p); } 67 }; 68 69 class ShenandoahPartialCollectionTask : public AbstractGangTask { 70 private: 71 ShenandoahRootProcessor* _rp; 72 ParallelTaskTerminator* _terminator; 73 ShenandoahHeapRegionSet* _root_regions; 74 ShenandoahHeap* _heap; 75 public: 76 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp, 77 ParallelTaskTerminator* terminator, 78 ShenandoahHeapRegionSet* root_regions) : 79 AbstractGangTask("Shenandoah Partial Collection"), 80 _rp(rp), _terminator(terminator), _root_regions(root_regions), 81 _heap(ShenandoahHeap::heap()) {} 82 83 void work(uint worker_id) { 84 ShenandoahObjToScanQueueSet* queues = _heap->partial_gc()->task_queues(); 85 ShenandoahObjToScanQueue* q = queues->queue(worker_id); 86 87 // Step 1: Process ordinary GC roots. 88 { 89 ShenandoahPartialEvacuateUpdateRootsClosure roots_cl(q); 90 CLDToOopClosure cld_cl(&roots_cl); 91 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); 92 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id); 93 } 94 if (check_and_handle_cancelled_gc()) return; 95 96 ShenandoahPartialEvacuateUpdateHeapClosure cl(q); 97 98 // Step 2: Process all root regions. 99 { 100 ShenandoahHeapRegion* r = _root_regions->claim_next(); 101 while (r != NULL) { 102 assert(r->is_root(), "must be root region"); 103 _heap->marked_object_oop_safe_iterate(r, &cl); 104 if (check_and_handle_cancelled_gc()) return; 105 r = _root_regions->claim_next(); 106 } 107 } 108 if (check_and_handle_cancelled_gc()) return; 109 110 // Step 3: Drain all outstanding work in queues. 111 { 112 int seed = 17; 113 ShenandoahMarkTask task; 114 115 uintx stride = ShenandoahMarkLoopStride; 116 117 while (true) { 118 if (check_and_handle_cancelled_gc()) return; 119 120 for (uint i = 0; i < stride; i++) { 121 if ((q->pop_buffer(task) || 122 q->pop_local(task) || 123 q->pop_overflow(task) || 124 queues->steal(worker_id, &seed, task))) { 125 oop obj = task.obj(); 126 assert(!oopDesc::is_null(obj), "must not be null"); 127 obj->oop_iterate(&cl); 128 } else { 129 if (_terminator->offer_termination()) return; 130 } 131 } 132 } 133 } 134 } 135 136 private: 137 bool check_and_handle_cancelled_gc() { 138 if (_heap->cancelled_concgc()) { 139 ShenandoahCancelledTerminatorTerminator tt; 140 while (! _terminator->offer_termination(&tt)); 141 return true; 142 } 143 return false; 144 } 145 }; 146 147 class ShenandoahPartialCollectionCleanupTask : public AbstractGangTask { 148 private: 149 ShenandoahHeap* _heap; 150 public: 151 ShenandoahPartialCollectionCleanupTask() : 152 AbstractGangTask("Shenandoah Partial Collection Cleanup"), 153 _heap(ShenandoahHeap::heap()) { 154 _heap->collection_set()->clear_current_index(); 155 } 156 157 void work(uint worker_id) { 158 ShenandoahCollectionSet* cset = _heap->collection_set(); 159 ShenandoahHeapRegion* r = cset->claim_next(); 160 while (r != NULL) { 161 HeapWord* bottom = r->bottom(); 162 HeapWord* top = _heap->top_at_mark_start(r->bottom()); 163 if (top > bottom) { 164 _heap->mark_bit_map()->clear_range_large(MemRegion(bottom, top)); 165 } 166 r = cset->claim_next(); 167 } 168 } 169 170 }; 171 172 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, size_t num_regions) : 173 _heap(heap), 174 _matrix(heap->connection_matrix()), 175 _root_regions(new ShenandoahHeapRegionSet(num_regions)), 176 _task_queues(new ShenandoahObjToScanQueueSet(heap->max_workers())) { 177 178 assert(_matrix != NULL, "need matrix"); 179 180 uint num_queues = heap->max_workers(); 181 for (uint i = 0; i < num_queues; ++i) { 182 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); 183 task_queue->initialize(); 184 _task_queues->register_queue(i, task_queue); 185 } 186 187 from_idxs = NEW_C_HEAP_ARRAY(size_t, ShenandoahPartialInboundThreshold, mtGC); 188 } 189 190 ShenandoahPartialGC::~ShenandoahPartialGC() { 191 FREE_C_HEAP_ARRAY(size_t, from_idxs); 192 } 193 194 bool ShenandoahPartialGC::prepare() { 195 _heap->collection_set()->clear(); 196 assert(_heap->collection_set()->count() == 0, "collection set not clear"); 197 198 _heap->ensure_parsability(true); 199 200 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 201 202 if (UseShenandoahMatrix && PrintShenandoahMatrix) { 203 LogTarget(Info, gc) lt; 204 LogStream ls(lt); 205 _heap->connection_matrix()->print_on(&ls); 206 } 207 208 ShenandoahHeapRegionSet* regions = _heap->regions(); 209 ShenandoahCollectionSet* collection_set = _heap->collection_set(); 210 size_t num_regions = _heap->num_regions(); 211 212 // First pass: reset all roots 213 for (uint to_idx = 0; to_idx < num_regions; to_idx++) { 214 ShenandoahHeapRegion* r = regions->get(to_idx); 215 r->set_root(false); 216 } 217 218 // Second pass: find collection set, and mark root candidates 219 _heap->shenandoahPolicy()->choose_collection_set(collection_set, true); 220 221 // Shortcut: no cset, bail 222 size_t num_cset = collection_set->count(); 223 224 if (num_cset == 0) { 225 log_info(gc, ergo)("No regions with fewer inbound connections than threshold (" UINTX_FORMAT ")", 226 ShenandoahPartialInboundThreshold); 227 return false; 228 } 229 230 // Final pass: rebuild free set and region set. 231 ShenandoahFreeSet* _free_regions = _heap->free_regions(); 232 _root_regions->clear(); 233 _free_regions->clear(); 234 235 assert(_root_regions->count() == 0, "must be cleared"); 236 237 for (uint from_idx = 0; from_idx < num_regions; from_idx++) { 238 ShenandoahHeapRegion* r = regions->get(from_idx); 239 if (r->is_alloc_allowed()) { 240 _free_regions->add_region(r); 241 } 242 if (r->is_root() && !r->in_collection_set()) { 243 _root_regions->add_region(r); 244 matrix->clear_region_outbound(from_idx); 245 246 // Since root region can be allocated at, we should bound the scans 247 // in it at current top. Otherwise, one thread may evacuate objects 248 // to that root region, while another would try to scan newly evac'ed 249 // objects under the race. 250 r->set_concurrent_iteration_safe_limit(r->top()); 251 } 252 } 253 254 log_info(gc,ergo)("Got "SIZE_FORMAT" collection set regions, "SIZE_FORMAT" root regions", 255 collection_set->count(), _root_regions->count()); 256 257 return true; 258 } 259 260 void ShenandoahPartialGC::do_partial_collection() { 261 assert(SafepointSynchronize::is_at_safepoint(), "STW partial GC"); 262 ShenandoahWorkerScope partial_gc_scope(_heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_stw_partial()); 263 264 _heap->set_alloc_seq_gc_start(); 265 266 ShenandoahGCSession session; 267 268 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true); 269 270 if (ShenandoahVerify) { 271 _heap->verifier()->verify_before_partial(); 272 } 273 274 bool has_work; 275 { 276 ShenandoahGCPhase phase_prepare(ShenandoahPhaseTimings::partial_gc_prepare); 277 ShenandoahHeapLocker lock(_heap->lock()); 278 has_work = prepare(); 279 } 280 281 if (!has_work) { 282 reset(); 283 return; 284 } 285 286 { 287 ShenandoahGCPhase phase_work(ShenandoahPhaseTimings::partial_gc_work); 288 assert(_task_queues->is_empty(), "queues must be empty after partial GC"); 289 290 #if defined(COMPILER2) || INCLUDE_JVMCI 291 DerivedPointerTable::clear(); 292 #endif 293 294 { 295 uint nworkers = _heap->workers()->active_workers(); 296 ShenandoahRootProcessor rp(_heap, nworkers, ShenandoahPhaseTimings::partial_gc_work); 297 298 if (UseShenandoahOWST) { 299 ShenandoahTaskTerminator terminator(nworkers, task_queues()); 300 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions); 301 _heap->workers()->run_task(&partial_task); 302 } else { 303 ParallelTaskTerminator terminator(nworkers, task_queues()); 304 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions); 305 _heap->workers()->run_task(&partial_task); 306 } 307 } 308 309 #if defined(COMPILER2) || INCLUDE_JVMCI 310 DerivedPointerTable::update_pointers(); 311 #endif 312 if (_heap->cancelled_concgc()) { 313 _task_queues->clear(); 314 } 315 assert(_task_queues->is_empty(), "queues must be empty after partial GC"); 316 } 317 318 319 if (! _heap->cancelled_concgc()) { 320 if (ShenandoahStringDedup::is_enabled()) { 321 ShenandoahGCPhase update_str_dedup_table(ShenandoahPhaseTimings::partial_gc_update_str_dedup_table); 322 ShenandoahStringDedup::parallel_partial_update_or_unlink(); 323 } 324 325 { 326 ShenandoahGCPhase phase_recycle(ShenandoahPhaseTimings::partial_gc_recycle); 327 328 ShenandoahCollectionSet* cset = _heap->collection_set(); 329 330 { 331 ShenandoahHeapLocker lock(_heap->lock()); 332 333 ShenandoahPartialCollectionCleanupTask cleanup; 334 _heap->workers()->run_task(&cleanup); 335 336 // Trash everything when bitmaps are cleared. 337 cset->clear_current_index(); 338 ShenandoahHeapRegion* r; 339 while((r = cset->next()) != NULL) { 340 r->make_trash(); 341 } 342 343 reset(); 344 } 345 } 346 347 if (ShenandoahVerify) { 348 _heap->verifier()->verify_after_partial(); 349 } 350 } else { 351 // Fixup roots to make them consistent 352 _heap->fixup_roots(); 353 } 354 } 355 356 void ShenandoahPartialGC::reset() { 357 _heap->collection_set()->clear(); 358 359 _root_regions->clear_current_index(); 360 ShenandoahHeapRegion* r; 361 while((r = _root_regions->claim_next()) != NULL) { 362 r->set_root(false); 363 } 364 _root_regions->clear(); 365 } 366 367 ShenandoahObjToScanQueueSet* ShenandoahPartialGC::task_queues() { 368 return _task_queues; 369 }