1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/shared/gcTraceTime.inline.hpp" 26 #include "gc/shared/workgroup.hpp" 27 #include "gc/shared/taskqueue.inline.hpp" 28 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 29 #include "gc/shenandoah/shenandoahCollectionSet.hpp" 30 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp" 31 #include "gc/shenandoah/shenandoahFreeSet.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" 33 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 34 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 35 #include "gc/shenandoah/shenandoahPartialGC.hpp" 36 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 37 #include "gc/shenandoah/shenandoahTaskqueue.hpp" 38 #include "memory/iterator.hpp" 39 40 class PartialEvacuateUpdateRootsClosure : public OopClosure { 41 ShenandoahPartialGC* _partial_gc; 42 Thread* _thread; 43 SCMObjToScanQueue* _queue; 44 private: 45 template <class T> 46 void do_oop_work(T* p) { _partial_gc->process_oop<T, false>(p, _thread, _queue); } 47 public: 48 PartialEvacuateUpdateRootsClosure(SCMObjToScanQueue* q) : 49 _partial_gc(ShenandoahHeap::heap()->partial_gc()), 50 _thread(Thread::current()), _queue(q) {} 51 void do_oop(oop* p) { 52 assert(! ShenandoahHeap::heap()->is_in_reserved(p), "sanity"); 53 do_oop_work(p); 54 } 55 void do_oop(narrowOop* p) { do_oop_work(p); } 56 }; 57 58 class ShenandoahPartialCollectionTask : public AbstractGangTask { 59 private: 60 ShenandoahRootProcessor* _rp; 61 ParallelTaskTerminator* _terminator; 62 ShenandoahHeapRegionSet* _root_regions; 63 ShenandoahHeap* _heap; 64 public: 65 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp, 66 ParallelTaskTerminator* terminator, 67 ShenandoahHeapRegionSet* root_regions) : 68 AbstractGangTask("Shenandoah Partial Collection"), 69 _rp(rp), _terminator(terminator), _root_regions(root_regions), 70 _heap(ShenandoahHeap::heap()) {} 71 72 void work(uint worker_id) { 73 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 74 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues(); 75 SCMObjToScanQueue* q = queues->queue(worker_id); 76 77 // Step 1: Process ordinary GC roots. 78 { 79 PartialEvacuateUpdateRootsClosure roots_cl(q); 80 CLDToOopClosure cld_cl(&roots_cl); 81 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); 82 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id); 83 } 84 if (check_and_handle_cancelled_gc()) return; 85 86 PartialEvacuateUpdateHeapClosure cl(q); 87 88 // Step 2: Process all root regions. 89 { 90 ShenandoahHeapRegion* r = _root_regions->claim_next(); 91 while (r != NULL) { 92 assert(r->is_root(), "must be root region"); 93 if (! r->is_humongous_continuation()) { 94 _heap->marked_object_oop_iterate(r, &cl); 95 } 96 r->set_root(false); 97 if (check_and_handle_cancelled_gc()) return; 98 r = _root_regions->claim_next(); 99 } 100 } 101 if (check_and_handle_cancelled_gc()) return; 102 103 // Step 3: Drain all outstanding work in queues. 104 { 105 int seed = 17; 106 SCMTask task; 107 108 uint stride = ShenandoahMarkLoopStride; 109 110 while (true) { 111 if (check_and_handle_cancelled_gc()) return; 112 113 for (uint i = 0; i < stride; i++) { 114 if ((q->pop_buffer(task) || 115 q->pop_local(task) || 116 q->pop_overflow(task) || 117 queues->steal(worker_id, &seed, task))) { 118 oop obj = task.obj(); 119 assert(!oopDesc::is_null(obj), "must not be null"); 120 obj->oop_iterate(&cl); 121 } else { 122 if (_terminator->offer_termination()) return; 123 } 124 } 125 } 126 } 127 } 128 129 private: 130 bool check_and_handle_cancelled_gc() { 131 if (_heap->cancelled_concgc()) { 132 ShenandoahCancelledTerminatorTerminator tt; 133 while (! _terminator->offer_termination(&tt)); 134 return true; 135 } 136 return false; 137 } 138 }; 139 140 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, size_t max_regions) : 141 _heap(heap), 142 _matrix(heap->connection_matrix()), 143 _root_regions(new ShenandoahHeapRegionSet(max_regions)), 144 _task_queues(new SCMObjToScanQueueSet(heap->max_workers())) { 145 146 assert(_matrix != NULL, "need matrix"); 147 148 uint num_queues = heap->max_workers(); 149 for (uint i = 0; i < num_queues; ++i) { 150 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue(); 151 task_queue->initialize(); 152 _task_queues->register_queue(i, task_queue); 153 } 154 155 } 156 157 void ShenandoahPartialGC::prepare() { 158 159 if (ShenandoahVerify || VerifyShenandoahMatrix) { 160 log_debug(gc, ergo)("Verifying before partial collection"); 161 _heap->verify_heap_reachable_at_safepoint(); 162 log_debug(gc, ergo)("Verification passed"); 163 } 164 165 _heap->collection_set()->clear(); 166 assert(_heap->collection_set()->count() == 0, "collection set not clear"); 167 168 _heap->ensure_parsability(true); 169 170 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 171 ShenandoahHeapRegionSet* regions = _heap->regions(); 172 ShenandoahCollectionSet* collection_set = _heap->collection_set(); 173 ShenandoahFreeSet* free_set = _heap->free_regions(); 174 free_set->clear(); 175 _root_regions->clear(); 176 assert(_root_regions->count() == 0, "must be cleared"); 177 size_t num_regions = _heap->num_regions(); 178 179 // First pass: find collection set. 180 for (uint to_idx = 0; to_idx < num_regions; to_idx++) { 181 ShenandoahHeapRegion* region = regions->get(to_idx); 182 region->set_root(false); 183 guarantee(! region->is_root(), "must not be root here"); 184 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue; 185 assert(! _heap->region_in_collection_set(to_idx), "must not be in cset yet"); 186 uint num_incoming = 0; 187 for (uint from_idx = 0; from_idx < num_regions; from_idx++) { 188 if (matrix->is_connected(from_idx, to_idx)) { 189 num_incoming++; 190 } 191 } 192 if (num_incoming < ShenandoahPartialInboundThreshold) { 193 collection_set->add_region(region); 194 _heap->set_region_in_collection_set(to_idx, true); 195 } 196 } 197 198 if (UseShenandoahMatrix && PrintShenandoahMatrix) { 199 outputStream* log = Log(gc)::info_stream(); 200 matrix->print_on(log); 201 } 202 203 // Second pass: find all root regions. 204 size_t num_cset = collection_set->count(); 205 for (uint idx = 0; idx < num_cset; idx++) { 206 ShenandoahHeapRegion* region = collection_set->get(idx); 207 size_t to_idx = region->region_number(); 208 assert(! region->is_humongous() && ! region->is_empty() && ! region->is_pinned(), "sanity"); 209 assert(_heap->region_in_collection_set(to_idx), "more sanity"); 210 for (uint from_idx = 0; from_idx < num_regions; from_idx++) { 211 if (matrix->is_connected(from_idx, to_idx)) { 212 ShenandoahHeapRegion* r = regions->get(from_idx); 213 if (! r->is_root() && ! _heap->region_in_collection_set(from_idx)) { 214 _root_regions->add_region(r); 215 r->set_root(true); 216 } 217 } 218 } 219 } 220 // Final pass: free regions. 221 for (uint to_idx = 0; to_idx < num_regions; to_idx++) { 222 ShenandoahHeapRegion* region = regions->get(to_idx); 223 if (! region->is_humongous() && 224 ! region->is_pinned() && 225 ! region->is_root() && 226 ! _heap->in_collection_set(region)) { 227 228 free_set->add_region(region); 229 } 230 if (region->is_root()) { 231 // We rebuild the outbound matrix for all roots, thus we can clear them before. 232 // This makes the matrix more precise. 233 matrix->clear_region_outbound(region->region_number()); 234 } 235 } 236 log_debug(gc, ergo)("got "SIZE_FORMAT" cset regions", collection_set->count()); 237 log_debug(gc, ergo)("got "SIZE_FORMAT" root regions", _root_regions->count()); 238 } 239 240 void ShenandoahPartialGC::do_partial_collection() { 241 242 _heap->gc_timer()->register_gc_start(); 243 ShenandoahCollectorPolicy* policy = _heap->shenandoahPolicy(); 244 policy->record_phase_start(ShenandoahCollectorPolicy::partial_gc); 245 246 { 247 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true); 248 249 assert(_task_queues->is_empty(), "queues must be empty after partial GC"); 250 251 COMPILER2_PRESENT(DerivedPointerTable::clear()); 252 253 { 254 ClassLoaderDataGraph::clear_claimed_marks(); 255 uint nworkers = _heap->workers()->active_workers(); 256 ShenandoahRootProcessor rp(_heap, nworkers); 257 258 if (UseShenandoahOWST) { 259 ShenandoahTaskTerminator terminator(nworkers, task_queues()); 260 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions); 261 _heap->workers()->run_task(&partial_task); 262 } else { 263 ParallelTaskTerminator terminator(nworkers, task_queues()); 264 ShenandoahPartialCollectionTask partial_task(&rp, &terminator, _root_regions); 265 _heap->workers()->run_task(&partial_task); 266 } 267 } 268 269 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); 270 271 if (_heap->cancelled_concgc()) { 272 _task_queues->clear(); 273 } 274 assert(_task_queues->is_empty(), "queues must be empty after partial GC"); 275 276 if (! _heap->cancelled_concgc()) { 277 if (ShenandoahVerify || VerifyShenandoahMatrix) { 278 log_debug(gc, ergo)("Verifying after partial collection"); 279 _heap->verify_heap_reachable_at_safepoint(); 280 log_debug(gc, ergo)("Verification passed"); 281 } 282 283 ShenandoahHeap::ShenandoahHeapLock heap_lock(_heap); 284 size_t num_cset = _heap->collection_set()->count(); 285 for (size_t i = 0; i < num_cset; i++) { 286 ShenandoahHeapRegion* r = _heap->collection_set()->get(i); 287 _heap->decrease_used(r->used()); 288 HeapWord* bottom = r->bottom(); 289 HeapWord* top = _heap->complete_top_at_mark_start(r->bottom()); 290 if (top > bottom) { 291 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top)); 292 } 293 r->recycle(); 294 _heap->free_regions()->add_region(r); 295 } 296 297 reset(); 298 } 299 } 300 301 policy->record_phase_end(ShenandoahCollectorPolicy::partial_gc); 302 _heap->gc_timer()->register_gc_end(); 303 } 304 305 void ShenandoahPartialGC::reset() { 306 _heap->collection_set()->clear(); 307 _heap->clear_cset_fast_test(); 308 _root_regions->clear(); 309 } 310 311 SCMObjToScanQueueSet* ShenandoahPartialGC::task_queues() { 312 return _task_queues; 313 }