1 /* 2 * Copyright (c) 2017, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "gc/shared/gcTraceTime.inline.hpp" 25 #include "gc/shared/workgroup.hpp" 26 #include "gc/shared/taskqueue.inline.hpp" 27 #include "gc/shenandoah/shenandoahBarrierSet.hpp" 28 #include "gc/shenandoah/shenandoahCollectionSet.hpp" 29 #include "gc/shenandoah/shenandoahFreeSet.hpp" 30 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahPartialGC.hpp" 33 #include "gc/shenandoah/shenandoahRootProcessor.hpp" 34 #include "gc/shenandoah/shenandoahTaskqueue.hpp" 35 #include "memory/iterator.hpp" 36 37 class PartialEvacuateUpdateRootsClosure : public OopClosure { 38 ShenandoahPartialGC* _partial_gc; 39 Thread* _thread; 40 SCMObjToScanQueue* _queue; 41 private: 42 template <class T> 43 void do_oop_work(T* p) { _partial_gc->process_oop(p, _thread, _queue, false); } 44 public: 45 PartialEvacuateUpdateRootsClosure(SCMObjToScanQueue* q) : 46 _partial_gc(ShenandoahHeap::heap()->partial_gc()), 47 _thread(Thread::current()), _queue(q) {} 48 void do_oop(oop* p) { do_oop_work(p); } 49 void do_oop(narrowOop* p) { do_oop_work(p); } 50 }; 51 52 class PartialEvacuateUpdateHeapClosure : public ExtendedOopClosure { 53 ShenandoahPartialGC* _partial_gc; 54 Thread* _thread; 55 SCMObjToScanQueue* _queue; 56 private: 57 template <class T> 58 void do_oop_work(T* p) { 59 _partial_gc->process_oop(p, _thread, _queue, true); 60 } 61 public: 62 PartialEvacuateUpdateHeapClosure(SCMObjToScanQueue* q) : 63 _partial_gc(ShenandoahHeap::heap()->partial_gc()), 64 _thread(Thread::current()), _queue(q) {} 65 void do_oop(oop* p) { do_oop_work(p); } 66 void do_oop(narrowOop* p) { do_oop_work(p); } 67 }; 68 69 class ShenandoahPartialCollectionTask : public AbstractGangTask { 70 private: 71 ShenandoahRootProcessor* _rp; 72 ShenandoahHeapRegionSet* _root_regions; 73 ShenandoahHeap* _heap; 74 public: 75 ShenandoahPartialCollectionTask(ShenandoahRootProcessor* rp, 76 ShenandoahHeapRegionSet* root_regions) : 77 AbstractGangTask("Shenandoah Partial Collection"), 78 _rp(rp), _root_regions(root_regions), 79 _heap(ShenandoahHeap::heap()) {} 80 81 void work(uint worker_id) { 82 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues(); 83 SCMObjToScanQueue* q = queues->queue(worker_id); 84 { 85 // First process ordinary GC roots. 86 PartialEvacuateUpdateRootsClosure roots_cl(q); 87 CLDToOopClosure cld_cl(&roots_cl); 88 MarkingCodeBlobClosure code_cl(&roots_cl, CodeBlobToOopClosure::FixRelocations); 89 _rp->process_all_roots(&roots_cl, &roots_cl, &cld_cl, &code_cl, worker_id); 90 } 91 drain_queue(worker_id); 92 if (_heap->cancelled_concgc()) { q->set_empty(); return; } 93 { 94 // Then process root regions. 95 PartialEvacuateUpdateHeapClosure cl(q); 96 ShenandoahHeapRegion* r = _root_regions->claim_next(); 97 while (r != NULL) { 98 r->oop_iterate(&cl); 99 drain_queue(worker_id); 100 if (_heap->cancelled_concgc()) { q->set_empty(); return; } 101 r = _root_regions->claim_next(); 102 } 103 } 104 } 105 106 void drain_queue(uint worker_id) { 107 SCMObjToScanQueueSet* queues = _heap->partial_gc()->task_queues(); 108 SCMObjToScanQueue* q = queues->queue(worker_id); 109 PartialEvacuateUpdateHeapClosure cl(q); 110 // Empty queue if necessary. 111 int seed = 17; 112 SCMTask task; 113 while ((q->pop_buffer(task) || 114 q->pop_local(task) || 115 q->pop_overflow(task)) && 116 !_heap->cancelled_concgc()) { 117 oop obj = task.obj(); 118 assert(! oopDesc::is_null(obj), "must not be null"); 119 obj->oop_iterate(&cl); 120 } 121 } 122 123 }; 124 125 ShenandoahPartialGC::ShenandoahPartialGC(ShenandoahHeap* heap, uint max_regions) : 126 _heap(heap), 127 _root_regions(new ShenandoahHeapRegionSet(max_regions)), 128 _task_queues(new SCMObjToScanQueueSet(heap->max_workers())) { 129 130 uint num_queues = heap->max_workers(); 131 for (uint i = 0; i < num_queues; ++i) { 132 SCMObjToScanQueue* task_queue = new SCMObjToScanQueue(); 133 task_queue->initialize(); 134 _task_queues->register_queue(i, task_queue); 135 } 136 137 } 138 139 ShenandoahHeapRegionSet* ShenandoahPartialGC::root_regions() { 140 return _root_regions; 141 } 142 143 void ShenandoahPartialGC::prepare() { 144 _heap->collection_set()->clear(); 145 assert(_heap->collection_set()->count() == 0, "collection set not clear"); 146 147 _heap->ensure_parsability(true); 148 149 ShenandoahConnectionMatrix* matrix = _heap->connection_matrix(); 150 ShenandoahHeapRegionSet* regions = _heap->regions(); 151 ShenandoahCollectionSet* collection_set = _heap->collection_set(); 152 ShenandoahFreeSet* free_set = _heap->free_regions(); 153 free_set->clear(); 154 _root_regions->clear(); 155 assert(_root_regions->count() == 0, "must be cleared"); 156 uint num_regions = _heap->num_regions(); 157 158 // First pass: find collection set. 159 for (uint to_idx = 0; to_idx < num_regions; to_idx++) { 160 ShenandoahHeapRegion* region = regions->get(to_idx); 161 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue; 162 assert(! _heap->region_in_collection_set(to_idx), "must not be in cset yet"); 163 uint num_incoming = 0; 164 for (uint from_idx = 0; from_idx < num_regions; from_idx++) { 165 if (matrix->is_connected(from_idx, to_idx)) { 166 num_incoming++; 167 } 168 } 169 if (num_incoming < ShenandoahPartialInboundThreshold) { 170 collection_set->add_region(region); 171 _heap->set_region_in_collection_set(to_idx, true); 172 } 173 } 174 // Second pass: find all root regions. 175 for (uint to_idx = 0; to_idx < num_regions; to_idx++) { 176 ShenandoahHeapRegion* region = regions->get(to_idx); 177 if (region->is_humongous() || region->is_empty() || region->is_pinned()) continue; 178 if (_heap->region_in_collection_set(to_idx)) { 179 for (uint from_idx = 0; from_idx < num_regions; from_idx++) { 180 if (matrix->is_connected(from_idx, to_idx)) { 181 ShenandoahHeapRegion* r = regions->get(from_idx); 182 if (! _root_regions->contains(r)) { 183 _root_regions->add_region(r); 184 } 185 } 186 } 187 } 188 } 189 // Final pass: free regions. 190 for (uint to_idx = 0; to_idx < num_regions; to_idx++) { 191 ShenandoahHeapRegion* region = regions->get(to_idx); 192 if (! region->is_humongous() && 193 ! region->is_pinned() && 194 ! _root_regions->contains(region) && 195 ! _heap->in_collection_set(region)) { 196 197 free_set->add_region(region); 198 } 199 } 200 log_debug(gc, ergo)("got "SIZE_FORMAT" cset regions", collection_set->count()); 201 log_debug(gc, ergo)("got "SIZE_FORMAT" root regions", _root_regions->count()); 202 } 203 204 void ShenandoahPartialGC::do_partial_collection() { 205 206 _heap->gc_timer()->register_gc_start(); 207 { 208 GCTraceTime(Info, gc) time("Pause Partial", _heap->gc_timer(), GCCause::_no_gc, true); 209 210 COMPILER2_PRESENT(DerivedPointerTable::clear()); 211 212 { 213 ClassLoaderDataGraph::clear_claimed_marks(); 214 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers()); 215 ShenandoahPartialCollectionTask partial_task(&rp, _root_regions); 216 _heap->workers()->run_task(&partial_task); 217 } 218 219 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); 220 221 if (! _heap->cancelled_concgc()) { 222 ShenandoahHeap::ShenandoahHeapLock heap_lock(_heap); 223 uint num_cset = _heap->collection_set()->count(); 224 for (uint i = 0; i < num_cset; i++) { 225 ShenandoahHeapRegion* r = _heap->collection_set()->get(i); 226 _heap->decrease_used(r->used()); 227 r->recycle(); 228 _heap->free_regions()->add_region(r); 229 } 230 231 reset(); 232 } 233 } 234 _heap->gc_timer()->register_gc_end(); 235 } 236 237 void ShenandoahPartialGC::reset() { 238 _heap->collection_set()->clear(); 239 _heap->clear_cset_fast_test(); 240 _root_regions->clear(); 241 } 242 243 template <class T> 244 void ShenandoahPartialGC::process_oop(T* p, Thread* thread, SCMObjToScanQueue* queue, bool update_matrix) { 245 T o = oopDesc::load_heap_oop(p); 246 if (! oopDesc::is_null(o)) { 247 oop obj = oopDesc::decode_heap_oop_not_null(o); 248 if (_heap->in_collection_set(obj)) { 249 oop forw = ShenandoahBarrierSet::resolve_oop_static_not_null(obj); 250 if (oopDesc::unsafe_equals(obj, forw)) { 251 forw = _heap->evacuate_object(obj, thread); 252 } 253 assert(! oopDesc::unsafe_equals(obj, forw) || _heap->cancelled_concgc(), "must be evacuated"); 254 // oopDesc::encode_store_heap_oop_not_null(p, forw); 255 // queue->push(forw); 256 // Can use simple store for GC roots. 257 if (oopDesc::unsafe_equals(obj, _heap->atomic_compare_exchange_oop(forw, p, obj))) { 258 if (update_matrix) { 259 uint from_idx = _heap->heap_region_index_containing(p); 260 uint to_idx = _heap->heap_region_index_containing(forw); 261 _heap->connection_matrix()->set_connected(from_idx, to_idx, true); 262 } 263 assert(forw->is_oop(), "sanity"); 264 bool succeeded = queue->push(SCMTask(forw)); 265 assert(succeeded, "must succeed to push to task queue"); 266 } 267 } 268 } 269 } 270 271 bool ShenandoahPartialGC::is_in_root_region(oop obj) { 272 // TODO: make this very fast!! 273 ShenandoahHeapRegion* r = _heap->heap_region_containing(obj); 274 return _root_regions->contains(r); 275 } 276 277 SCMObjToScanQueueSet* ShenandoahPartialGC::task_queues() { 278 return _task_queues; 279 }