1 /* 2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "memory/allocation.hpp" 25 #include "gc/g1/heapRegionBounds.inline.hpp" 26 #include "gc/shenandoah/brooksPointer.hpp" 27 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 28 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 29 #include "gc/shared/space.inline.hpp" 30 #include "memory/universe.hpp" 31 #include "runtime/mutexLocker.hpp" 32 #include "runtime/os.hpp" 33 34 size_t ShenandoahHeapRegion::RegionSizeShift = 0; 35 size_t ShenandoahHeapRegion::RegionSizeBytes = 0; 36 37 jint ShenandoahHeapRegion::initialize_heap_region(HeapWord* start, 38 size_t regionSizeWords, int index) { 39 40 reserved = MemRegion((HeapWord*) start, regionSizeWords); 41 ContiguousSpace::initialize(reserved, true, false); 42 liveData = 0; 43 _is_in_collection_set = false; 44 _region_number = index; 45 #ifdef ASSERT 46 _mem_protection_level = 1; // Off, level 1. 47 #endif 48 return JNI_OK; 49 } 50 51 int ShenandoahHeapRegion::region_number() const { 52 return _region_number; 53 } 54 55 bool ShenandoahHeapRegion::rollback_allocation(uint size) { 56 set_top(top() - size); 57 return true; 58 } 59 60 void ShenandoahHeapRegion::clearLiveData() { 61 setLiveData(0); 62 } 63 64 void ShenandoahHeapRegion::setLiveData(size_t s) { 65 Atomic::store_ptr(s, (intptr_t*) &liveData); 66 } 67 68 void ShenandoahHeapRegion::increase_live_data(size_t s) { 69 size_t new_live_data = Atomic::add(s, &liveData); 70 assert(new_live_data <= used() || is_humongous(), "can't have more live data than used"); 71 } 72 73 size_t ShenandoahHeapRegion::getLiveData() const { 74 return liveData; 75 } 76 77 size_t ShenandoahHeapRegion::garbage() const { 78 assert(used() >= getLiveData() || is_humongous(), err_msg("Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT, getLiveData(), used())); 79 size_t result = used() - getLiveData(); 80 return result; 81 } 82 83 bool ShenandoahHeapRegion::is_in_collection_set() const { 84 return _is_in_collection_set; 85 } 86 87 #include <sys/mman.h> 88 89 #ifdef ASSERT 90 91 void ShenandoahHeapRegion::memProtectionOn() { 92 /* 93 tty->print_cr("protect memory on region level: "INT32_FORMAT, _mem_protection_level); 94 print(tty); 95 */ 96 MutexLockerEx ml(ShenandoahMemProtect_lock, true); 97 assert(_mem_protection_level >= 1, "invariant"); 98 99 if (--_mem_protection_level == 0) { 100 if (ShenandoahVerifyWritesToFromSpace) { 101 assert(! ShenandoahVerifyReadsToFromSpace, "can't verify from-space reads when verifying from-space writes"); 102 os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_READ); 103 } else { 104 assert(ShenandoahVerifyReadsToFromSpace, "need to be verifying reads here"); 105 assert(! ShenandoahConcurrentEvacuation, "concurrent evacuation needs to be turned off for verifying from-space-reads"); 106 os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_NONE); 107 } 108 } 109 } 110 111 void ShenandoahHeapRegion::memProtectionOff() { 112 /* 113 tty->print_cr("unprotect memory on region level: "INT32_FORMAT, _mem_protection_level); 114 print(tty); 115 */ 116 MutexLockerEx ml(ShenandoahMemProtect_lock, true); 117 assert(_mem_protection_level >= 0, "invariant"); 118 if (_mem_protection_level++ == 0) { 119 os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_RW); 120 } 121 } 122 123 #endif 124 125 void ShenandoahHeapRegion::set_is_in_collection_set(bool b) { 126 assert(! (is_humongous() && b), "never ever enter a humongous region into the collection set"); 127 128 _is_in_collection_set = b; 129 130 if (b) { 131 // tty->print_cr("registering region in fast-cset"); 132 // print(); 133 ShenandoahHeap::heap()->register_region_with_in_cset_fast_test(this); 134 } 135 136 #ifdef ASSERT 137 if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) { 138 if (b) { 139 memProtectionOn(); 140 assert(_mem_protection_level == 0, "need to be protected here"); 141 } else { 142 assert(_mem_protection_level == 0, "need to be protected here"); 143 memProtectionOff(); 144 } 145 } 146 #endif 147 } 148 149 ByteSize ShenandoahHeapRegion::is_in_collection_set_offset() { 150 return byte_offset_of(ShenandoahHeapRegion, _is_in_collection_set); 151 } 152 153 void ShenandoahHeapRegion::print_on(outputStream* st) const { 154 st->print_cr("ShenandoahHeapRegion: "PTR_FORMAT"/"INT32_FORMAT, p2i(this), _region_number); 155 156 if (is_in_collection_set()) 157 st->print("C"); 158 if (is_humongous_start()) { 159 st->print("H"); 160 } 161 if (is_humongous_continuation()) { 162 st->print("h"); 163 } 164 //else 165 st->print(" "); 166 167 st->print_cr("live = "SIZE_FORMAT" garbage = "SIZE_FORMAT" bottom = "PTR_FORMAT" end = "PTR_FORMAT" top = "PTR_FORMAT, 168 getLiveData(), garbage(), p2i(bottom()), p2i(end()), p2i(top())); 169 } 170 171 172 class SkipUnreachableObjectToOopClosure: public ObjectClosure { 173 ExtendedOopClosure* _cl; 174 bool _skip_unreachable_objects; 175 ShenandoahHeap* _heap; 176 177 public: 178 SkipUnreachableObjectToOopClosure(ExtendedOopClosure* cl, bool skip_unreachable_objects) : 179 _cl(cl), _skip_unreachable_objects(skip_unreachable_objects), _heap(ShenandoahHeap::heap()) {} 180 181 void do_object(oop obj) { 182 183 if ((! _skip_unreachable_objects) || _heap->is_marked_current(obj)) { 184 if (_skip_unreachable_objects) { 185 assert(_heap->is_marked_current(obj), "obj must be live"); 186 } 187 obj->oop_iterate(_cl); 188 } 189 190 } 191 }; 192 193 void ShenandoahHeapRegion::object_iterate_interruptible(ObjectClosure* blk, bool allow_cancel) { 194 HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 195 ShenandoahHeap* heap = ShenandoahHeap::heap(); 196 while (p < top() && !(allow_cancel && heap->cancelled_concgc())) { 197 blk->do_object(oop(p)); 198 #ifdef ASSERT 199 if (ShenandoahVerifyReadsToFromSpace) { 200 memProtectionOff(); 201 p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 202 memProtectionOn(); 203 } else { 204 p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 205 } 206 #else 207 p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 208 #endif 209 } 210 } 211 212 HeapWord* ShenandoahHeapRegion::object_iterate_careful(ObjectClosureCareful* blk) { 213 HeapWord * limit = concurrent_iteration_safe_limit(); 214 assert(limit <= top(), "sanity check"); 215 for (HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; p < limit;) { 216 size_t size = blk->do_object_careful(oop(p)); 217 if (size == 0) { 218 return p; // failed at p 219 } else { 220 p += size + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 221 } 222 } 223 return NULL; // all done 224 } 225 226 void ShenandoahHeapRegion::oop_iterate_skip_unreachable(ExtendedOopClosure* cl, bool skip_unreachable_objects) { 227 SkipUnreachableObjectToOopClosure cl2(cl, skip_unreachable_objects); 228 object_iterate_interruptible(&cl2, false); 229 } 230 231 void ShenandoahHeapRegion::fill_region() { 232 ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); 233 234 if (free() > (BrooksPointer::BROOKS_POINTER_OBJ_SIZE + CollectedHeap::min_fill_size())) { 235 HeapWord* filler = allocate(BrooksPointer::BROOKS_POINTER_OBJ_SIZE); 236 HeapWord* obj = allocate(end() - top()); 237 sh->fill_with_object(obj, end() - obj); 238 sh->initialize_brooks_ptr(filler, obj); 239 } 240 } 241 242 void ShenandoahHeapRegion::set_humongous_start(bool start) { 243 _humongous_start = start; 244 } 245 246 void ShenandoahHeapRegion::set_humongous_continuation(bool continuation) { 247 _humongous_continuation = continuation; 248 } 249 250 bool ShenandoahHeapRegion::is_humongous() const { 251 return _humongous_start || _humongous_continuation; 252 } 253 254 bool ShenandoahHeapRegion::is_humongous_start() const { 255 return _humongous_start; 256 } 257 258 bool ShenandoahHeapRegion::is_humongous_continuation() const { 259 return _humongous_continuation; 260 } 261 262 void ShenandoahHeapRegion::do_reset() { 263 ContiguousSpace::initialize(reserved, true, false); 264 clearLiveData(); 265 _humongous_start = false; 266 _humongous_continuation = false; 267 } 268 269 void ShenandoahHeapRegion::recycle() { 270 do_reset(); 271 set_is_in_collection_set(false); 272 } 273 274 void ShenandoahHeapRegion::reset() { 275 assert(_mem_protection_level == 1, "needs to be unprotected here"); 276 do_reset(); 277 _is_in_collection_set = false; 278 } 279 280 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const { 281 assert(MemRegion(bottom(), end()).contains(p), 282 err_msg("p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")", 283 p2i(p), p2i(bottom()), p2i(end()))); 284 if (p >= top()) { 285 return top(); 286 } else { 287 HeapWord* last = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 288 HeapWord* cur = last; 289 while (cur <= p) { 290 last = cur; 291 cur += oop(cur)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 292 } 293 assert(oop(last)->is_oop(), 294 err_msg(PTR_FORMAT" should be an object start", p2i(last))); 295 return last; 296 } 297 } 298 299 void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) { 300 uintx region_size = ShenandoahHeapRegionSize; 301 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) { 302 size_t average_heap_size = (initial_heap_size + max_heap_size) / 2; 303 region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(), 304 (uintx) HeapRegionBounds::min_size()); 305 } 306 307 int region_size_log = log2_long((jlong) region_size); 308 // Recalculate the region size to make sure it's a power of 309 // 2. This means that region_size is the largest power of 2 that's 310 // <= what we've calculated so far. 311 region_size = ((uintx)1 << region_size_log); 312 313 // Now make sure that we don't go over or under our limits. 314 if (region_size < HeapRegionBounds::min_size()) { 315 region_size = HeapRegionBounds::min_size(); 316 } else if (region_size > HeapRegionBounds::max_size()) { 317 region_size = HeapRegionBounds::max_size(); 318 } 319 320 // And recalculate the log. 321 region_size_log = log2_long((jlong) region_size); 322 323 // Now, set up the globals. 324 guarantee(RegionSizeShift == 0, "we should only set it once"); 325 RegionSizeShift = region_size_log; 326 327 guarantee(RegionSizeBytes == 0, "we should only set it once"); 328 RegionSizeBytes = (size_t)region_size; 329 330 if (ShenandoahLogConfig) { 331 tty->print_cr("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes); 332 tty->print_cr("Region size shift: "SIZE_FORMAT, RegionSizeShift); 333 tty->print_cr("Initial number of regions: "SIZE_FORMAT, initial_heap_size / RegionSizeBytes); 334 tty->print_cr("Maximum number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes); 335 } 336 } 337 338 CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const { 339 return ShenandoahHeap::heap()->next_compaction_region(this); 340 } 341 342 void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) { 343 scan_and_forward(this, cp); 344 } 345 346 void ShenandoahHeapRegion::adjust_pointers() { 347 // Check first is there is any work to do. 348 if (used() == 0) { 349 return; // Nothing to do. 350 } 351 352 scan_and_adjust_pointers(this); 353 } 354 355 void ShenandoahHeapRegion::compact() { 356 scan_and_compact(this); 357 }