1 /* 2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates. 3 * 4 * This code is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License version 2 only, as 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "memory/allocation.hpp" 25 #include "gc/g1/heapRegionBounds.inline.hpp" 26 #include "gc/shenandoah/brooksPointer.hpp" 27 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 28 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 29 #include "gc/shared/space.inline.hpp" 30 #include "memory/universe.hpp" 31 #include "oops/oop.inline.hpp" 32 #include "runtime/mutexLocker.hpp" 33 #include "runtime/os.hpp" 34 35 Monitor ShenandoahHeapRegion::_mem_protect_lock(Mutex::special, "ShenandoahMemProtect_lock", true, Monitor::_safepoint_check_never); 36 size_t ShenandoahHeapRegion::RegionSizeShift = 0; 37 size_t ShenandoahHeapRegion::RegionSizeBytes = 0; 38 39 jint ShenandoahHeapRegion::initialize_heap_region(HeapWord* start, 40 size_t regionSizeWords, int index) { 41 42 reserved = MemRegion((HeapWord*) start, regionSizeWords); 43 ContiguousSpace::initialize(reserved, true, false); 44 liveData = 0; 45 _is_in_collection_set = false; 46 _region_number = index; 47 #ifdef ASSERT 48 _mem_protection_level = 1; // Off, level 1. 49 #endif 50 _top_at_mark_start = bottom(); 51 _top_at_prev_mark_start = bottom(); 52 _top_prev_mark_bitmap = bottom(); 53 return JNI_OK; 54 } 55 56 size_t ShenandoahHeapRegion::region_number() const { 57 return _region_number; 58 } 59 60 bool ShenandoahHeapRegion::rollback_allocation(uint size) { 61 set_top(top() - size); 62 return true; 63 } 64 65 void ShenandoahHeapRegion::clearLiveData() { 66 setLiveData(0); 67 } 68 69 void ShenandoahHeapRegion::setLiveData(size_t s) { 70 Atomic::store_ptr(s, (intptr_t*) &liveData); 71 } 72 73 size_t ShenandoahHeapRegion::getLiveData() const { 74 return liveData; 75 } 76 77 size_t ShenandoahHeapRegion::garbage() const { 78 assert(used() >= getLiveData() || is_humongous(), "Live Data must be a subset of used() live: "SIZE_FORMAT" used: "SIZE_FORMAT, getLiveData(), used()); 79 size_t result = used() - getLiveData(); 80 return result; 81 } 82 83 bool ShenandoahHeapRegion::is_in_collection_set() const { 84 return _is_in_collection_set; 85 } 86 87 #include <sys/mman.h> 88 89 #ifdef ASSERT 90 91 void ShenandoahHeapRegion::memProtectionOn() { 92 /* 93 tty->print_cr("protect memory on region level: "INT32_FORMAT, _mem_protection_level); 94 print(tty); 95 */ 96 MutexLockerEx ml(&_mem_protect_lock, true); 97 assert(_mem_protection_level >= 1, "invariant"); 98 99 if (--_mem_protection_level == 0) { 100 if (ShenandoahVerifyWritesToFromSpace) { 101 assert(! ShenandoahVerifyReadsToFromSpace, "can't verify from-space reads when verifying from-space writes"); 102 os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_READ); 103 } else { 104 assert(ShenandoahVerifyReadsToFromSpace, "need to be verifying reads here"); 105 os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_NONE); 106 } 107 } 108 } 109 110 void ShenandoahHeapRegion::memProtectionOff() { 111 /* 112 tty->print_cr("unprotect memory on region level: "INT32_FORMAT, _mem_protection_level); 113 print(tty); 114 */ 115 MutexLockerEx ml(&_mem_protect_lock, true); 116 assert(_mem_protection_level >= 0, "invariant"); 117 if (_mem_protection_level++ == 0) { 118 os::protect_memory((char*) bottom(), end() - bottom(), os::MEM_PROT_RW); 119 } 120 } 121 122 #endif 123 124 void ShenandoahHeapRegion::set_is_in_collection_set(bool b) { 125 assert(! (is_humongous() && b), "never ever enter a humongous region into the collection set"); 126 127 _is_in_collection_set = b; 128 129 if (b) { 130 // tty->print_cr("registering region in fast-cset"); 131 // print(); 132 ShenandoahHeap::heap()->register_region_with_in_cset_fast_test(this); 133 } 134 135 #ifdef ASSERT 136 if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) { 137 if (b) { 138 memProtectionOn(); 139 assert(_mem_protection_level == 0, "need to be protected here"); 140 } else { 141 assert(_mem_protection_level == 0, "need to be protected here"); 142 memProtectionOff(); 143 } 144 } 145 #endif 146 } 147 148 ByteSize ShenandoahHeapRegion::is_in_collection_set_offset() { 149 return byte_offset_of(ShenandoahHeapRegion, _is_in_collection_set); 150 } 151 152 void ShenandoahHeapRegion::print_on(outputStream* st) const { 153 st->print_cr("ShenandoahHeapRegion: "PTR_FORMAT"/"SIZE_FORMAT, p2i(this), _region_number); 154 155 if (is_in_collection_set()) 156 st->print("C"); 157 if (is_humongous_start()) { 158 st->print("H"); 159 } 160 if (is_humongous_continuation()) { 161 st->print("h"); 162 } 163 //else 164 st->print(" "); 165 166 st->print_cr("live = "SIZE_FORMAT" garbage = "SIZE_FORMAT" bottom = "PTR_FORMAT" end = "PTR_FORMAT" top = "PTR_FORMAT, 167 getLiveData(), garbage(), p2i(bottom()), p2i(end()), p2i(top())); 168 } 169 170 171 class SkipUnreachableObjectToOopClosure: public ObjectClosure { 172 ExtendedOopClosure* _cl; 173 bool _skip_unreachable_objects; 174 ShenandoahHeap* _heap; 175 176 public: 177 SkipUnreachableObjectToOopClosure(ExtendedOopClosure* cl, bool skip_unreachable_objects) : 178 _cl(cl), _skip_unreachable_objects(skip_unreachable_objects), _heap(ShenandoahHeap::heap()) {} 179 180 void do_object(oop obj) { 181 182 if ((! _skip_unreachable_objects) || _heap->is_marked_current(obj)) { 183 #ifdef ASSERT 184 if (_skip_unreachable_objects) { 185 assert(_heap->is_marked_current(obj), "obj must be live"); 186 } 187 #endif 188 obj->oop_iterate(_cl); 189 } 190 191 } 192 }; 193 194 void ShenandoahHeapRegion::object_iterate_interruptible(ObjectClosure* blk, bool allow_cancel) { 195 HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 196 ShenandoahHeap* heap = ShenandoahHeap::heap(); 197 while (p < top() && !(allow_cancel && heap->cancelled_concgc())) { 198 blk->do_object(oop(p)); 199 #ifdef ASSERT 200 if (ShenandoahVerifyReadsToFromSpace) { 201 memProtectionOff(); 202 p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 203 memProtectionOn(); 204 } else { 205 p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 206 } 207 #else 208 p += oop(p)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 209 #endif 210 } 211 } 212 213 void ShenandoahHeapRegion::marked_object_iterate(ObjectClosure* blk) { 214 HeapWord* p = bottom(); 215 ShenandoahHeap* heap = ShenandoahHeap::heap(); 216 CMBitMap* bitmap = heap->next_mark_bit_map(); 217 while (p < top()) { 218 p += BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 219 p = bitmap->getNextMarkedWordAddress(p, top()); 220 if (p < top()) { 221 oop obj = oop(p); 222 assert(heap->is_marked_current(obj), "must be marked"); 223 assert(p >= bottom() && p < top(), "must be within region bounds"); 224 assert(obj->is_oop(), "sanity"); 225 size_t size = obj->size(); 226 blk->do_object(obj); 227 p += size; 228 } 229 } 230 } 231 232 HeapWord* ShenandoahHeapRegion::object_iterate_careful(ObjectClosureCareful* blk) { 233 HeapWord * limit = concurrent_iteration_safe_limit(); 234 assert(limit <= top(), "sanity check"); 235 for (HeapWord* p = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; p < limit;) { 236 size_t size = blk->do_object_careful(oop(p)); 237 if (size == 0) { 238 return p; // failed at p 239 } else { 240 p += size + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 241 } 242 } 243 return NULL; // all done 244 } 245 246 void ShenandoahHeapRegion::oop_iterate_skip_unreachable(ExtendedOopClosure* cl, bool skip_unreachable_objects) { 247 SkipUnreachableObjectToOopClosure cl2(cl, skip_unreachable_objects); 248 object_iterate_interruptible(&cl2, false); 249 } 250 251 void ShenandoahHeapRegion::fill_region() { 252 ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap(); 253 254 if (free() > (BrooksPointer::BROOKS_POINTER_OBJ_SIZE + CollectedHeap::min_fill_size())) { 255 HeapWord* filler = allocate(BrooksPointer::BROOKS_POINTER_OBJ_SIZE); 256 HeapWord* obj = allocate(end() - top()); 257 sh->fill_with_object(obj, end() - obj); 258 sh->initialize_brooks_ptr(oop(obj)); 259 } 260 } 261 262 void ShenandoahHeapRegion::set_humongous_start(bool start) { 263 _humongous_start = start; 264 } 265 266 void ShenandoahHeapRegion::set_humongous_continuation(bool continuation) { 267 _humongous_continuation = continuation; 268 } 269 270 bool ShenandoahHeapRegion::is_humongous() const { 271 return _humongous_start || _humongous_continuation; 272 } 273 274 bool ShenandoahHeapRegion::is_humongous_start() const { 275 return _humongous_start; 276 } 277 278 bool ShenandoahHeapRegion::is_humongous_continuation() const { 279 return _humongous_continuation; 280 } 281 282 void ShenandoahHeapRegion::do_reset() { 283 ContiguousSpace::initialize(reserved, true, false); 284 clearLiveData(); 285 _humongous_start = false; 286 _humongous_continuation = false; 287 // _top_at_mark_start = bottom(); 288 _top_at_prev_mark_start = bottom(); 289 } 290 291 void ShenandoahHeapRegion::recycle() { 292 do_reset(); 293 set_is_in_collection_set(false); 294 } 295 296 void ShenandoahHeapRegion::reset() { 297 assert(_mem_protection_level == 1, "needs to be unprotected here"); 298 do_reset(); 299 _is_in_collection_set = false; 300 } 301 302 HeapWord* ShenandoahHeapRegion::block_start_const(const void* p) const { 303 assert(MemRegion(bottom(), end()).contains(p), 304 "p ("PTR_FORMAT") not in space ["PTR_FORMAT", "PTR_FORMAT")", 305 p2i(p), p2i(bottom()), p2i(end())); 306 if (p >= top()) { 307 return top(); 308 } else { 309 HeapWord* last = bottom() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 310 HeapWord* cur = last; 311 while (cur <= p) { 312 last = cur; 313 cur += oop(cur)->size() + BrooksPointer::BROOKS_POINTER_OBJ_SIZE; 314 } 315 assert(oop(last)->is_oop(), 316 PTR_FORMAT" should be an object start", p2i(last)); 317 return last; 318 } 319 } 320 321 void ShenandoahHeapRegion::setup_heap_region_size(size_t initial_heap_size, size_t max_heap_size) { 322 uintx region_size = ShenandoahHeapRegionSize; 323 if (FLAG_IS_DEFAULT(ShenandoahHeapRegionSize)) { 324 size_t average_heap_size = (initial_heap_size + max_heap_size) / 2; 325 region_size = MAX2(average_heap_size / HeapRegionBounds::target_number(), 326 (uintx) HeapRegionBounds::min_size()); 327 } 328 329 int region_size_log = log2_long((jlong) region_size); 330 // Recalculate the region size to make sure it's a power of 331 // 2. This means that region_size is the largest power of 2 that's 332 // <= what we've calculated so far. 333 region_size = ((uintx)1 << region_size_log); 334 335 // Now make sure that we don't go over or under our limits. 336 if (region_size < HeapRegionBounds::min_size()) { 337 region_size = HeapRegionBounds::min_size(); 338 } else if (region_size > HeapRegionBounds::max_size()) { 339 region_size = HeapRegionBounds::max_size(); 340 } 341 342 // And recalculate the log. 343 region_size_log = log2_long((jlong) region_size); 344 345 // Now, set up the globals. 346 guarantee(RegionSizeShift == 0, "we should only set it once"); 347 RegionSizeShift = region_size_log; 348 349 guarantee(RegionSizeBytes == 0, "we should only set it once"); 350 RegionSizeBytes = (size_t)region_size; 351 352 if (ShenandoahLogConfig) { 353 tty->print_cr("Region size in bytes: "SIZE_FORMAT, RegionSizeBytes); 354 tty->print_cr("Region size shift: "SIZE_FORMAT, RegionSizeShift); 355 tty->print_cr("Initial number of regions: "SIZE_FORMAT, initial_heap_size / RegionSizeBytes); 356 tty->print_cr("Maximum number of regions: "SIZE_FORMAT, max_heap_size / RegionSizeBytes); 357 } 358 } 359 360 CompactibleSpace* ShenandoahHeapRegion::next_compaction_space() const { 361 return ShenandoahHeap::heap()->next_compaction_region(this); 362 } 363 364 void ShenandoahHeapRegion::prepare_for_compaction(CompactPoint* cp) { 365 scan_and_forward(this, cp); 366 } 367 368 void ShenandoahHeapRegion::adjust_pointers() { 369 // Check first is there is any work to do. 370 if (used() == 0) { 371 return; // Nothing to do. 372 } 373 374 scan_and_adjust_pointers(this); 375 } 376 377 void ShenandoahHeapRegion::compact() { 378 assert(!is_humongous(), "Shouldn't be compacting humongous regions"); 379 scan_and_compact(this); 380 } 381 382 void ShenandoahHeapRegion::init_top_at_mark_start() { 383 _top_at_mark_start = top(); 384 ShenandoahHeap::heap()->set_top_at_mark_start(bottom(), top()); 385 } 386 387 void ShenandoahHeapRegion::set_top_at_mark_start(HeapWord* top) { 388 _top_at_mark_start = top; 389 ShenandoahHeap::heap()->set_top_at_mark_start(bottom(), top); 390 } 391 392 void ShenandoahHeapRegion::reset_top_at_prev_mark_start() { 393 _top_at_prev_mark_start = bottom(); 394 } 395 396 HeapWord* ShenandoahHeapRegion::top_at_mark_start() { 397 return _top_at_mark_start; 398 } 399 400 HeapWord* ShenandoahHeapRegion::top_at_prev_mark_start() { 401 return _top_at_prev_mark_start; 402 } 403 404 HeapWord* ShenandoahHeapRegion::top_prev_mark_bitmap() { 405 return _top_prev_mark_bitmap; 406 } 407 408 bool ShenandoahHeapRegion::allocated_after_prev_mark_start(HeapWord* addr) const { 409 return addr >= _top_at_prev_mark_start; 410 } 411 412 void ShenandoahHeapRegion::swap_top_at_mark_start() { 413 HeapWord* tmp = _top_at_prev_mark_start; 414 _top_at_prev_mark_start = _top_at_mark_start; 415 _top_at_mark_start = tmp; 416 ShenandoahHeap::heap()->set_top_at_mark_start(bottom(), tmp); 417 } 418 419 void ShenandoahHeapRegion::set_top_prev_mark_bitmap(HeapWord* top) { 420 _top_prev_mark_bitmap = top; 421 }