1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "classfile/javaClasses.hpp" 27 #include "gc/shared/allocTracer.hpp" 28 #include "gc/shared/collectedHeap.hpp" 29 #include "gc/shared/memAllocator.hpp" 30 #include "gc/shared/threadLocalAllocBuffer.inline.hpp" 31 #include "memory/universe.hpp" 32 #include "oops/arrayOop.hpp" 33 #include "oops/oop.inline.hpp" 34 #include "prims/jvmtiExport.hpp" 35 #include "runtime/sharedRuntime.hpp" 36 #include "runtime/handles.inline.hpp" 37 #include "runtime/thread.inline.hpp" 38 #include "services/lowMemoryDetector.hpp" 39 #include "utilities/align.hpp" 40 #include "utilities/copy.hpp" 41 42 class MemAllocator::Allocation: StackObj { 43 friend class MemAllocator; 44 45 const MemAllocator& _allocator; 46 Thread* _thread; 47 oop* _obj_ptr; 48 bool _overhead_limit_exceeded; 49 bool _allocated_outside_tlab; 50 size_t _allocated_tlab_size; 51 bool _tlab_end_reset_for_sample; 52 53 bool check_out_of_memory(); 54 void verify_before(); 55 void verify_after(); 56 void notify_allocation(); 57 void notify_allocation_jvmti_allocation_event(); 58 void notify_allocation_jvmti_sampler(); 59 void notify_allocation_low_memory_detector(); 60 void notify_allocation_jfr_sampler(); 61 void notify_allocation_dtrace_sampler(); 62 void check_for_bad_heap_word_value() const; 63 #ifdef ASSERT 64 void check_for_valid_allocation_state() const; 65 #endif 66 67 class PreserveObj; 68 69 public: 70 Allocation(const MemAllocator& allocator, oop* obj_ptr) 71 : _allocator(allocator), 72 _thread(Thread::current()), 73 _obj_ptr(obj_ptr), 74 _overhead_limit_exceeded(false), 75 _allocated_outside_tlab(false), 76 _allocated_tlab_size(0), 77 _tlab_end_reset_for_sample(false) 78 { 79 verify_before(); 80 } 81 82 ~Allocation() { 83 if (!check_out_of_memory()) { 84 verify_after(); 85 notify_allocation(); 86 } 87 } 88 89 oop obj() const { return *_obj_ptr; } 90 }; 91 92 class MemAllocator::Allocation::PreserveObj: StackObj { 93 HandleMark _handle_mark; 94 Handle _handle; 95 oop* const _obj_ptr; 96 97 public: 98 PreserveObj(Thread* thread, oop* obj_ptr) 99 : _handle_mark(thread), 100 _handle(thread, *obj_ptr), 101 _obj_ptr(obj_ptr) 102 { 103 *obj_ptr = NULL; 104 } 105 106 ~PreserveObj() { 107 *_obj_ptr = _handle(); 108 } 109 110 oop operator()() const { 111 return _handle(); 112 } 113 }; 114 115 bool MemAllocator::Allocation::check_out_of_memory() { 116 Thread* THREAD = _thread; 117 assert(!HAS_PENDING_EXCEPTION, "Unexpected exception, will result in uninitialized storage"); 118 119 if (obj() != NULL) { 120 return false; 121 } 122 123 if (!_overhead_limit_exceeded) { 124 // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support 125 report_java_out_of_memory("Java heap space"); 126 127 if (JvmtiExport::should_post_resource_exhausted()) { 128 JvmtiExport::post_resource_exhausted( 129 JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, 130 "Java heap space"); 131 } 132 THROW_OOP_(Universe::out_of_memory_error_java_heap(), true); 133 } else { 134 // -XX:+HeapDumpOnOutOfMemoryError and -XX:OnOutOfMemoryError support 135 report_java_out_of_memory("GC overhead limit exceeded"); 136 137 if (JvmtiExport::should_post_resource_exhausted()) { 138 JvmtiExport::post_resource_exhausted( 139 JVMTI_RESOURCE_EXHAUSTED_OOM_ERROR | JVMTI_RESOURCE_EXHAUSTED_JAVA_HEAP, 140 "GC overhead limit exceeded"); 141 } 142 143 THROW_OOP_(Universe::out_of_memory_error_gc_overhead_limit(), true); 144 } 145 } 146 147 void MemAllocator::Allocation::verify_before() { 148 // Clear unhandled oops for memory allocation. Memory allocation might 149 // not take out a lock if from tlab, so clear here. 150 Thread* THREAD = _thread; 151 CHECK_UNHANDLED_OOPS_ONLY(THREAD->clear_unhandled_oops();) 152 assert(!HAS_PENDING_EXCEPTION, "Should not allocate with exception pending"); 153 debug_only(check_for_valid_allocation_state()); 154 assert(!Universe::heap()->is_gc_active(), "Allocation during gc not allowed"); 155 } 156 157 void MemAllocator::Allocation::verify_after() { 158 NOT_PRODUCT(check_for_bad_heap_word_value();) 159 } 160 161 void MemAllocator::Allocation::check_for_bad_heap_word_value() const { 162 MemRegion obj_range = _allocator.obj_memory_range(obj()); 163 HeapWord* addr = obj_range.start(); 164 size_t size = obj_range.word_size(); 165 if (CheckMemoryInitialization && ZapUnusedHeapArea) { 166 for (size_t slot = 0; slot < size; slot += 1) { 167 assert((*(intptr_t*) (addr + slot)) != ((intptr_t) badHeapWordVal), 168 "Found badHeapWordValue in post-allocation check"); 169 } 170 } 171 } 172 173 #ifdef ASSERT 174 void MemAllocator::Allocation::check_for_valid_allocation_state() const { 175 // How to choose between a pending exception and a potential 176 // OutOfMemoryError? Don't allow pending exceptions. 177 // This is a VM policy failure, so how do we exhaustively test it? 178 assert(!_thread->has_pending_exception(), 179 "shouldn't be allocating with pending exception"); 180 if (StrictSafepointChecks) { 181 assert(_thread->allow_allocation(), 182 "Allocation done by thread for which allocation is blocked " 183 "by No_Allocation_Verifier!"); 184 // Allocation of an oop can always invoke a safepoint, 185 // hence, the true argument 186 _thread->check_for_valid_safepoint_state(true); 187 } 188 } 189 #endif 190 191 void MemAllocator::Allocation::notify_allocation_jvmti_sampler() { 192 // support for JVMTI VMObjectAlloc event (no-op if not enabled) 193 JvmtiExport::vm_object_alloc_event_collector(obj()); 194 195 if (!ThreadHeapSampler::enabled()) { 196 // Sampling disabled 197 return; 198 } 199 200 if (!_allocated_outside_tlab && _allocated_tlab_size == 0 && !_tlab_end_reset_for_sample) { 201 // Sample if it's a non-TLAB allocation, or a TLAB allocation that either refills the TLAB 202 // or expands it due to taking a sampler induced slow path. 203 return; 204 } 205 206 assert(JavaThread::current()->heap_sampler().add_sampling_collector(), 207 "Should never return false."); 208 209 // Only check if the sampler could actually sample something in this path. 210 assert(!JvmtiExport::should_post_sampled_object_alloc() || 211 !JvmtiSampledObjectAllocEventCollector::object_alloc_is_safe_to_sample() || 212 _thread->heap_sampler().sampling_collector_present(), 213 "Sampling collector not present."); 214 215 if (JvmtiExport::should_post_sampled_object_alloc()) { 216 // If we want to be sampling, protect the allocated object with a Handle 217 // before doing the callback. The callback is done in the destructor of 218 // the JvmtiSampledObjectAllocEventCollector. 219 PreserveObj obj_h(_thread, _obj_ptr); 220 JvmtiSampledObjectAllocEventCollector collector; 221 size_t size_in_bytes = _allocator._word_size * HeapWordSize; 222 ThreadLocalAllocBuffer& tlab = _thread->tlab(); 223 size_t bytes_since_last = _allocated_outside_tlab ? 0 : tlab.bytes_since_last_sample_point(); 224 _thread->heap_sampler().check_for_sampling(obj_h(), size_in_bytes, bytes_since_last); 225 } 226 227 assert(JavaThread::current()->heap_sampler().remove_sampling_collector(), "Should never return false."); 228 229 if (_tlab_end_reset_for_sample || _allocated_tlab_size != 0) { 230 _thread->tlab().set_sample_end(); 231 } 232 } 233 234 void MemAllocator::Allocation::notify_allocation_low_memory_detector() { 235 // support low memory notifications (no-op if not enabled) 236 LowMemoryDetector::detect_low_memory_for_collected_pools(); 237 } 238 239 void MemAllocator::Allocation::notify_allocation_jfr_sampler() { 240 HeapWord* mem = (HeapWord*)obj(); 241 size_t size_in_bytes = _allocator._word_size * HeapWordSize; 242 243 if (_allocated_outside_tlab) { 244 AllocTracer::send_allocation_outside_tlab(_allocator._klass, mem, size_in_bytes, _thread); 245 } else if (_allocated_tlab_size != 0) { 246 // TLAB was refilled 247 AllocTracer::send_allocation_in_new_tlab(_allocator._klass, mem, _allocated_tlab_size * HeapWordSize, 248 size_in_bytes, _thread); 249 } 250 } 251 252 void MemAllocator::Allocation::notify_allocation_dtrace_sampler() { 253 if (DTraceAllocProbes) { 254 // support for Dtrace object alloc event (no-op most of the time) 255 Klass* klass = _allocator._klass; 256 size_t word_size = _allocator._word_size; 257 if (klass != NULL && klass->name() != NULL) { 258 SharedRuntime::dtrace_object_alloc(obj(), (int)word_size); 259 } 260 } 261 } 262 263 void MemAllocator::Allocation::notify_allocation() { 264 notify_allocation_low_memory_detector(); 265 notify_allocation_jfr_sampler(); 266 notify_allocation_dtrace_sampler(); 267 notify_allocation_jvmti_sampler(); 268 } 269 270 HeapWord* MemAllocator::allocate_outside_tlab(Allocation& allocation) const { 271 allocation._allocated_outside_tlab = true; 272 HeapWord* mem = _heap->mem_allocate(_word_size, &allocation._overhead_limit_exceeded); 273 if (mem == NULL) { 274 return mem; 275 } 276 277 NOT_PRODUCT(_heap->check_for_non_bad_heap_word_value(mem, _word_size)); 278 size_t size_in_bytes = _word_size * HeapWordSize; 279 _thread->incr_allocated_bytes(size_in_bytes); 280 281 return mem; 282 } 283 284 HeapWord* MemAllocator::allocate_inside_tlab(Allocation& allocation) const { 285 assert(UseTLAB, "should use UseTLAB"); 286 287 // Try allocating from an existing TLAB. 288 HeapWord* mem = _thread->tlab().allocate(_word_size); 289 if (mem != NULL) { 290 return mem; 291 } 292 293 // Try refilling the TLAB and allocating the object in it. 294 return allocate_inside_tlab_slow(allocation); 295 } 296 297 HeapWord* MemAllocator::allocate_inside_tlab_slow(Allocation& allocation) const { 298 HeapWord* mem = NULL; 299 ThreadLocalAllocBuffer& tlab = _thread->tlab(); 300 301 if (ThreadHeapSampler::enabled()) { 302 // Try to allocate the sampled object from TLAB, it is possible a sample 303 // point was put and the TLAB still has space. 304 tlab.set_back_allocation_end(); 305 mem = tlab.allocate(_word_size); 306 if (mem != NULL) { 307 allocation._tlab_end_reset_for_sample = true; 308 return mem; 309 } 310 } 311 312 // Retain tlab and allocate object in shared space if 313 // the amount free in the tlab is too large to discard. 314 if (tlab.free() > tlab.refill_waste_limit()) { 315 tlab.record_slow_allocation(_word_size); 316 return NULL; 317 } 318 319 // Discard tlab and allocate a new one. 320 // To minimize fragmentation, the last TLAB may be smaller than the rest. 321 size_t new_tlab_size = tlab.compute_size(_word_size); 322 323 tlab.clear_before_allocation(); 324 325 if (new_tlab_size == 0) { 326 return NULL; 327 } 328 329 // Allocate a new TLAB requesting new_tlab_size. Any size 330 // between minimal and new_tlab_size is accepted. 331 size_t min_tlab_size = ThreadLocalAllocBuffer::compute_min_size(_word_size); 332 mem = _heap->allocate_new_tlab(min_tlab_size, new_tlab_size, &allocation._allocated_tlab_size); 333 if (mem == NULL) { 334 assert(allocation._allocated_tlab_size == 0, 335 "Allocation failed, but actual size was updated. min: " SIZE_FORMAT 336 ", desired: " SIZE_FORMAT ", actual: " SIZE_FORMAT, 337 min_tlab_size, new_tlab_size, allocation._allocated_tlab_size); 338 return NULL; 339 } 340 assert(allocation._allocated_tlab_size != 0, "Allocation succeeded but actual size not updated. mem at: " 341 PTR_FORMAT " min: " SIZE_FORMAT ", desired: " SIZE_FORMAT, 342 p2i(mem), min_tlab_size, new_tlab_size); 343 344 if (ZeroTLAB) { 345 // ..and clear it. 346 Copy::zero_to_words(mem, allocation._allocated_tlab_size); 347 } else { 348 // ...and zap just allocated object. 349 #ifdef ASSERT 350 // Skip mangling the space corresponding to the object header to 351 // ensure that the returned space is not considered parsable by 352 // any concurrent GC thread. 353 size_t hdr_size = oopDesc::header_size(); 354 Copy::fill_to_words(mem + hdr_size, allocation._allocated_tlab_size - hdr_size, badHeapWordVal); 355 #endif // ASSERT 356 } 357 358 tlab.fill(mem, mem + _word_size, allocation._allocated_tlab_size); 359 return mem; 360 } 361 362 HeapWord* MemAllocator::mem_allocate(Allocation& allocation) const { 363 if (UseTLAB) { 364 HeapWord* result = allocate_inside_tlab(allocation); 365 if (result != NULL) { 366 return result; 367 } 368 } 369 370 return allocate_outside_tlab(allocation); 371 } 372 373 oop MemAllocator::allocate() const { 374 oop obj = NULL; 375 { 376 Allocation allocation(*this, &obj); 377 HeapWord* mem = mem_allocate(allocation); 378 if (mem != NULL) { 379 obj = initialize(mem); 380 } 381 } 382 return obj; 383 } 384 385 void MemAllocator::mem_clear(HeapWord* mem) const { 386 assert(mem != NULL, "cannot initialize NULL object"); 387 const size_t hs = oopDesc::header_size(); 388 assert(_word_size >= hs, "unexpected object size"); 389 oopDesc::set_klass_gap(mem, 0); 390 Copy::fill_to_aligned_words(mem + hs, _word_size - hs); 391 } 392 393 oop MemAllocator::finish(HeapWord* mem) const { 394 assert(mem != NULL, "NULL object pointer"); 395 if (UseBiasedLocking) { 396 oopDesc::set_mark_raw(mem, _klass->prototype_header()); 397 } else { 398 // May be bootstrapping 399 oopDesc::set_mark_raw(mem, markOopDesc::prototype()); 400 } 401 // Need a release store to ensure array/class length, mark word, and 402 // object zeroing are visible before setting the klass non-NULL, for 403 // concurrent collectors. 404 oopDesc::release_set_klass(mem, _klass); 405 return oop(mem); 406 } 407 408 oop ObjAllocator::initialize(HeapWord* mem) const { 409 mem_clear(mem); 410 return finish(mem); 411 } 412 413 MemRegion ObjArrayAllocator::obj_memory_range(oop obj) const { 414 if (_do_zero) { 415 return MemAllocator::obj_memory_range(obj); 416 } 417 ArrayKlass* array_klass = ArrayKlass::cast(_klass); 418 const size_t hs = arrayOopDesc::header_size(array_klass->element_type()); 419 return MemRegion(((HeapWord*)obj) + hs, _word_size - hs); 420 } 421 422 oop ObjArrayAllocator::initialize(HeapWord* mem) const { 423 // Set array length before setting the _klass field because a 424 // non-NULL klass field indicates that the object is parsable by 425 // concurrent GC. 426 assert(_length >= 0, "length should be non-negative"); 427 if (_do_zero) { 428 mem_clear(mem); 429 } 430 arrayOopDesc::set_length(mem, _length); 431 return finish(mem); 432 } 433 434 oop ClassAllocator::initialize(HeapWord* mem) const { 435 // Set oop_size field before setting the _klass field because a 436 // non-NULL _klass field indicates that the object is parsable by 437 // concurrent GC. 438 assert(_word_size > 0, "oop_size must be positive."); 439 mem_clear(mem); 440 java_lang_Class::set_oop_size(mem, (int)_word_size); 441 return finish(mem); 442 }