1 /* 2 * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "gc/shared/locationPrinter.hpp" 26 #include "gc/z/zAddress.inline.hpp" 27 #include "gc/z/zGlobals.hpp" 28 #include "gc/z/zHeap.inline.hpp" 29 #include "gc/z/zHeapIterator.hpp" 30 #include "gc/z/zHeuristics.hpp" 31 #include "gc/z/zMark.inline.hpp" 32 #include "gc/z/zPage.inline.hpp" 33 #include "gc/z/zPageTable.inline.hpp" 34 #include "gc/z/zRelocationSet.inline.hpp" 35 #include "gc/z/zRelocationSetSelector.inline.hpp" 36 #include "gc/z/zResurrection.hpp" 37 #include "gc/z/zStat.hpp" 38 #include "gc/z/zTask.hpp" 39 #include "gc/z/zThread.inline.hpp" 40 #include "gc/z/zVerify.hpp" 41 #include "gc/z/zWorkers.inline.hpp" 42 #include "logging/log.hpp" 43 #include "memory/iterator.hpp" 44 #include "memory/resourceArea.hpp" 45 #include "runtime/handshake.hpp" 46 #include "runtime/safepoint.hpp" 47 #include "runtime/thread.hpp" 48 #include "utilities/debug.hpp" 49 50 static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes); 51 static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes); 52 static const ZStatSampler ZSamplerHeapUsedBeforeRelocation("Memory", "Heap Used Before Relocation", ZStatUnitBytes); 53 static const ZStatSampler ZSamplerHeapUsedAfterRelocation("Memory", "Heap Used After Relocation", ZStatUnitBytes); 54 static const ZStatCounter ZCounterUndoPageAllocation("Memory", "Undo Page Allocation", ZStatUnitOpsPerSecond); 55 static const ZStatCounter ZCounterOutOfMemory("Memory", "Out Of Memory", ZStatUnitOpsPerSecond); 56 57 ZHeap* ZHeap::_heap = NULL; 58 59 ZHeap::ZHeap() : 60 _workers(), 61 _object_allocator(), 62 _page_allocator(&_workers, MinHeapSize, InitialHeapSize, MaxHeapSize, ZHeuristics::max_reserve()), 63 _page_table(), 64 _forwarding_table(), 65 _mark(&_workers, &_page_table), 66 _reference_processor(&_workers), 67 _weak_roots_processor(&_workers), 68 _relocate(&_workers), 69 _relocation_set(), 70 _unload(&_workers), 71 _serviceability(min_capacity(), max_capacity()) { 72 // Install global heap instance 73 assert(_heap == NULL, "Already initialized"); 74 _heap = this; 75 76 // Update statistics 77 ZStatHeap::set_at_initialize(min_capacity(), max_capacity(), max_reserve()); 78 } 79 80 bool ZHeap::is_initialized() const { 81 return _page_allocator.is_initialized() && _mark.is_initialized(); 82 } 83 84 size_t ZHeap::min_capacity() const { 85 return _page_allocator.min_capacity(); 86 } 87 88 size_t ZHeap::max_capacity() const { 89 return _page_allocator.max_capacity(); 90 } 91 92 size_t ZHeap::soft_max_capacity() const { 93 return _page_allocator.soft_max_capacity(); 94 } 95 96 size_t ZHeap::capacity() const { 97 return _page_allocator.capacity(); 98 } 99 100 size_t ZHeap::max_reserve() const { 101 return _page_allocator.max_reserve(); 102 } 103 104 size_t ZHeap::used_high() const { 105 return _page_allocator.used_high(); 106 } 107 108 size_t ZHeap::used_low() const { 109 return _page_allocator.used_low(); 110 } 111 112 size_t ZHeap::used() const { 113 return _page_allocator.used(); 114 } 115 116 size_t ZHeap::unused() const { 117 return _page_allocator.unused(); 118 } 119 120 size_t ZHeap::allocated() const { 121 return _page_allocator.allocated(); 122 } 123 124 size_t ZHeap::reclaimed() const { 125 return _page_allocator.reclaimed(); 126 } 127 128 size_t ZHeap::tlab_capacity() const { 129 return capacity(); 130 } 131 132 size_t ZHeap::tlab_used() const { 133 return _object_allocator.used(); 134 } 135 136 size_t ZHeap::max_tlab_size() const { 137 return ZObjectSizeLimitSmall; 138 } 139 140 size_t ZHeap::unsafe_max_tlab_alloc() const { 141 size_t size = _object_allocator.remaining(); 142 143 if (size < MinTLABSize) { 144 // The remaining space in the allocator is not enough to 145 // fit the smallest possible TLAB. This means that the next 146 // TLAB allocation will force the allocator to get a new 147 // backing page anyway, which in turn means that we can then 148 // fit the largest possible TLAB. 149 size = max_tlab_size(); 150 } 151 152 return MIN2(size, max_tlab_size()); 153 } 154 155 bool ZHeap::is_in(uintptr_t addr) const { 156 // An address is considered to be "in the heap" if it points into 157 // the allocated part of a page, regardless of which heap view is 158 // used. Note that an address with the finalizable metadata bit set 159 // is not pointing into a heap view, and therefore not considered 160 // to be "in the heap". 161 162 if (ZAddress::is_in(addr)) { 163 const ZPage* const page = _page_table.get(addr); 164 if (page != NULL) { 165 return page->is_in(addr); 166 } 167 } 168 169 return false; 170 } 171 172 uint ZHeap::nconcurrent_worker_threads() const { 173 return _workers.nconcurrent(); 174 } 175 176 uint ZHeap::nconcurrent_no_boost_worker_threads() const { 177 return _workers.nconcurrent_no_boost(); 178 } 179 180 void ZHeap::set_boost_worker_threads(bool boost) { 181 _workers.set_boost(boost); 182 } 183 184 void ZHeap::threads_do(ThreadClosure* tc) const { 185 _page_allocator.threads_do(tc); 186 _workers.threads_do(tc); 187 } 188 189 void ZHeap::out_of_memory() { 190 ResourceMark rm; 191 192 ZStatInc(ZCounterOutOfMemory); 193 log_info(gc)("Out Of Memory (%s)", Thread::current()->name()); 194 } 195 196 ZPage* ZHeap::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) { 197 ZPage* const page = _page_allocator.alloc_page(type, size, flags); 198 if (page != NULL) { 199 // Insert page table entry 200 _page_table.insert(page); 201 } 202 203 return page; 204 } 205 206 void ZHeap::undo_alloc_page(ZPage* page) { 207 assert(page->is_allocating(), "Invalid page state"); 208 209 ZStatInc(ZCounterUndoPageAllocation); 210 log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT, 211 ZThread::id(), ZThread::name(), p2i(page), page->size()); 212 213 free_page(page, false /* reclaimed */); 214 } 215 216 void ZHeap::free_page(ZPage* page, bool reclaimed) { 217 // Remove page table entry 218 _page_table.remove(page); 219 220 // Free page 221 _page_allocator.free_page(page, reclaimed); 222 } 223 224 void ZHeap::flip_to_marked() { 225 ZVerifyViewsFlip flip(&_page_allocator); 226 ZAddress::flip_to_marked(); 227 } 228 229 void ZHeap::flip_to_remapped() { 230 ZVerifyViewsFlip flip(&_page_allocator); 231 ZAddress::flip_to_remapped(); 232 } 233 234 void ZHeap::mark_start() { 235 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); 236 237 // Update statistics 238 ZStatSample(ZSamplerHeapUsedBeforeMark, used()); 239 240 // Flip address view 241 flip_to_marked(); 242 243 // Retire allocating pages 244 _object_allocator.retire_pages(); 245 246 // Reset allocated/reclaimed/used statistics 247 _page_allocator.reset_statistics(); 248 249 // Reset encountered/dropped/enqueued statistics 250 _reference_processor.reset_statistics(); 251 252 // Enter mark phase 253 ZGlobalPhase = ZPhaseMark; 254 255 // Reset marking information and mark roots 256 _mark.start(); 257 258 // Update statistics 259 ZStatHeap::set_at_mark_start(soft_max_capacity(), capacity(), used()); 260 } 261 262 void ZHeap::mark(bool initial) { 263 _mark.mark(initial); 264 } 265 266 void ZHeap::mark_flush_and_free(Thread* thread) { 267 _mark.flush_and_free(thread); 268 } 269 270 bool ZHeap::mark_end() { 271 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); 272 273 // Try end marking 274 if (!_mark.end()) { 275 // Marking not completed, continue concurrent mark 276 return false; 277 } 278 279 // Enter mark completed phase 280 ZGlobalPhase = ZPhaseMarkCompleted; 281 282 // Verify after mark 283 ZVerify::after_mark(); 284 285 // Update statistics 286 ZStatSample(ZSamplerHeapUsedAfterMark, used()); 287 ZStatHeap::set_at_mark_end(capacity(), allocated(), used()); 288 289 // Block resurrection of weak/phantom references 290 ZResurrection::block(); 291 292 // Process weak roots 293 _weak_roots_processor.process_weak_roots(); 294 295 // Prepare to unload stale metadata and nmethods 296 _unload.prepare(); 297 298 return true; 299 } 300 301 void ZHeap::keep_alive(oop obj) { 302 ZBarrier::keep_alive_barrier_on_oop(obj); 303 } 304 305 void ZHeap::set_soft_reference_policy(bool clear) { 306 _reference_processor.set_soft_reference_policy(clear); 307 } 308 309 class ZRendezvousClosure : public HandshakeClosure { 310 public: 311 ZRendezvousClosure() : 312 HandshakeClosure("ZRendezvous") {} 313 314 void do_thread(Thread* thread) {} 315 }; 316 317 void ZHeap::process_non_strong_references() { 318 // Process Soft/Weak/Final/PhantomReferences 319 _reference_processor.process_references(); 320 321 // Process concurrent weak roots 322 _weak_roots_processor.process_concurrent_weak_roots(); 323 324 // Unlink stale metadata and nmethods 325 _unload.unlink(); 326 327 // Perform a handshake. This is needed 1) to make sure that stale 328 // metadata and nmethods are no longer observable. And 2), to 329 // prevent the race where a mutator first loads an oop, which is 330 // logically null but not yet cleared. Then this oop gets cleared 331 // by the reference processor and resurrection is unblocked. At 332 // this point the mutator could see the unblocked state and pass 333 // this invalid oop through the normal barrier path, which would 334 // incorrectly try to mark the oop. 335 ZRendezvousClosure cl; 336 Handshake::execute(&cl); 337 338 // Unblock resurrection of weak/phantom references 339 ZResurrection::unblock(); 340 341 // Purge stale metadata and nmethods that were unlinked 342 _unload.purge(); 343 344 // Enqueue Soft/Weak/Final/PhantomReferences. Note that this 345 // must be done after unblocking resurrection. Otherwise the 346 // Finalizer thread could call Reference.get() on the Finalizers 347 // that were just enqueued, which would incorrectly return null 348 // during the resurrection block window, since such referents 349 // are only Finalizable marked. 350 _reference_processor.enqueue_references(); 351 } 352 353 void ZHeap::select_relocation_set() { 354 // Do not allow pages to be deleted 355 _page_allocator.enable_deferred_delete(); 356 357 // Register relocatable pages with selector 358 ZRelocationSetSelector selector; 359 ZPageTableIterator pt_iter(&_page_table); 360 for (ZPage* page; pt_iter.next(&page);) { 361 if (!page->is_relocatable()) { 362 // Not relocatable, don't register 363 continue; 364 } 365 366 if (page->is_marked()) { 367 // Register live page 368 selector.register_live_page(page); 369 } else { 370 // Register garbage page 371 selector.register_garbage_page(page); 372 373 // Reclaim page immediately 374 free_page(page, true /* reclaimed */); 375 } 376 } 377 378 // Allow pages to be deleted 379 _page_allocator.disable_deferred_delete(); 380 381 // Select pages to relocate 382 selector.select(&_relocation_set); 383 384 // Setup forwarding table 385 ZRelocationSetIterator rs_iter(&_relocation_set); 386 for (ZForwarding* forwarding; rs_iter.next(&forwarding);) { 387 _forwarding_table.insert(forwarding); 388 } 389 390 // Update statistics 391 ZStatRelocation::set_at_select_relocation_set(selector.stats()); 392 ZStatHeap::set_at_select_relocation_set(selector.stats(), reclaimed()); 393 } 394 395 void ZHeap::reset_relocation_set() { 396 // Reset forwarding table 397 ZRelocationSetIterator iter(&_relocation_set); 398 for (ZForwarding* forwarding; iter.next(&forwarding);) { 399 _forwarding_table.remove(forwarding); 400 } 401 402 // Reset relocation set 403 _relocation_set.reset(); 404 } 405 406 void ZHeap::relocate_start() { 407 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); 408 409 // Finish unloading stale metadata and nmethods 410 _unload.finish(); 411 412 // Flip address view 413 flip_to_remapped(); 414 415 // Enter relocate phase 416 ZGlobalPhase = ZPhaseRelocate; 417 418 // Update statistics 419 ZStatSample(ZSamplerHeapUsedBeforeRelocation, used()); 420 ZStatHeap::set_at_relocate_start(capacity(), allocated(), used()); 421 422 // Remap/Relocate roots 423 _relocate.start(); 424 } 425 426 void ZHeap::relocate() { 427 // Relocate relocation set 428 const bool success = _relocate.relocate(&_relocation_set); 429 430 // Update statistics 431 ZStatSample(ZSamplerHeapUsedAfterRelocation, used()); 432 ZStatRelocation::set_at_relocate_end(success); 433 ZStatHeap::set_at_relocate_end(capacity(), allocated(), reclaimed(), 434 used(), used_high(), used_low()); 435 } 436 437 void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) { 438 assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); 439 440 ZHeapIterator iter; 441 iter.objects_do(cl, visit_weaks); 442 } 443 444 void ZHeap::pages_do(ZPageClosure* cl) { 445 ZPageTableIterator iter(&_page_table); 446 for (ZPage* page; iter.next(&page);) { 447 cl->do_page(page); 448 } 449 _page_allocator.pages_do(cl); 450 } 451 452 void ZHeap::serviceability_initialize() { 453 _serviceability.initialize(); 454 } 455 456 GCMemoryManager* ZHeap::serviceability_memory_manager() { 457 return _serviceability.memory_manager(); 458 } 459 460 MemoryPool* ZHeap::serviceability_memory_pool() { 461 return _serviceability.memory_pool(); 462 } 463 464 ZServiceabilityCounters* ZHeap::serviceability_counters() { 465 return _serviceability.counters(); 466 } 467 468 void ZHeap::print_on(outputStream* st) const { 469 st->print_cr(" ZHeap used " SIZE_FORMAT "M, capacity " SIZE_FORMAT "M, max capacity " SIZE_FORMAT "M", 470 used() / M, 471 capacity() / M, 472 max_capacity() / M); 473 MetaspaceUtils::print_on(st); 474 } 475 476 void ZHeap::print_extended_on(outputStream* st) const { 477 print_on(st); 478 st->cr(); 479 480 // Do not allow pages to be deleted 481 _page_allocator.enable_deferred_delete(); 482 483 // Print all pages 484 st->print_cr("ZGC Page Table:"); 485 ZPageTableIterator iter(&_page_table); 486 for (ZPage* page; iter.next(&page);) { 487 page->print_on(st); 488 } 489 490 // Allow pages to be deleted 491 _page_allocator.enable_deferred_delete(); 492 } 493 494 bool ZHeap::print_location(outputStream* st, uintptr_t addr) const { 495 if (LocationPrinter::is_valid_obj((void*)addr)) { 496 st->print(PTR_FORMAT " is a %s oop: ", addr, ZAddress::is_good(addr) ? "good" : "bad"); 497 ZOop::from_address(addr)->print_on(st); 498 return true; 499 } 500 501 return false; 502 } 503 504 void ZHeap::verify() { 505 // Heap verification can only be done between mark end and 506 // relocate start. This is the only window where all oop are 507 // good and the whole heap is in a consistent state. 508 guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase"); 509 510 ZVerify::after_weak_processing(); 511 }