1 /*
   2  * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 #include "precompiled.hpp"
  25 #include "gc/shared/locationPrinter.hpp"
  26 #include "gc/z/zAddress.inline.hpp"
  27 #include "gc/z/zGlobals.hpp"
  28 #include "gc/z/zHeap.inline.hpp"
  29 #include "gc/z/zHeapIterator.hpp"
  30 #include "gc/z/zHeuristics.hpp"
  31 #include "gc/z/zMark.inline.hpp"
  32 #include "gc/z/zPage.inline.hpp"
  33 #include "gc/z/zPageTable.inline.hpp"
  34 #include "gc/z/zRelocationSet.inline.hpp"
  35 #include "gc/z/zRelocationSetSelector.inline.hpp"
  36 #include "gc/z/zResurrection.hpp"
  37 #include "gc/z/zStat.hpp"
  38 #include "gc/z/zThread.inline.hpp"
  39 #include "gc/z/zVerify.hpp"
  40 #include "gc/z/zWorkers.inline.hpp"
  41 #include "logging/log.hpp"
  42 #include "memory/iterator.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "runtime/handshake.hpp"
  45 #include "runtime/safepoint.hpp"
  46 #include "runtime/thread.hpp"
  47 #include "utilities/debug.hpp"
  48 
  49 static const ZStatSampler ZSamplerHeapUsedBeforeMark("Memory", "Heap Used Before Mark", ZStatUnitBytes);
  50 static const ZStatSampler ZSamplerHeapUsedAfterMark("Memory", "Heap Used After Mark", ZStatUnitBytes);
  51 static const ZStatSampler ZSamplerHeapUsedBeforeRelocation("Memory", "Heap Used Before Relocation", ZStatUnitBytes);
  52 static const ZStatSampler ZSamplerHeapUsedAfterRelocation("Memory", "Heap Used After Relocation", ZStatUnitBytes);
  53 static const ZStatCounter ZCounterUndoPageAllocation("Memory", "Undo Page Allocation", ZStatUnitOpsPerSecond);
  54 static const ZStatCounter ZCounterOutOfMemory("Memory", "Out Of Memory", ZStatUnitOpsPerSecond);
  55 
  56 ZHeap* ZHeap::_heap = NULL;
  57 
  58 ZHeap::ZHeap() :
  59     _workers(),
  60     _object_allocator(),
  61     _page_allocator(&_workers, MinHeapSize, InitialHeapSize, MaxHeapSize, ZHeuristics::max_reserve()),
  62     _page_table(),
  63     _forwarding_table(),
  64     _mark(&_workers, &_page_table),
  65     _reference_processor(&_workers),
  66     _weak_roots_processor(&_workers),
  67     _relocate(&_workers),
  68     _relocation_set(),
  69     _unload(&_workers),
  70     _serviceability(min_capacity(), max_capacity()) {
  71   // Install global heap instance
  72   assert(_heap == NULL, "Already initialized");
  73   _heap = this;
  74 
  75   // Update statistics
  76   ZStatHeap::set_at_initialize(min_capacity(), max_capacity(), max_reserve());
  77 }
  78 
  79 bool ZHeap::is_initialized() const {
  80   return _page_allocator.is_initialized() && _mark.is_initialized();
  81 }
  82 
  83 size_t ZHeap::min_capacity() const {
  84   return _page_allocator.min_capacity();
  85 }
  86 
  87 size_t ZHeap::max_capacity() const {
  88   return _page_allocator.max_capacity();
  89 }
  90 
  91 size_t ZHeap::soft_max_capacity() const {
  92   return _page_allocator.soft_max_capacity();
  93 }
  94 
  95 size_t ZHeap::capacity() const {
  96   return _page_allocator.capacity();
  97 }
  98 
  99 size_t ZHeap::max_reserve() const {
 100   return _page_allocator.max_reserve();
 101 }
 102 
 103 size_t ZHeap::used_high() const {
 104   return _page_allocator.used_high();
 105 }
 106 
 107 size_t ZHeap::used_low() const {
 108   return _page_allocator.used_low();
 109 }
 110 
 111 size_t ZHeap::used() const {
 112   return _page_allocator.used();
 113 }
 114 
 115 size_t ZHeap::unused() const {
 116   return _page_allocator.unused();
 117 }
 118 
 119 size_t ZHeap::allocated() const {
 120   return _page_allocator.allocated();
 121 }
 122 
 123 size_t ZHeap::reclaimed() const {
 124   return _page_allocator.reclaimed();
 125 }
 126 
 127 size_t ZHeap::tlab_capacity() const {
 128   return capacity();
 129 }
 130 
 131 size_t ZHeap::tlab_used() const {
 132   return _object_allocator.used();
 133 }
 134 
 135 size_t ZHeap::max_tlab_size() const {
 136   return ZObjectSizeLimitSmall;
 137 }
 138 
 139 size_t ZHeap::unsafe_max_tlab_alloc() const {
 140   size_t size = _object_allocator.remaining();
 141 
 142   if (size < MinTLABSize) {
 143     // The remaining space in the allocator is not enough to
 144     // fit the smallest possible TLAB. This means that the next
 145     // TLAB allocation will force the allocator to get a new
 146     // backing page anyway, which in turn means that we can then
 147     // fit the largest possible TLAB.
 148     size = max_tlab_size();
 149   }
 150 
 151   return MIN2(size, max_tlab_size());
 152 }
 153 
 154 bool ZHeap::is_in(uintptr_t addr) const {
 155   // An address is considered to be "in the heap" if it points into
 156   // the allocated part of a page, regardless of which heap view is
 157   // used. Note that an address with the finalizable metadata bit set
 158   // is not pointing into a heap view, and therefore not considered
 159   // to be "in the heap".
 160 
 161   if (ZAddress::is_in(addr)) {
 162     const ZPage* const page = _page_table.get(addr);
 163     if (page != NULL) {
 164       return page->is_in(addr);
 165     }
 166   }
 167 
 168   return false;
 169 }
 170 
 171 uint ZHeap::nconcurrent_worker_threads() const {
 172   return _workers.nconcurrent();
 173 }
 174 
 175 uint ZHeap::nconcurrent_no_boost_worker_threads() const {
 176   return _workers.nconcurrent_no_boost();
 177 }
 178 
 179 void ZHeap::set_boost_worker_threads(bool boost) {
 180   _workers.set_boost(boost);
 181 }
 182 
 183 void ZHeap::threads_do(ThreadClosure* tc) const {
 184   _page_allocator.threads_do(tc);
 185   _workers.threads_do(tc);
 186 }
 187 
 188 void ZHeap::out_of_memory() {
 189   ResourceMark rm;
 190 
 191   ZStatInc(ZCounterOutOfMemory);
 192   log_info(gc)("Out Of Memory (%s)", Thread::current()->name());
 193 }
 194 
 195 ZPage* ZHeap::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
 196   ZPage* const page = _page_allocator.alloc_page(type, size, flags);
 197   if (page != NULL) {
 198     // Insert page table entry
 199     _page_table.insert(page);
 200   }
 201 
 202   return page;
 203 }
 204 
 205 void ZHeap::undo_alloc_page(ZPage* page) {
 206   assert(page->is_allocating(), "Invalid page state");
 207 
 208   ZStatInc(ZCounterUndoPageAllocation);
 209   log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT,
 210                 ZThread::id(), ZThread::name(), p2i(page), page->size());
 211 
 212   free_page(page, false /* reclaimed */);
 213 }
 214 
 215 void ZHeap::free_page(ZPage* page, bool reclaimed) {
 216   // Remove page table entry
 217   _page_table.remove(page);
 218 
 219   // Free page
 220   _page_allocator.free_page(page, reclaimed);
 221 }
 222 
 223 void ZHeap::flip_to_marked() {
 224   ZVerifyViewsFlip flip(&_page_allocator);
 225   ZAddress::flip_to_marked();
 226 }
 227 
 228 void ZHeap::flip_to_remapped() {
 229   ZVerifyViewsFlip flip(&_page_allocator);
 230   ZAddress::flip_to_remapped();
 231 }
 232 
 233 void ZHeap::mark_start() {
 234   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 235 
 236   // Update statistics
 237   ZStatSample(ZSamplerHeapUsedBeforeMark, used());
 238 
 239   // Flip address view
 240   flip_to_marked();
 241 
 242   // Retire allocating pages
 243   _object_allocator.retire_pages();
 244 
 245   // Reset allocated/reclaimed/used statistics
 246   _page_allocator.reset_statistics();
 247 
 248   // Reset encountered/dropped/enqueued statistics
 249   _reference_processor.reset_statistics();
 250 
 251   // Enter mark phase
 252   ZGlobalPhase = ZPhaseMark;
 253 
 254   // Reset marking information and mark roots
 255   _mark.start();
 256 
 257   // Update statistics
 258   ZStatHeap::set_at_mark_start(soft_max_capacity(), capacity(), used());
 259 }
 260 
 261 void ZHeap::mark(bool initial) {
 262   _mark.mark(initial);
 263 }
 264 
 265 void ZHeap::mark_flush_and_free(Thread* thread) {
 266   _mark.flush_and_free(thread);
 267 }
 268 
 269 bool ZHeap::mark_end() {
 270   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 271 
 272   // Try end marking
 273   if (!_mark.end()) {
 274     // Marking not completed, continue concurrent mark
 275     return false;
 276   }
 277 
 278   // Enter mark completed phase
 279   ZGlobalPhase = ZPhaseMarkCompleted;
 280 
 281   // Verify after mark
 282   ZVerify::after_mark();
 283 
 284   // Update statistics
 285   ZStatSample(ZSamplerHeapUsedAfterMark, used());
 286   ZStatHeap::set_at_mark_end(capacity(), allocated(), used());
 287 
 288   // Block resurrection of weak/phantom references
 289   ZResurrection::block();
 290 
 291   // Process weak roots
 292   _weak_roots_processor.process_weak_roots();
 293 
 294   // Prepare to unload stale metadata and nmethods
 295   _unload.prepare();
 296 
 297   return true;
 298 }
 299 
 300 void ZHeap::keep_alive(oop obj) {
 301   ZBarrier::keep_alive_barrier_on_oop(obj);
 302 }
 303 
 304 void ZHeap::set_soft_reference_policy(bool clear) {
 305   _reference_processor.set_soft_reference_policy(clear);
 306 }
 307 
 308 class ZRendezvousClosure : public HandshakeClosure {
 309 public:
 310   ZRendezvousClosure() :
 311       HandshakeClosure("ZRendezvous") {}
 312 
 313   void do_thread(Thread* thread) {}
 314 };
 315 
 316 void ZHeap::process_non_strong_references() {
 317   // Process Soft/Weak/Final/PhantomReferences
 318   _reference_processor.process_references();
 319 
 320   // Process concurrent weak roots
 321   _weak_roots_processor.process_concurrent_weak_roots();
 322 
 323   // Unlink stale metadata and nmethods
 324   _unload.unlink();
 325 
 326   // Perform a handshake. This is needed 1) to make sure that stale
 327   // metadata and nmethods are no longer observable. And 2), to
 328   // prevent the race where a mutator first loads an oop, which is
 329   // logically null but not yet cleared. Then this oop gets cleared
 330   // by the reference processor and resurrection is unblocked. At
 331   // this point the mutator could see the unblocked state and pass
 332   // this invalid oop through the normal barrier path, which would
 333   // incorrectly try to mark the oop.
 334   ZRendezvousClosure cl;
 335   Handshake::execute(&cl);
 336 
 337   // Unblock resurrection of weak/phantom references
 338   ZResurrection::unblock();
 339 
 340   // Purge stale metadata and nmethods that were unlinked
 341   _unload.purge();
 342 
 343   // Enqueue Soft/Weak/Final/PhantomReferences. Note that this
 344   // must be done after unblocking resurrection. Otherwise the
 345   // Finalizer thread could call Reference.get() on the Finalizers
 346   // that were just enqueued, which would incorrectly return null
 347   // during the resurrection block window, since such referents
 348   // are only Finalizable marked.
 349   _reference_processor.enqueue_references();
 350 }
 351 
 352 void ZHeap::select_relocation_set() {
 353   // Do not allow pages to be deleted
 354   _page_allocator.enable_deferred_delete();
 355 
 356   // Register relocatable pages with selector
 357   ZRelocationSetSelector selector;
 358   ZPageTableIterator pt_iter(&_page_table);
 359   for (ZPage* page; pt_iter.next(&page);) {
 360     if (!page->is_relocatable()) {
 361       // Not relocatable, don't register
 362       continue;
 363     }
 364 
 365     if (page->is_marked()) {
 366       // Register live page
 367       selector.register_live_page(page);
 368     } else {
 369       // Register garbage page
 370       selector.register_garbage_page(page);
 371 
 372       // Reclaim page immediately
 373       free_page(page, true /* reclaimed */);
 374     }
 375   }
 376 
 377   // Allow pages to be deleted
 378   _page_allocator.disable_deferred_delete();
 379 
 380   // Select pages to relocate
 381   selector.select(&_relocation_set);
 382 
 383   // Setup forwarding table
 384   ZRelocationSetIterator rs_iter(&_relocation_set);
 385   for (ZForwarding* forwarding; rs_iter.next(&forwarding);) {
 386     _forwarding_table.insert(forwarding);
 387   }
 388 
 389   // Update statistics
 390   ZStatRelocation::set_at_select_relocation_set(selector.stats());
 391   ZStatHeap::set_at_select_relocation_set(selector.stats(), reclaimed());
 392 }
 393 
 394 void ZHeap::reset_relocation_set() {
 395   // Reset forwarding table
 396   ZRelocationSetIterator iter(&_relocation_set);
 397   for (ZForwarding* forwarding; iter.next(&forwarding);) {
 398     _forwarding_table.remove(forwarding);
 399   }
 400 
 401   // Reset relocation set
 402   _relocation_set.reset();
 403 }
 404 
 405 void ZHeap::relocate_start() {
 406   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 407 
 408   // Finish unloading stale metadata and nmethods
 409   _unload.finish();
 410 
 411   // Flip address view
 412   flip_to_remapped();
 413 
 414   // Enter relocate phase
 415   ZGlobalPhase = ZPhaseRelocate;
 416 
 417   // Update statistics
 418   ZStatSample(ZSamplerHeapUsedBeforeRelocation, used());
 419   ZStatHeap::set_at_relocate_start(capacity(), allocated(), used());
 420 
 421   // Remap/Relocate roots
 422   _relocate.start();
 423 }
 424 
 425 void ZHeap::relocate() {
 426   // Relocate relocation set
 427   const bool success = _relocate.relocate(&_relocation_set);
 428 
 429   // Update statistics
 430   ZStatSample(ZSamplerHeapUsedAfterRelocation, used());
 431   ZStatRelocation::set_at_relocate_end(success);
 432   ZStatHeap::set_at_relocate_end(capacity(), allocated(), reclaimed(),
 433                                  used(), used_high(), used_low());
 434 }
 435 
 436 void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) {
 437   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 438 
 439   ZHeapIterator iter;
 440   iter.objects_do(cl, visit_weaks);
 441 }
 442 
 443 void ZHeap::pages_do(ZPageClosure* cl) {
 444   ZPageTableIterator iter(&_page_table);
 445   for (ZPage* page; iter.next(&page);) {
 446     cl->do_page(page);
 447   }
 448   _page_allocator.pages_do(cl);
 449 }
 450 
 451 void ZHeap::serviceability_initialize() {
 452   _serviceability.initialize();
 453 }
 454 
 455 GCMemoryManager* ZHeap::serviceability_memory_manager() {
 456   return _serviceability.memory_manager();
 457 }
 458 
 459 MemoryPool* ZHeap::serviceability_memory_pool() {
 460   return _serviceability.memory_pool();
 461 }
 462 
 463 ZServiceabilityCounters* ZHeap::serviceability_counters() {
 464   return _serviceability.counters();
 465 }
 466 
 467 void ZHeap::print_on(outputStream* st) const {
 468   st->print_cr(" ZHeap           used " SIZE_FORMAT "M, capacity " SIZE_FORMAT "M, max capacity " SIZE_FORMAT "M",
 469                used() / M,
 470                capacity() / M,
 471                max_capacity() / M);
 472   MetaspaceUtils::print_on(st);
 473 }
 474 
 475 void ZHeap::print_extended_on(outputStream* st) const {
 476   print_on(st);
 477   st->cr();
 478 
 479   // Do not allow pages to be deleted
 480   _page_allocator.enable_deferred_delete();
 481 
 482   // Print all pages
 483   st->print_cr("ZGC Page Table:");
 484   ZPageTableIterator iter(&_page_table);
 485   for (ZPage* page; iter.next(&page);) {
 486     page->print_on(st);
 487   }
 488 
 489   // Allow pages to be deleted
 490   _page_allocator.enable_deferred_delete();
 491 }
 492 
 493 bool ZHeap::print_location(outputStream* st, uintptr_t addr) const {
 494   if (LocationPrinter::is_valid_obj((void*)addr)) {
 495     st->print(PTR_FORMAT " is a %s oop: ", addr, ZAddress::is_good(addr) ? "good" : "bad");
 496     ZOop::from_address(addr)->print_on(st);
 497     return true;
 498   }
 499 
 500   return false;
 501 }
 502 
 503 void ZHeap::verify() {
 504   // Heap verification can only be done between mark end and
 505   // relocate start. This is the only window where all oop are
 506   // good and the whole heap is in a consistent state.
 507   guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
 508 
 509   ZVerify::after_weak_processing();
 510 }