1 /*
   2  * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  27 
  28 #include "gc/shared/markBitMap.hpp"
  29 #include "gc/shared/softRefPolicy.hpp"
  30 #include "gc/shared/collectedHeap.hpp"
  31 #include "gc/shenandoah/shenandoahAsserts.hpp"
  32 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  33 #include "gc/shenandoah/shenandoahLock.hpp"
  34 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
  35 #include "gc/shenandoah/shenandoahPadding.hpp"
  36 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
  37 #include "gc/shenandoah/shenandoahUnload.hpp"
  38 #include "services/memoryManager.hpp"
  39 #include "utilities/globalDefinitions.hpp"
  40 
  41 class ConcurrentGCTimer;
  42 class ReferenceProcessor;
  43 class ShenandoahCollectorPolicy;
  44 class ShenandoahControlThread;
  45 class ShenandoahGCSession;
  46 class ShenandoahGCStateResetter;
  47 class ShenandoahHeuristics;
  48 class ShenandoahMarkingContext;
  49 class ShenandoahMarkCompact;
  50 class ShenandoahMode;
  51 class ShenandoahPhaseTimings;
  52 class ShenandoahHeap;
  53 class ShenandoahHeapRegion;
  54 class ShenandoahHeapRegionClosure;
  55 class ShenandoahCollectionSet;
  56 class ShenandoahFreeSet;
  57 class ShenandoahConcurrentMark;
  58 class ShenandoahMarkCompact;
  59 class ShenandoahMonitoringSupport;
  60 class ShenandoahPacer;
  61 class ShenandoahVerifier;
  62 class ShenandoahWorkGang;
  63 class VMStructs;
  64 
  65 // Used for buffering per-region liveness data.
  66 // Needed since ShenandoahHeapRegion uses atomics to update liveness.
  67 // The ShenandoahHeap array has max-workers elements, each of which is an array of
  68 // uint16_t * max_regions. The choice of uint16_t is not accidental:
  69 // there is a tradeoff between static/dynamic footprint that translates
  70 // into cache pressure (which is already high during marking), and
  71 // too many atomic updates. uint32_t is too large, uint8_t is too small.
  72 typedef uint16_t ShenandoahLiveData;
  73 #define SHENANDOAH_LIVEDATA_MAX ((ShenandoahLiveData)-1)
  74 
  75 class ShenandoahRegionIterator : public StackObj {
  76 private:
  77   ShenandoahHeap* _heap;
  78 
  79   shenandoah_padding(0);
  80   volatile size_t _index;
  81   shenandoah_padding(1);
  82 
  83   // No implicit copying: iterators should be passed by reference to capture the state
  84   NONCOPYABLE(ShenandoahRegionIterator);
  85 
  86 public:
  87   ShenandoahRegionIterator();
  88   ShenandoahRegionIterator(ShenandoahHeap* heap);
  89 
  90   // Reset iterator to default state
  91   void reset();
  92 
  93   // Returns next region, or NULL if there are no more regions.
  94   // This is multi-thread-safe.
  95   inline ShenandoahHeapRegion* next();
  96 
  97   // This is *not* MT safe. However, in the absence of multithreaded access, it
  98   // can be used to determine if there is more work to do.
  99   bool has_next() const;
 100 };
 101 
 102 class ShenandoahHeapRegionClosure : public StackObj {
 103 public:
 104   virtual void heap_region_do(ShenandoahHeapRegion* r) = 0;
 105   virtual bool is_thread_safe() { return false; }
 106 };
 107 
 108 #ifdef ASSERT
 109 class ShenandoahAssertToSpaceClosure : public OopClosure {
 110 private:
 111   template <class T>
 112   void do_oop_work(T* p);
 113 public:
 114   void do_oop(narrowOop* p);
 115   void do_oop(oop* p);
 116 };
 117 #endif
 118 
 119 typedef ShenandoahLock    ShenandoahHeapLock;
 120 typedef ShenandoahLocker  ShenandoahHeapLocker;
 121 
 122 // Shenandoah GC is low-pause concurrent GC that uses Brooks forwarding pointers
 123 // to encode forwarding data. See BrooksPointer for details on forwarding data encoding.
 124 // See ShenandoahControlThread for GC cycle structure.
 125 //
 126 class ShenandoahHeap : public CollectedHeap {
 127   friend class ShenandoahAsserts;
 128   friend class VMStructs;
 129   friend class ShenandoahGCSession;
 130   friend class ShenandoahGCStateResetter;
 131 
 132 // ---------- Locks that guard important data structures in Heap
 133 //
 134 private:
 135   ShenandoahHeapLock _lock;
 136 
 137 public:
 138   ShenandoahHeapLock* lock() {
 139     return &_lock;
 140   }
 141 
 142 // ---------- Initialization, termination, identification, printing routines
 143 //
 144 private:
 145   static ShenandoahHeap* _heap;
 146 
 147 public:
 148   static ShenandoahHeap* heap();
 149 
 150   const char* name()          const { return "Shenandoah"; }
 151   ShenandoahHeap::Name kind() const { return CollectedHeap::Shenandoah; }
 152 
 153   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 154   jint initialize();
 155   void post_initialize();
 156   void initialize_heuristics();
 157 
 158   void initialize_serviceability();
 159 
 160   void print_on(outputStream* st)              const;
 161   void print_extended_on(outputStream *st)     const;
 162   void print_tracing_info()                    const;
 163   void print_gc_threads_on(outputStream* st)   const;
 164   void print_heap_regions_on(outputStream* st) const;
 165 
 166   void stop();
 167 
 168   void prepare_for_verify();
 169   void verify(VerifyOption vo);
 170 
 171 // ---------- Heap counters and metrics
 172 //
 173 private:
 174            size_t _initial_size;
 175            size_t _minimum_size;
 176   shenandoah_padding(0);
 177   volatile size_t _used;
 178   volatile size_t _committed;
 179   volatile size_t _bytes_allocated_since_gc_start;
 180   shenandoah_padding(1);
 181 
 182 public:
 183   void increase_used(size_t bytes);
 184   void decrease_used(size_t bytes);
 185   void set_used(size_t bytes);
 186 
 187   void increase_committed(size_t bytes);
 188   void decrease_committed(size_t bytes);
 189   void increase_allocated(size_t bytes);
 190 
 191   size_t bytes_allocated_since_gc_start();
 192   void reset_bytes_allocated_since_gc_start();
 193 
 194   size_t min_capacity()     const;
 195   size_t max_capacity()     const;
 196   size_t initial_capacity() const;
 197   size_t capacity()         const;
 198   size_t used()             const;
 199   size_t committed()        const;
 200 
 201 // ---------- Workers handling
 202 //
 203 private:
 204   uint _max_workers;
 205   ShenandoahWorkGang* _workers;
 206   ShenandoahWorkGang* _safepoint_workers;
 207 
 208 public:
 209   uint max_workers();
 210   void assert_gc_workers(uint nworker) NOT_DEBUG_RETURN;
 211 
 212   WorkGang* workers() const;
 213   WorkGang* get_safepoint_workers();
 214 
 215   void gc_threads_do(ThreadClosure* tcl) const;
 216 
 217 // ---------- Heap regions handling machinery
 218 //
 219 private:
 220   MemRegion _heap_region;
 221   bool      _heap_region_special;
 222   size_t    _num_regions;
 223   ShenandoahHeapRegion** _regions;
 224   ShenandoahRegionIterator _update_refs_iterator;
 225 
 226 public:
 227 
 228   inline HeapWord* base() const { return _heap_region.start(); }
 229 
 230   inline size_t num_regions() const { return _num_regions; }
 231   inline bool is_heap_region_special() { return _heap_region_special; }
 232 
 233   inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;
 234   inline size_t heap_region_index_containing(const void* addr) const;
 235 
 236   inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;
 237 
 238   void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 239   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
 240 
 241 // ---------- GC state machinery
 242 //
 243 // GC state describes the important parts of collector state, that may be
 244 // used to make barrier selection decisions in the native and generated code.
 245 // Multiple bits can be set at once.
 246 //
 247 // Important invariant: when GC state is zero, the heap is stable, and no barriers
 248 // are required.
 249 //
 250 public:
 251   enum GCStateBitPos {
 252     // Heap has forwarded objects: needs LRB barriers.
 253     HAS_FORWARDED_BITPOS   = 0,
 254 
 255     // Heap is under marking: needs SATB barriers.
 256     MARKING_BITPOS    = 1,
 257 
 258     // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED)
 259     EVACUATION_BITPOS = 2,
 260 
 261     // Heap is under updating: needs no additional barriers.
 262     UPDATEREFS_BITPOS = 3,
 263   };
 264 
 265   enum GCState {
 266     STABLE        = 0,
 267     HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,
 268     MARKING       = 1 << MARKING_BITPOS,
 269     EVACUATION    = 1 << EVACUATION_BITPOS,
 270     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
 271   };
 272 
 273 private:
 274   ShenandoahSharedBitmap _gc_state;
 275   ShenandoahSharedFlag   _degenerated_gc_in_progress;
 276   ShenandoahSharedFlag   _full_gc_in_progress;
 277   ShenandoahSharedFlag   _full_gc_move_in_progress;
 278   ShenandoahSharedFlag   _progress_last_gc;
 279   ShenandoahSharedFlag   _concurrent_strong_root_in_progress;
 280   ShenandoahSharedFlag   _concurrent_weak_root_in_progress;
 281 
 282   void set_gc_state_all_threads(char state);
 283   void set_gc_state_mask(uint mask, bool value);
 284 
 285 public:
 286   char gc_state() const;
 287   static address gc_state_addr();
 288 
 289   void set_concurrent_mark_in_progress(bool in_progress);
 290   void set_evacuation_in_progress(bool in_progress);
 291   void set_update_refs_in_progress(bool in_progress);
 292   void set_degenerated_gc_in_progress(bool in_progress);
 293   void set_full_gc_in_progress(bool in_progress);
 294   void set_full_gc_move_in_progress(bool in_progress);
 295   void set_has_forwarded_objects(bool cond);
 296   void set_concurrent_strong_root_in_progress(bool cond);
 297   void set_concurrent_weak_root_in_progress(bool cond);
 298 
 299   inline bool is_stable() const;
 300   inline bool is_idle() const;
 301   inline bool is_concurrent_mark_in_progress() const;
 302   inline bool is_update_refs_in_progress() const;
 303   inline bool is_evacuation_in_progress() const;
 304   inline bool is_degenerated_gc_in_progress() const;
 305   inline bool is_full_gc_in_progress() const;
 306   inline bool is_full_gc_move_in_progress() const;
 307   inline bool has_forwarded_objects() const;
 308   inline bool is_gc_in_progress_mask(uint mask) const;
 309   inline bool is_stw_gc_in_progress() const;
 310   inline bool is_concurrent_strong_root_in_progress() const;
 311   inline bool is_concurrent_weak_root_in_progress() const;
 312 
 313 // ---------- GC cancellation and degeneration machinery
 314 //
 315 // Cancelled GC flag is used to notify concurrent phases that they should terminate.
 316 //
 317 public:
 318   enum ShenandoahDegenPoint {
 319     _degenerated_unset,
 320     _degenerated_outside_cycle,
 321     _degenerated_mark,
 322     _degenerated_evac,
 323     _degenerated_updaterefs,
 324     _DEGENERATED_LIMIT
 325   };
 326 
 327   static const char* degen_point_to_string(ShenandoahDegenPoint point) {
 328     switch (point) {
 329       case _degenerated_unset:
 330         return "<UNSET>";
 331       case _degenerated_outside_cycle:
 332         return "Outside of Cycle";
 333       case _degenerated_mark:
 334         return "Mark";
 335       case _degenerated_evac:
 336         return "Evacuation";
 337       case _degenerated_updaterefs:
 338         return "Update Refs";
 339       default:
 340         ShouldNotReachHere();
 341         return "ERROR";
 342     }
 343   };
 344 
 345 private:
 346   enum CancelState {
 347     // Normal state. GC has not been cancelled and is open for cancellation.
 348     // Worker threads can suspend for safepoint.
 349     CANCELLABLE,
 350 
 351     // GC has been cancelled. Worker threads can not suspend for
 352     // safepoint but must finish their work as soon as possible.
 353     CANCELLED,
 354 
 355     // GC has not been cancelled and must not be cancelled. At least
 356     // one worker thread checks for pending safepoint and may suspend
 357     // if a safepoint is pending.
 358     NOT_CANCELLED
 359   };
 360 
 361   ShenandoahSharedEnumFlag<CancelState> _cancelled_gc;
 362   bool try_cancel_gc();
 363 
 364 public:
 365   static address cancelled_gc_addr();
 366 
 367   inline bool cancelled_gc() const;
 368   inline bool check_cancelled_gc_and_yield(bool sts_active = true);
 369 
 370   inline void clear_cancelled_gc();
 371 
 372   void cancel_gc(GCCause::Cause cause);
 373 
 374 // ---------- GC operations entry points
 375 //
 376 public:
 377   // Entry points to STW GC operations, these cause a related safepoint, that then
 378   // call the entry method below
 379   void vmop_entry_init_mark();
 380   void vmop_entry_final_mark();
 381   void vmop_entry_init_updaterefs();
 382   void vmop_entry_final_updaterefs();
 383   void vmop_entry_full(GCCause::Cause cause);
 384   void vmop_degenerated(ShenandoahDegenPoint point);
 385 
 386   // Entry methods to normally STW GC operations. These set up logging, monitoring
 387   // and workers for net VM operation
 388   void entry_init_mark();
 389   void entry_final_mark();
 390   void entry_init_updaterefs();
 391   void entry_final_updaterefs();
 392   void entry_full(GCCause::Cause cause);
 393   void entry_degenerated(int point);
 394 
 395   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
 396   // for concurrent operation.
 397   void entry_reset();
 398   void entry_mark();
 399   void entry_preclean();
 400   void entry_weak_roots();
 401   void entry_class_unloading();
 402   void entry_strong_roots();
 403   void entry_cleanup_early();
 404   void entry_evac();
 405   void entry_updaterefs();
 406   void entry_cleanup_complete();
 407   void entry_uncommit(double shrink_before);
 408 
 409 private:
 410   // Actual work for the phases
 411   void op_init_mark();
 412   void op_final_mark();
 413   void op_init_updaterefs();
 414   void op_final_updaterefs();
 415   void op_full(GCCause::Cause cause);
 416   void op_degenerated(ShenandoahDegenPoint point);
 417   void op_degenerated_fail();
 418   void op_degenerated_futile();
 419 
 420   void op_reset();
 421   void op_mark();
 422   void op_preclean();
 423   void op_weak_roots();
 424   void op_class_unloading();
 425   void op_strong_roots();
 426   void op_cleanup_early();
 427   void op_conc_evac();
 428   void op_stw_evac();
 429   void op_updaterefs();
 430   void op_cleanup_complete();
 431   void op_uncommit(double shrink_before);
 432 
 433   // Messages for GC trace events, they have to be immortal for
 434   // passing around the logging/tracing systems
 435   const char* init_mark_event_message() const;
 436   const char* final_mark_event_message() const;
 437   const char* conc_mark_event_message() const;
 438   const char* degen_event_message(ShenandoahDegenPoint point) const;
 439 
 440 // ---------- GC subsystems
 441 //
 442 private:
 443   ShenandoahControlThread*   _control_thread;
 444   ShenandoahCollectorPolicy* _shenandoah_policy;
 445   ShenandoahMode*            _gc_mode;
 446   ShenandoahHeuristics*      _heuristics;
 447   ShenandoahFreeSet*         _free_set;
 448   ShenandoahConcurrentMark*  _scm;
 449   ShenandoahMarkCompact*     _full_gc;
 450   ShenandoahPacer*           _pacer;
 451   ShenandoahVerifier*        _verifier;
 452 
 453   ShenandoahPhaseTimings*    _phase_timings;
 454 
 455   ShenandoahControlThread*   control_thread()          { return _control_thread;    }
 456   ShenandoahMarkCompact*     full_gc()                 { return _full_gc;           }
 457 
 458 public:
 459   ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; }
 460   ShenandoahMode*            mode()              const { return _gc_mode;           }
 461   ShenandoahHeuristics*      heuristics()        const { return _heuristics;        }
 462   ShenandoahFreeSet*         free_set()          const { return _free_set;          }
 463   ShenandoahConcurrentMark*  concurrent_mark()         { return _scm;               }
 464   ShenandoahPacer*           pacer()             const { return _pacer;             }
 465 
 466   ShenandoahPhaseTimings*    phase_timings()     const { return _phase_timings;     }
 467 
 468   ShenandoahVerifier*        verifier();
 469 
 470 // ---------- VM subsystem bindings
 471 //
 472 private:
 473   ShenandoahMonitoringSupport* _monitoring_support;
 474   MemoryPool*                  _memory_pool;
 475   GCMemoryManager              _stw_memory_manager;
 476   GCMemoryManager              _cycle_memory_manager;
 477   ConcurrentGCTimer*           _gc_timer;
 478   SoftRefPolicy                _soft_ref_policy;
 479 
 480   // For exporting to SA
 481   int                          _log_min_obj_alignment_in_bytes;
 482 public:
 483   ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support;    }
 484   GCMemoryManager* cycle_memory_manager()           { return &_cycle_memory_manager; }
 485   GCMemoryManager* stw_memory_manager()             { return &_stw_memory_manager;   }
 486   SoftRefPolicy* soft_ref_policy()                  { return &_soft_ref_policy;      }
 487 
 488   GrowableArray<GCMemoryManager*> memory_managers();
 489   GrowableArray<MemoryPool*> memory_pools();
 490   MemoryUsage memory_usage();
 491   GCTracer* tracer();
 492   ConcurrentGCTimer* gc_timer() const;
 493 
 494 // ---------- Reference processing
 495 //
 496 private:
 497   AlwaysTrueClosure    _subject_to_discovery;
 498   ReferenceProcessor*  _ref_processor;
 499   ShenandoahSharedFlag _process_references;
 500   bool                 _ref_proc_mt_discovery;
 501   bool                 _ref_proc_mt_processing;
 502 
 503   void ref_processing_init();
 504 
 505 public:
 506   ReferenceProcessor* ref_processor() { return _ref_processor; }
 507   bool ref_processor_mt_discovery()   { return _ref_proc_mt_discovery;  }
 508   bool ref_processor_mt_processing()  { return _ref_proc_mt_processing; }
 509   void set_process_references(bool pr);
 510   bool process_references() const;
 511 
 512 // ---------- Class Unloading
 513 //
 514 private:
 515   ShenandoahSharedFlag _unload_classes;
 516   ShenandoahUnload     _unloader;
 517 
 518 public:
 519   void set_unload_classes(bool uc);
 520   bool unload_classes() const;
 521 
 522   // Perform STW class unloading and weak root cleaning
 523   void parallel_cleaning(bool full_gc);
 524 
 525 private:
 526   void stw_unload_classes(bool full_gc);
 527   void stw_process_weak_roots(bool full_gc);
 528 
 529   // Prepare concurrent root processing
 530   void prepare_concurrent_roots();
 531   // Prepare and finish concurrent unloading
 532   void prepare_concurrent_unloading();
 533   void finish_concurrent_unloading();
 534 
 535 // ---------- Generic interface hooks
 536 // Minor things that super-interface expects us to implement to play nice with
 537 // the rest of runtime. Some of the things here are not required to be implemented,
 538 // and can be stubbed out.
 539 //
 540 public:
 541   AdaptiveSizePolicy* size_policy() shenandoah_not_implemented_return(NULL);
 542   bool is_maximal_no_gc() const shenandoah_not_implemented_return(false);
 543 
 544   bool is_in(const void* p) const;
 545 
 546   MemRegion reserved_region() const { return _reserved; }
 547   bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); }
 548 
 549   void collect(GCCause::Cause cause);
 550   void do_full_collection(bool clear_all_soft_refs);
 551 
 552   // Used for parsing heap during error printing
 553   HeapWord* block_start(const void* addr) const;
 554   bool block_is_obj(const HeapWord* addr) const;
 555   bool print_location(outputStream* st, void* addr) const;
 556 
 557   // Used for native heap walkers: heap dumpers, mostly
 558   void object_iterate(ObjectClosure* cl);
 559 
 560   // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
 561   void keep_alive(oop obj);
 562 
 563   // Used by RMI
 564   jlong millis_since_last_gc();
 565 
 566 // ---------- Safepoint interface hooks
 567 //
 568 public:
 569   void safepoint_synchronize_begin();
 570   void safepoint_synchronize_end();
 571 
 572 // ---------- Code roots handling hooks
 573 //
 574 public:
 575   void register_nmethod(nmethod* nm);
 576   void unregister_nmethod(nmethod* nm);
 577   void flush_nmethod(nmethod* nm);
 578   void verify_nmethod(nmethod* nm) {}
 579 
 580 // ---------- Pinning hooks
 581 //
 582 public:
 583   // Shenandoah supports per-object (per-region) pinning
 584   bool supports_object_pinning() const { return true; }
 585 
 586   oop pin_object(JavaThread* thread, oop obj);
 587   void unpin_object(JavaThread* thread, oop obj);
 588 
 589   void sync_pinned_region_status();
 590   void assert_pinned_region_status() NOT_DEBUG_RETURN;
 591 
 592 // ---------- Allocation support
 593 //
 594 private:
 595   HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region);
 596   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 597   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 598   HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size);
 599   void retire_and_reset_gclabs();
 600 
 601 public:
 602   HeapWord* allocate_memory(ShenandoahAllocRequest& request);
 603   HeapWord* mem_allocate(size_t size, bool* what);
 604   MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
 605                                                size_t size,
 606                                                Metaspace::MetadataType mdtype);
 607 
 608   void notify_mutator_alloc_words(size_t words, bool waste);
 609 
 610   // Shenandoah supports TLAB allocation
 611   bool supports_tlab_allocation() const { return true; }
 612 
 613   HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size);
 614   size_t tlab_capacity(Thread *thr) const;
 615   size_t unsafe_max_tlab_alloc(Thread *thread) const;
 616   size_t max_tlab_size() const;
 617   size_t tlab_used(Thread* ignored) const;
 618 
 619   void resize_tlabs();
 620 
 621   void ensure_parsability(bool retire_tlabs);
 622   void make_parsable(bool retire_tlabs);
 623 
 624 // ---------- Marking support
 625 //
 626 private:
 627   ShenandoahMarkingContext* _marking_context;
 628   MemRegion  _bitmap_region;
 629   MemRegion  _aux_bitmap_region;
 630   MarkBitMap _verification_bit_map;
 631   MarkBitMap _aux_bit_map;
 632 
 633   size_t _bitmap_size;
 634   size_t _bitmap_regions_per_slice;
 635   size_t _bitmap_bytes_per_slice;
 636 
 637   size_t _pretouch_heap_page_size;
 638   size_t _pretouch_bitmap_page_size;
 639 
 640   bool _bitmap_region_special;
 641   bool _aux_bitmap_region_special;
 642 
 643   ShenandoahLiveData** _liveness_cache;
 644 
 645 public:
 646   inline ShenandoahMarkingContext* complete_marking_context() const;
 647   inline ShenandoahMarkingContext* marking_context() const;
 648   inline void mark_complete_marking_context();
 649   inline void mark_incomplete_marking_context();
 650 
 651   template<class T>
 652   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 653 
 654   template<class T>
 655   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 656 
 657   template<class T>
 658   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 659 
 660   void reset_mark_bitmap();
 661 
 662   // SATB barriers hooks
 663   template<bool RESOLVE>
 664   inline bool requires_marking(const void* entry) const;
 665   void force_satb_flush_all_threads();
 666 
 667   // Support for bitmap uncommits
 668   bool commit_bitmap_slice(ShenandoahHeapRegion *r);
 669   bool uncommit_bitmap_slice(ShenandoahHeapRegion *r);
 670   bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false);
 671 
 672   // Liveness caching support
 673   ShenandoahLiveData* get_liveness_cache(uint worker_id);
 674   void flush_liveness_cache(uint worker_id);
 675 
 676   size_t pretouch_heap_page_size() { return _pretouch_heap_page_size; }
 677 
 678 // ---------- Evacuation support
 679 //
 680 private:
 681   ShenandoahCollectionSet* _collection_set;
 682   ShenandoahEvacOOMHandler _oom_evac_handler;
 683 
 684   void evacuate_and_update_roots();
 685 
 686 public:
 687   static address in_cset_fast_test_addr();
 688 
 689   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 690 
 691   // Checks if object is in the collection set.
 692   inline bool in_collection_set(oop obj) const;
 693 
 694   // Checks if location is in the collection set. Can be interior pointer, not the oop itself.
 695   inline bool in_collection_set_loc(void* loc) const;
 696 
 697   // Evacuates object src. Returns the evacuated object, either evacuated
 698   // by this thread, or by some other thread.
 699   inline oop evacuate_object(oop src, Thread* thread);
 700 
 701   // Call before/after evacuation.
 702   inline void enter_evacuation(Thread* t);
 703   inline void leave_evacuation(Thread* t);
 704 
 705 // ---------- Helper functions
 706 //
 707 public:
 708   template <class T>
 709   inline oop evac_update_with_forwarded(T* p);
 710 
 711   template <class T>
 712   inline oop maybe_update_with_forwarded(T* p);
 713 
 714   template <class T>
 715   inline oop maybe_update_with_forwarded_not_null(T* p, oop obj);
 716 
 717   template <class T>
 718   inline oop update_with_forwarded_not_null(T* p, oop obj);
 719 
 720   static inline oop cas_oop(oop n, narrowOop* addr, oop c);
 721   static inline oop cas_oop(oop n, oop* addr, oop c);
 722   static inline oop cas_oop(oop n, narrowOop* addr, narrowOop c);
 723 
 724   void trash_humongous_region_at(ShenandoahHeapRegion *r);
 725 
 726   void deduplicate_string(oop str);
 727 
 728 private:
 729   void trash_cset_regions();
 730   void update_heap_references(bool concurrent);
 731 
 732 // ---------- Testing helpers functions
 733 //
 734 private:
 735   ShenandoahSharedFlag _inject_alloc_failure;
 736 
 737   void try_inject_alloc_failure();
 738   bool should_inject_alloc_failure();
 739 };
 740 
 741 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP