1 /*
   2  * Copyright (c) 2013, 2018, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/markBitMap.hpp"
  28 #include "gc/shared/collectedHeap.hpp"
  29 #include "gc/shenandoah/shenandoahHeapLock.hpp"
  30 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
  31 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
  32 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  33 #include "services/memoryManager.hpp"
  34 
  35 class ConcurrentGCTimer;
  36 class PLABStats;
  37 class ShenandoahAsserts;
  38 class ShenandoahAllocTracker;
  39 class ShenandoahCollectorPolicy;
  40 class ShenandoahConnectionMatrix;
  41 class ShenandoahFastRegionSet;
  42 class ShenandoahPhaseTimings;
  43 class ShenandoahHeap;
  44 class ShenandoahHeapRegion;
  45 class ShenandoahHeapRegionClosure;
  46 class ShenandoahHeapRegionSet;
  47 class ShenandoahCollectionSet;
  48 class ShenandoahFreeSet;
  49 class ShenandoahConcurrentMark;
  50 class ShenandoahMarkCompact;
  51 class ShenandoahPacer;
  52 class ShenandoahTraversalGC;
  53 class ShenandoahVerifier;
  54 class ShenandoahControlThread;
  55 class ShenandoahMonitoringSupport;
  56 
  57 class ShenandoahRegionIterator : public StackObj {
  58 private:
  59   volatile size_t _index;
  60   ShenandoahHeap* _heap;
  61 
  62   // No implicit copying: iterators should be passed by reference to capture the state,
  63   // or be copied explicitly by "=" operator
  64   ShenandoahRegionIterator(const ShenandoahRegionIterator& that);
  65 
  66 public:
  67   ShenandoahRegionIterator();
  68   ShenandoahRegionIterator(ShenandoahHeap* heap);
  69 
  70   ShenandoahRegionIterator& operator=(const ShenandoahRegionIterator& o);
  71 
  72   // Returns next region, or NULL if there are no more regions.
  73   // This is multi-thread-safe.
  74   inline ShenandoahHeapRegion* next();
  75 
  76   // This is *not* MT safe. However, in the absence of multithreaded access, it
  77   // can be used to determine if there is more work to do.
  78   bool has_next() const;
  79 };
  80 
  81 class ShenandoahHeapRegionClosure : public StackObj {
  82 public:
  83   // typically called on each region until it returns true;
  84   virtual bool heap_region_do(ShenandoahHeapRegion* r) = 0;
  85 };
  86 
  87 class ShenandoahUpdateRefsClosure: public OopClosure {
  88 private:
  89   ShenandoahHeap* _heap;
  90 
  91   template <class T>
  92   inline void do_oop_work(T* p);
  93 
  94 public:
  95   ShenandoahUpdateRefsClosure();
  96   inline void do_oop(oop* p);
  97   inline void do_oop(narrowOop* p);
  98 };
  99 
 100 #ifdef ASSERT
 101 class ShenandoahAssertToSpaceClosure : public OopClosure {
 102 private:
 103   template <class T>
 104   void do_oop_nv(T* p);
 105 public:
 106   void do_oop(narrowOop* p);
 107   void do_oop(oop* p);
 108 };
 109 #endif
 110 
 111 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
 112 public:
 113   bool do_object_b(oop p) { return true; }
 114 };
 115 
 116 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
 117 private:
 118   ShenandoahHeap* _heap;
 119 public:
 120   ShenandoahForwardedIsAliveClosure();
 121   void init(ShenandoahHeap* heap) {
 122     _heap = heap;
 123   }
 124   bool do_object_b(oop obj);
 125 };
 126 
 127 class ShenandoahIsAliveClosure: public BoolObjectClosure {
 128 private:
 129   ShenandoahHeap* _heap;
 130 public:
 131   ShenandoahIsAliveClosure();
 132   void init(ShenandoahHeap* heap) {
 133     _heap = heap;
 134   }
 135   bool do_object_b(oop obj);
 136 };
 137 
 138 class VMStructs;
 139 
 140 // // A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
 141 // // It uses a new pauseless GC algorithm based on Brooks pointers.
 142 // // Derived from G1
 143 
 144 // //
 145 // // CollectedHeap
 146 // //    SharedHeap
 147 // //      ShenandoahHeap
 148 class ShenandoahHeap : public CollectedHeap {
 149   friend class ShenandoahAsserts;
 150   friend class VMStructs;
 151 
 152   enum CancelState {
 153 
 154     // Normal state. GC has not been cancelled and is open for cancellation.
 155     // Worker threads can suspend for safepoint.
 156     CANCELLABLE,
 157 
 158     // GC has been cancelled. Worker threads can not suspend for
 159     // safepoint but must finish their work as soon as possible.
 160     CANCELLED,
 161 
 162     // GC has not been cancelled and must not be cancelled. At least
 163     // one worker thread checks for pending safepoint and may suspend
 164     // if a safepoint is pending.
 165     NOT_CANCELLED
 166 
 167   };
 168 
 169 public:
 170   // GC state describes the important parts of collector state, that may be
 171   // used to make barrier selection decisions in the native and generated code.
 172   // Multiple bits can be set at once.
 173   //
 174   // Important invariant: when GC state is zero, the heap is stable, and no barriers
 175   // are required.
 176   enum GCStateBitPos {
 177     // Heap has forwarded objects: need RB, ACMP, CAS barriers.
 178     HAS_FORWARDED_BITPOS   = 0,
 179 
 180     // Heap is under marking: needs SATB barriers.
 181     MARKING_BITPOS    = 1,
 182 
 183     // Heap is under evacuation: needs WB barriers. (Set together with UNSTABLE)
 184     EVACUATION_BITPOS = 2,
 185 
 186     // Heap is under updating: needs SVRB/SVWB barriers.
 187     UPDATEREFS_BITPOS = 3,
 188 
 189     // Heap is under traversal collection
 190     TRAVERSAL_BITPOS  = 4,
 191   };
 192 
 193   enum GCState {
 194     STABLE        = 0,
 195     HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,
 196     MARKING       = 1 << MARKING_BITPOS,
 197     EVACUATION    = 1 << EVACUATION_BITPOS,
 198     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
 199     TRAVERSAL     = 1 << TRAVERSAL_BITPOS,
 200   };
 201 
 202   enum ShenandoahDegenPoint {
 203     _degenerated_unset,
 204     _degenerated_traversal,
 205     _degenerated_outside_cycle,
 206     _degenerated_mark,
 207     _degenerated_evac,
 208     _degenerated_updaterefs,
 209     _DEGENERATED_LIMIT,
 210   };
 211 
 212   enum GCCycleMode {
 213     NONE,
 214     MINOR,
 215     MAJOR
 216   };
 217 
 218   static const char* degen_point_to_string(ShenandoahDegenPoint point) {
 219     switch (point) {
 220       case _degenerated_unset:
 221         return "<UNSET>";
 222       case _degenerated_traversal:
 223         return "Traversal";
 224       case _degenerated_outside_cycle:
 225         return "Outside of Cycle";
 226       case _degenerated_mark:
 227         return "Mark";
 228       case _degenerated_evac:
 229         return "Evacuation";
 230       case _degenerated_updaterefs:
 231         return "Update Refs";
 232       default:
 233         ShouldNotReachHere();
 234         return "ERROR";
 235     }
 236   };
 237 
 238 private:
 239   ShenandoahSharedBitmap _gc_state;
 240   ShenandoahHeapLock _lock;
 241   ShenandoahCollectorPolicy* _shenandoah_policy;
 242   size_t _bitmap_size;
 243   size_t _bitmap_regions_per_slice;
 244   size_t _bitmap_bytes_per_slice;
 245   MemRegion _heap_region;
 246   MemRegion _bitmap0_region;
 247   MemRegion _bitmap1_region;
 248   MemRegion _aux_bitmap_region;
 249 
 250   ShenandoahHeapRegion** _regions;
 251   ShenandoahFreeSet* _free_set;
 252   ShenandoahCollectionSet* _collection_set;
 253 
 254   ShenandoahRegionIterator _update_refs_iterator;
 255 
 256   ShenandoahConcurrentMark* _scm;
 257   ShenandoahMarkCompact* _full_gc;
 258   ShenandoahTraversalGC* _traversal_gc;
 259   ShenandoahVerifier*  _verifier;
 260   ShenandoahPacer*  _pacer;
 261 
 262   ShenandoahControlThread* _control_thread;
 263 
 264   ShenandoahMonitoringSupport* _monitoring_support;
 265 
 266   ShenandoahPhaseTimings*      _phase_timings;
 267   ShenandoahAllocTracker*      _alloc_tracker;
 268 
 269   size_t _num_regions;
 270   size_t _initial_size;
 271 
 272   uint _max_workers;
 273   ShenandoahWorkGang* _workers;
 274   ShenandoahWorkGang* _safepoint_workers;
 275 
 276   volatile size_t _used;
 277   volatile size_t _committed;
 278 
 279   MarkBitMap _verification_bit_map;
 280   MarkBitMap _mark_bit_map0;
 281   MarkBitMap _mark_bit_map1;
 282   MarkBitMap* _complete_mark_bit_map;
 283   MarkBitMap* _next_mark_bit_map;
 284   MarkBitMap _aux_bit_map;
 285 
 286   HeapWord** _complete_top_at_mark_starts;
 287   HeapWord** _complete_top_at_mark_starts_base;
 288 
 289   HeapWord** _next_top_at_mark_starts;
 290   HeapWord** _next_top_at_mark_starts_base;
 291 
 292   volatile size_t _bytes_allocated_since_gc_start;
 293 
 294   ShenandoahSharedFlag _degenerated_gc_in_progress;
 295   ShenandoahSharedFlag _full_gc_in_progress;
 296   ShenandoahSharedFlag _full_gc_move_in_progress;
 297 
 298   ShenandoahSharedFlag _inject_alloc_failure;
 299 
 300   ShenandoahSharedFlag _process_references;
 301   ShenandoahSharedFlag _unload_classes;
 302 
 303   ShenandoahSharedEnumFlag<CancelState> _cancelled_concgc;
 304 
 305   ReferenceProcessor* _ref_processor;
 306 
 307   ShenandoahForwardedIsAliveClosure _forwarded_is_alive;
 308   ShenandoahIsAliveClosure _is_alive;
 309 
 310   ConcurrentGCTimer* _gc_timer;
 311 
 312   ShenandoahConnectionMatrix* _connection_matrix;
 313 
 314   GCMemoryManager _stw_memory_manager;
 315   GCMemoryManager _cycle_memory_manager;
 316 
 317   MemoryPool* _memory_pool;
 318 
 319   ShenandoahEvacOOMHandler _oom_evac_handler;
 320 
 321   ShenandoahSharedEnumFlag<GCCycleMode> _gc_cycle_mode;
 322 
 323   PLABStats* _mutator_gclab_stats;
 324   PLABStats* _collector_gclab_stats;
 325 
 326 #ifdef ASSERT
 327   int     _heap_expansion_count;
 328 #endif
 329 
 330 public:
 331   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 332 
 333   const char* name() const /* override */;
 334   HeapWord* allocate_new_tlab(size_t word_size) /* override */;
 335   void print_on(outputStream* st) const /* override */;
 336   void print_extended_on(outputStream *st) const /* override */;
 337 
 338   ShenandoahHeap::Name kind() const  /* override */{
 339     return CollectedHeap::ShenandoahHeap;
 340   }
 341 
 342   jint initialize() /* override */;
 343   void post_initialize() /* override */;
 344   size_t capacity() const /* override */;
 345   size_t used() const /* override */;
 346   size_t committed() const;
 347   bool is_maximal_no_gc() const /* override */;
 348   size_t max_capacity() const /* override */;
 349   size_t initial_capacity() const /* override */;
 350   bool is_in(const void* p) const /* override */;
 351   bool is_scavengable(oop obj) /* override */;
 352   HeapWord* mem_allocate(size_t size, bool* what) /* override */;
 353   bool can_elide_tlab_store_barriers() const /* override */;
 354   oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */;
 355   bool can_elide_initializing_store_barrier(oop new_obj) /* override */;
 356   bool card_mark_must_follow_store() const /* override */;
 357   void collect(GCCause::Cause cause) /* override */;
 358   void do_full_collection(bool clear_all_soft_refs) /* override */;
 359   AdaptiveSizePolicy* size_policy() /* override */;
 360   CollectorPolicy* collector_policy() const /* override */;
 361   void ensure_parsability(bool retire_tlabs) /* override */;
 362   HeapWord* block_start(const void* addr) const /* override */;
 363   size_t block_size(const HeapWord* addr) const /* override */;
 364   bool block_is_obj(const HeapWord* addr) const /* override */;
 365   jlong millis_since_last_gc() /* override */;
 366   void prepare_for_verify() /* override */;
 367   void print_gc_threads_on(outputStream* st) const /* override */;
 368   void gc_threads_do(ThreadClosure* tcl) const /* override */;
 369   void print_tracing_info() const /* override */;
 370   void verify(VerifyOption vo) /* override */;
 371   bool supports_tlab_allocation() const /* override */;
 372   size_t tlab_capacity(Thread *thr) const /* override */;
 373   void object_iterate(ObjectClosure* cl) /* override */;
 374   void safe_object_iterate(ObjectClosure* cl) /* override */;
 375   size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */;
 376   size_t max_tlab_size() const /* override */;
 377   HeapWord* tlab_post_allocation_setup(HeapWord* obj) /* override */;
 378   uint oop_extra_words() /* override */;
 379   size_t tlab_used(Thread* ignored) const /* override */;
 380   void stop() /* override */;
 381   virtual void safepoint_synchronize_begin();
 382   virtual void safepoint_synchronize_end();
 383 
 384   WorkGang* get_safepoint_workers() { return _safepoint_workers; }
 385 
 386 #ifndef CC_INTERP
 387   void compile_prepare_oop(MacroAssembler* masm, Register obj) /* override */;
 388 #endif
 389 
 390   void register_nmethod(nmethod* nm);
 391   void unregister_nmethod(nmethod* nm);
 392 
 393   /* override: object pinning support */
 394   bool supports_object_pinning() const { return true; }
 395   oop pin_object(JavaThread* thread, oop obj);
 396   void unpin_object(JavaThread* thread, oop obj);
 397 
 398   static ShenandoahHeap* heap();
 399   static ShenandoahHeap* heap_no_check();
 400   static address in_cset_fast_test_addr();
 401   static address cancelled_concgc_addr();
 402   static address gc_state_addr();
 403 
 404   ShenandoahCollectorPolicy *shenandoahPolicy() const { return _shenandoah_policy; }
 405   ShenandoahPhaseTimings*   phase_timings()     const { return _phase_timings; }
 406   ShenandoahAllocTracker*   alloc_tracker()     const { return _alloc_tracker; }
 407 
 408   void accumulate_statistics_all_gclabs();
 409   PLABStats* mutator_gclab_stats()   const { return _mutator_gclab_stats; }
 410   PLABStats* collector_gclab_stats() const { return _collector_gclab_stats; }
 411 
 412   inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;
 413   inline size_t heap_region_index_containing(const void* addr) const;
 414   inline bool requires_marking(const void* entry) const;
 415 
 416   template <class T>
 417   inline oop evac_update_with_forwarded(T* p);
 418 
 419   template <class T>
 420   inline oop maybe_update_with_forwarded(T* p);
 421 
 422   template <class T>
 423   inline oop maybe_update_with_forwarded_not_null(T* p, oop obj);
 424 
 425   template <class T>
 426   inline oop update_with_forwarded_not_null(T* p, oop obj);
 427 
 428   void trash_cset_regions();
 429 
 430   void stop_concurrent_marking();
 431 
 432   void prepare_for_concurrent_evacuation();
 433   void evacuate_and_update_roots();
 434   // Fixup roots after concurrent cycle failed
 435   void fixup_roots();
 436 
 437   void update_heap_references(bool concurrent);
 438 
 439   void roots_iterate(OopClosure* cl);
 440 
 441 private:
 442   void set_gc_state_mask(uint mask, bool value);
 443 
 444 public:
 445   void set_concurrent_mark_in_progress(bool in_progress);
 446   void set_evacuation_in_progress(bool in_progress);
 447   void set_update_refs_in_progress(bool in_progress);
 448   void set_degenerated_gc_in_progress(bool in_progress);
 449   void set_full_gc_in_progress(bool in_progress);
 450   void set_full_gc_move_in_progress(bool in_progress);
 451   void set_concurrent_traversal_in_progress(bool in_progress);
 452   void set_has_forwarded_objects(bool cond);
 453 
 454   void set_process_references(bool pr);
 455   void set_unload_classes(bool uc);
 456 
 457   inline bool is_stable() const;
 458   inline bool is_idle() const;
 459   inline bool is_concurrent_mark_in_progress() const;
 460   inline bool is_update_refs_in_progress() const;
 461   inline bool is_evacuation_in_progress() const;
 462   inline bool is_degenerated_gc_in_progress() const;
 463   inline bool is_full_gc_in_progress() const;
 464   inline bool is_full_gc_move_in_progress() const;
 465   inline bool is_concurrent_traversal_in_progress() const;
 466   inline bool has_forwarded_objects() const;
 467   inline bool is_gc_in_progress_mask(uint mask) const;
 468 
 469   char gc_state();
 470 
 471   bool process_references() const;
 472   bool unload_classes() const;
 473 
 474   bool is_minor_gc() const;
 475   bool is_major_gc() const;
 476   void set_cycle_mode(GCCycleMode gc_cycle_mode);
 477 
 478   inline bool region_in_collection_set(size_t region_index) const;
 479 
 480   // Mainly there to avoid accidentally calling the templated
 481   // method below with ShenandoahHeapRegion* which would be *wrong*.
 482   inline bool in_collection_set(ShenandoahHeapRegion* r) const;
 483 
 484   template <class T>
 485   inline bool in_collection_set(T obj) const;
 486 
 487   inline bool allocated_after_next_mark_start(HeapWord* addr) const;
 488   void set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 489   HeapWord* next_top_at_mark_start(HeapWord* region_base);
 490 
 491   inline bool allocated_after_complete_mark_start(HeapWord* addr) const;
 492   void set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 493   HeapWord* complete_top_at_mark_start(HeapWord* region_base);
 494 
 495   // Evacuates object src. Returns the evacuated object if this thread
 496   // succeeded, otherwise rolls back the evacuation and returns the
 497   // evacuated object by the competing thread.
 498   inline oop  evacuate_object(oop src, Thread* thread);
 499   inline bool cancelled_concgc() const;
 500   inline bool check_cancelled_concgc_and_yield(bool sts_active = true);
 501   inline bool try_cancel_concgc();
 502   inline void clear_cancelled_concgc();
 503 
 504   inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;
 505   void heap_region_iterate(ShenandoahHeapRegionClosure& cl) const;
 506 
 507   ShenandoahFreeSet* free_set()             const { return _free_set; }
 508   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 509 
 510   ShenandoahConnectionMatrix* connection_matrix() const;
 511 
 512   void increase_used(size_t bytes);
 513   void decrease_used(size_t bytes);
 514 
 515   void set_used(size_t bytes);
 516 
 517   void increase_committed(size_t bytes);
 518   void decrease_committed(size_t bytes);
 519 
 520   void increase_allocated(size_t bytes);
 521 
 522   void notify_alloc(size_t words, bool waste);
 523 
 524   void handle_heap_shrinkage(double shrink_before);
 525 
 526   void reset_next_mark_bitmap();
 527   void reset_next_mark_bitmap_traversal();
 528 
 529   MarkBitMap* complete_mark_bit_map();
 530   MarkBitMap* next_mark_bit_map();
 531   inline bool is_marked_complete(oop obj) const;
 532   inline bool mark_next(oop obj) const;
 533   inline bool is_marked_next(oop obj) const;
 534   bool is_next_bitmap_clear();
 535   bool is_next_bitmap_clear_range(HeapWord* start, HeapWord* end);
 536   bool is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end);
 537 
 538   bool commit_bitmap_slice(ShenandoahHeapRegion *r);
 539   bool uncommit_bitmap_slice(ShenandoahHeapRegion *r);
 540 
 541   // Hint that the bitmap slice is not needed
 542   bool idle_bitmap_slice(ShenandoahHeapRegion* r);
 543   void activate_bitmap_slice(ShenandoahHeapRegion* r);
 544 
 545   bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false);
 546 
 547   void print_heap_regions_on(outputStream* st) const;
 548 
 549   size_t bytes_allocated_since_gc_start();
 550   void reset_bytes_allocated_since_gc_start();
 551 
 552   void trash_humongous_region_at(ShenandoahHeapRegion *r);
 553 
 554   virtual GrowableArray<GCMemoryManager*> memory_managers();
 555   virtual GrowableArray<MemoryPool*> memory_pools();
 556 
 557   ShenandoahMonitoringSupport* monitoring_support();
 558   ShenandoahConcurrentMark* concurrentMark() { return _scm; }
 559   ShenandoahMarkCompact* full_gc() { return _full_gc; }
 560   ShenandoahTraversalGC* traversal_gc();
 561   ShenandoahVerifier* verifier();
 562   ShenandoahPacer* pacer() const;
 563 
 564   ReferenceProcessor* ref_processor() { return _ref_processor;}
 565 
 566   WorkGang* workers() const { return _workers;}
 567 
 568   uint max_workers();
 569 
 570   void assert_gc_workers(uint nworker) PRODUCT_RETURN;
 571 
 572   void do_evacuation();
 573   ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
 574 
 575   void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_cset_regions = false, bool skip_humongous_continuation = false) const;
 576 
 577   // Delete entries for dead interned string and clean up unreferenced symbols
 578   // in symbol table, possibly in parallel.
 579   void unload_classes_and_cleanup_tables(bool full_gc);
 580 
 581   inline size_t num_regions() const { return _num_regions; }
 582 
 583   BoolObjectClosure* is_alive_closure();
 584 
 585   // Call before starting evacuation.
 586   void enter_evacuation();
 587   // Call after finished with evacuation.
 588   void leave_evacuation();
 589 
 590   void initialize_gclab(Thread* thread);
 591   void finalize_mutator_gclab(Thread* thread);
 592 
 593 private:
 594   template<class T>
 595   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 596 
 597   template<class T>
 598   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 599 
 600 public:
 601   template<class T>
 602   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 603 
 604   template<class T>
 605   inline void marked_object_safe_iterate(ShenandoahHeapRegion* region, T* cl);
 606 
 607   template<class T>
 608   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl);
 609 
 610   template<class T>
 611   inline void marked_object_oop_safe_iterate(ShenandoahHeapRegion* region, T* cl);
 612 
 613   GCTimer* gc_timer() const;
 614 
 615   void swap_mark_bitmaps();
 616 
 617   void cancel_concgc(GCCause::Cause cause);
 618 
 619   ShenandoahHeapLock* lock() { return &_lock; }
 620   void assert_heaplock_owned_by_current_thread() PRODUCT_RETURN;
 621   void assert_heaplock_not_owned_by_current_thread() PRODUCT_RETURN;
 622   void assert_heaplock_or_safepoint() PRODUCT_RETURN;
 623 
 624 public:
 625   typedef enum {
 626     _alloc_shared,      // Allocate common, outside of TLAB
 627     _alloc_shared_gc,   // Allocate common, outside of GCLAB
 628     _alloc_tlab,        // Allocate TLAB
 629     _alloc_gclab,       // Allocate GCLAB
 630     _ALLOC_LIMIT,
 631   } AllocType;
 632 
 633   static const char* alloc_type_to_string(AllocType type) {
 634     switch (type) {
 635       case _alloc_shared:
 636         return "Shared";
 637       case _alloc_shared_gc:
 638         return "Shared GC";
 639       case _alloc_tlab:
 640         return "TLAB";
 641       case _alloc_gclab:
 642         return "GCLAB";
 643       default:
 644         ShouldNotReachHere();
 645         return "";
 646     }
 647   }
 648 private:
 649 
 650   virtual void initialize_serviceability();
 651 
 652   HeapWord* allocate_new_lab(size_t word_size, AllocType type);
 653   HeapWord* allocate_memory_under_lock(size_t word_size, AllocType type, bool &new_region);
 654   HeapWord* allocate_memory(size_t word_size, AllocType type);
 655   // Shenandoah functionality.
 656   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 657   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 658   HeapWord* allocate_new_gclab(size_t word_size);
 659 
 660   template<class T>
 661   inline void do_object_marked_complete(T* cl, oop obj);
 662 
 663   ShenandoahControlThread* control_thread() { return _control_thread; }
 664 
 665 public:
 666   inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
 667   inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
 668 
 669 private:
 670   void ref_processing_init();
 671 
 672   GCTracer* tracer();
 673 
 674 private:
 675   uint64_t _alloc_seq_at_last_gc_start;
 676   uint64_t _alloc_seq_at_last_gc_end;
 677   size_t _used_at_last_gc;
 678 
 679 public:
 680   uint64_t alloc_seq_at_last_gc_end()   const { return _alloc_seq_at_last_gc_end;  }
 681   uint64_t alloc_seq_at_last_gc_start() const { return _alloc_seq_at_last_gc_start;}
 682   size_t used_at_last_gc()              const { return _used_at_last_gc;}
 683 
 684   void set_alloc_seq_gc_start();
 685   void set_alloc_seq_gc_end();
 686 
 687   void set_used_at_last_gc() {_used_at_last_gc = used();}
 688 
 689   void make_tlabs_parsable(bool retire_tlabs) /* override */;
 690 
 691   GCMemoryManager* cycle_memory_manager() { return &_cycle_memory_manager; }
 692   GCMemoryManager* stw_memory_manager()   { return &_stw_memory_manager; }
 693 
 694 public:
 695   // Entry points to STW GC operations, these cause a related safepoint, that then
 696   // call the entry method below
 697   void vmop_entry_init_mark();
 698   void vmop_entry_final_mark();
 699   void vmop_entry_final_evac();
 700   void vmop_entry_init_updaterefs();
 701   void vmop_entry_final_updaterefs();
 702   void vmop_entry_init_traversal();
 703   void vmop_entry_final_traversal();
 704   void vmop_entry_full(GCCause::Cause cause);
 705   void vmop_degenerated(ShenandoahDegenPoint point);
 706 
 707   // Entry methods to normally STW GC operations. These set up logging, monitoring
 708   // and workers for net VM operation
 709   void entry_init_mark();
 710   void entry_final_mark();
 711   void entry_final_evac();
 712   void entry_init_updaterefs();
 713   void entry_final_updaterefs();
 714   void entry_init_traversal();
 715   void entry_final_traversal();
 716   void entry_full(GCCause::Cause cause);
 717   void entry_degenerated(int point);
 718 
 719   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
 720   // for concurrent operation.
 721   void entry_mark();
 722   void entry_preclean();
 723   void entry_cleanup();
 724   void entry_cleanup_bitmaps();
 725   void entry_cleanup_traversal();
 726   void entry_evac();
 727   void entry_updaterefs();
 728   void entry_traversal();
 729 
 730 private:
 731   // Actual work for the phases
 732   void op_init_mark();
 733   void op_final_mark();
 734   void op_final_evac();
 735   void op_init_updaterefs();
 736   void op_final_updaterefs();
 737   void op_init_traversal();
 738   void op_final_traversal();
 739   void op_full(GCCause::Cause cause);
 740   void op_degenerated(ShenandoahDegenPoint point);
 741   void op_degenerated_fail();
 742   void op_degenerated_futile();
 743 
 744   void op_mark();
 745   void op_preclean();
 746   void op_cleanup();
 747   void op_evac();
 748   void op_updaterefs();
 749   void op_cleanup_bitmaps();
 750   void op_cleanup_traversal();
 751   void op_traversal();
 752 
 753 private:
 754   void try_inject_alloc_failure();
 755   bool should_inject_alloc_failure();
 756 };
 757 
 758 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP