1 /*
   2  * Copyright (c) 2013, 2018, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/markBitMap.hpp"
  28 #include "gc/shared/softRefPolicy.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "gc/shenandoah/shenandoahHeapLock.hpp"
  31 #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp"
  32 #include "gc/shenandoah/shenandoahSharedVariables.hpp"
  33 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  34 #include "services/memoryManager.hpp"
  35 
  36 class ConcurrentGCTimer;
  37 class PLABStats;
  38 class ShenandoahAsserts;
  39 class ShenandoahAllocTracker;
  40 class ShenandoahCollectorPolicy;
  41 class ShenandoahConnectionMatrix;
  42 class ShenandoahControlThread;
  43 class ShenandoahFastRegionSet;
  44 class ShenandoahPhaseTimings;
  45 class ShenandoahHeap;
  46 class ShenandoahHeapRegion;
  47 class ShenandoahHeapRegionClosure;
  48 class ShenandoahHeapRegionSet;
  49 class ShenandoahCollectionSet;
  50 class ShenandoahFreeSet;
  51 class ShenandoahConcurrentMark;
  52 class ShenandoahMarkCompact;
  53 class ShenandoahPacer;
  54 class ShenandoahTraversalGC;
  55 class ShenandoahVerifier;
  56 class ShenandoahMonitoringSupport;
  57 
  58 class ShenandoahRegionIterator : public StackObj {
  59 private:
  60   volatile size_t _index;
  61   ShenandoahHeap* _heap;
  62 
  63   // No implicit copying: iterators should be passed by reference to capture the state,
  64   // or be copied explicitly by "=" operator
  65   ShenandoahRegionIterator(const ShenandoahRegionIterator& that);
  66 
  67 public:
  68   ShenandoahRegionIterator();
  69   ShenandoahRegionIterator(ShenandoahHeap* heap);
  70 
  71   ShenandoahRegionIterator& operator=(const ShenandoahRegionIterator& o);
  72 
  73   // Returns next region, or NULL if there are no more regions.
  74   // This is multi-thread-safe.
  75   inline ShenandoahHeapRegion* next();
  76 
  77   // This is *not* MT safe. However, in the absence of multithreaded access, it
  78   // can be used to determine if there is more work to do.
  79   bool has_next() const;
  80 };
  81 
  82 class ShenandoahHeapRegionClosure : public StackObj {
  83 public:
  84   // typically called on each region until it returns true;
  85   virtual bool heap_region_do(ShenandoahHeapRegion* r) = 0;
  86 };
  87 
  88 class ShenandoahUpdateRefsClosure: public OopClosure {
  89 private:
  90   ShenandoahHeap* _heap;
  91 
  92   template <class T>
  93   inline void do_oop_work(T* p);
  94 
  95 public:
  96   ShenandoahUpdateRefsClosure();
  97   inline void do_oop(oop* p);
  98   inline void do_oop(narrowOop* p);
  99 };
 100 
 101 #ifdef ASSERT
 102 class ShenandoahAssertToSpaceClosure : public OopClosure {
 103 private:
 104   template <class T>
 105   void do_oop_nv(T* p);
 106 public:
 107   void do_oop(narrowOop* p);
 108   void do_oop(oop* p);
 109 };
 110 #endif
 111 
 112 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
 113 public:
 114   bool do_object_b(oop p) { return true; }
 115 };
 116 
 117 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
 118 private:
 119   ShenandoahHeap* _heap;
 120 public:
 121   ShenandoahForwardedIsAliveClosure();
 122   void init(ShenandoahHeap* heap) {
 123     _heap = heap;
 124   }
 125   bool do_object_b(oop obj);
 126 };
 127 
 128 class ShenandoahIsAliveClosure: public BoolObjectClosure {
 129 private:
 130   ShenandoahHeap* _heap;
 131 public:
 132   ShenandoahIsAliveClosure();
 133   void init(ShenandoahHeap* heap) {
 134     _heap = heap;
 135   }
 136   bool do_object_b(oop obj);
 137 };
 138 
 139 class VMStructs;
 140 
 141 // // A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
 142 // // It uses a new pauseless GC algorithm based on Brooks pointers.
 143 // // Derived from G1
 144 
 145 // //
 146 // // CollectedHeap
 147 // //    SharedHeap
 148 // //      ShenandoahHeap
 149 class ShenandoahHeap : public CollectedHeap {
 150   friend class ShenandoahAsserts;
 151   friend class VMStructs;
 152 
 153   enum CancelState {
 154 
 155     // Normal state. GC has not been cancelled and is open for cancellation.
 156     // Worker threads can suspend for safepoint.
 157     CANCELLABLE,
 158 
 159     // GC has been cancelled. Worker threads can not suspend for
 160     // safepoint but must finish their work as soon as possible.
 161     CANCELLED,
 162 
 163     // GC has not been cancelled and must not be cancelled. At least
 164     // one worker thread checks for pending safepoint and may suspend
 165     // if a safepoint is pending.
 166     NOT_CANCELLED
 167 
 168   };
 169 
 170 public:
 171   // GC state describes the important parts of collector state, that may be
 172   // used to make barrier selection decisions in the native and generated code.
 173   // Multiple bits can be set at once.
 174   //
 175   // Important invariant: when GC state is zero, the heap is stable, and no barriers
 176   // are required.
 177   enum GCStateBitPos {
 178     // Heap has forwarded objects: need RB, ACMP, CAS barriers.
 179     HAS_FORWARDED_BITPOS   = 0,
 180 
 181     // Heap is under marking: needs SATB barriers.
 182     MARKING_BITPOS    = 1,
 183 
 184     // Heap is under evacuation: needs WB barriers. (Set together with UNSTABLE)
 185     EVACUATION_BITPOS = 2,
 186 
 187     // Heap is under updating: needs SVRB/SVWB barriers.
 188     UPDATEREFS_BITPOS = 3,
 189 
 190     // Heap is under traversal collection
 191     TRAVERSAL_BITPOS  = 4,
 192   };
 193 
 194   enum GCState {
 195     STABLE        = 0,
 196     HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS,
 197     MARKING       = 1 << MARKING_BITPOS,
 198     EVACUATION    = 1 << EVACUATION_BITPOS,
 199     UPDATEREFS    = 1 << UPDATEREFS_BITPOS,
 200     TRAVERSAL     = 1 << TRAVERSAL_BITPOS,
 201   };
 202 
 203   enum ShenandoahDegenPoint {
 204     _degenerated_unset,
 205     _degenerated_traversal,
 206     _degenerated_outside_cycle,
 207     _degenerated_mark,
 208     _degenerated_evac,
 209     _degenerated_updaterefs,
 210     _DEGENERATED_LIMIT,
 211   };
 212 
 213   enum GCCycleMode {
 214     NONE,
 215     MINOR,
 216     MAJOR
 217   };
 218 
 219   static const char* degen_point_to_string(ShenandoahDegenPoint point) {
 220     switch (point) {
 221       case _degenerated_unset:
 222         return "<UNSET>";
 223       case _degenerated_traversal:
 224         return "Traversal";
 225       case _degenerated_outside_cycle:
 226         return "Outside of Cycle";
 227       case _degenerated_mark:
 228         return "Mark";
 229       case _degenerated_evac:
 230         return "Evacuation";
 231       case _degenerated_updaterefs:
 232         return "Update Refs";
 233       default:
 234         ShouldNotReachHere();
 235         return "ERROR";
 236     }
 237   };
 238 
 239 private:
 240   ShenandoahSharedBitmap _gc_state;
 241   ShenandoahHeapLock _lock;
 242   ShenandoahCollectorPolicy* _shenandoah_policy;
 243   SoftRefPolicy _soft_ref_policy;
 244   size_t _bitmap_size;
 245   size_t _bitmap_regions_per_slice;
 246   size_t _bitmap_bytes_per_slice;
 247   MemRegion _heap_region;
 248   MemRegion _bitmap0_region;
 249   MemRegion _bitmap1_region;
 250   MemRegion _aux_bitmap_region;
 251 
 252   ShenandoahHeapRegion** _regions;
 253   ShenandoahFreeSet* _free_set;
 254   ShenandoahCollectionSet* _collection_set;
 255 
 256   ShenandoahRegionIterator _update_refs_iterator;
 257 
 258   ShenandoahConcurrentMark* _scm;
 259   ShenandoahMarkCompact* _full_gc;
 260   ShenandoahTraversalGC* _traversal_gc;
 261   ShenandoahVerifier*  _verifier;
 262   ShenandoahPacer*  _pacer;
 263 
 264   ShenandoahControlThread* _control_thread;
 265 
 266   ShenandoahMonitoringSupport* _monitoring_support;
 267 
 268   ShenandoahPhaseTimings*      _phase_timings;
 269   ShenandoahAllocTracker*      _alloc_tracker;
 270 
 271   size_t _num_regions;
 272   size_t _initial_size;
 273 
 274   uint _max_workers;
 275   ShenandoahWorkGang* _workers;
 276   ShenandoahWorkGang* _safepoint_workers;
 277 
 278   volatile size_t _used;
 279   volatile size_t _committed;
 280 
 281   MarkBitMap _verification_bit_map;
 282   MarkBitMap _mark_bit_map0;
 283   MarkBitMap _mark_bit_map1;
 284   MarkBitMap* _complete_mark_bit_map;
 285   MarkBitMap* _next_mark_bit_map;
 286   MarkBitMap _aux_bit_map;
 287 
 288   HeapWord** _complete_top_at_mark_starts;
 289   HeapWord** _complete_top_at_mark_starts_base;
 290 
 291   HeapWord** _next_top_at_mark_starts;
 292   HeapWord** _next_top_at_mark_starts_base;
 293 
 294   volatile size_t _bytes_allocated_since_gc_start;
 295 
 296   ShenandoahSharedFlag _degenerated_gc_in_progress;
 297   ShenandoahSharedFlag _full_gc_in_progress;
 298   ShenandoahSharedFlag _full_gc_move_in_progress;
 299 
 300   ShenandoahSharedFlag _inject_alloc_failure;
 301 
 302   ShenandoahSharedFlag _process_references;
 303   ShenandoahSharedFlag _unload_classes;
 304 
 305   ShenandoahSharedEnumFlag<CancelState> _cancelled_concgc;
 306 
 307   ReferenceProcessor* _ref_processor;
 308 
 309   ShenandoahForwardedIsAliveClosure _forwarded_is_alive;
 310   ShenandoahIsAliveClosure _is_alive;
 311 
 312   ConcurrentGCTimer* _gc_timer;
 313 
 314   ShenandoahConnectionMatrix* _connection_matrix;
 315 
 316   GCMemoryManager _stw_memory_manager;
 317   GCMemoryManager _cycle_memory_manager;
 318 
 319   MemoryPool* _memory_pool;
 320 
 321   ShenandoahEvacOOMHandler _oom_evac_handler;
 322 
 323   ShenandoahSharedEnumFlag<GCCycleMode> _gc_cycle_mode;
 324 
 325   PLABStats* _mutator_gclab_stats;
 326   PLABStats* _collector_gclab_stats;
 327 
 328 #ifdef ASSERT
 329   int     _heap_expansion_count;
 330 #endif
 331 
 332 public:
 333   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 334 
 335   const char* name() const /* override */;
 336   HeapWord* allocate_new_tlab(size_t word_size) /* override */;
 337   void print_on(outputStream* st) const /* override */;
 338   void print_extended_on(outputStream *st) const /* override */;
 339 
 340   ShenandoahHeap::Name kind() const  /* override */{
 341     return CollectedHeap::Shenandoah;
 342   }
 343 
 344   jint initialize() /* override */;
 345   void post_initialize() /* override */;
 346   size_t capacity() const /* override */;
 347   size_t used() const /* override */;
 348   size_t committed() const;
 349   bool is_maximal_no_gc() const /* override */;
 350   size_t max_capacity() const /* override */;
 351   size_t initial_capacity() const /* override */;
 352   bool is_in(const void* p) const /* override */;
 353   bool is_scavengable(oop obj) /* override */;
 354   HeapWord* mem_allocate(size_t size,  Klass* klass, Thread* thread, bool* what) /* override */;
 355   void fill_with_object_impl(HeapWord* start, size_t words, bool zap = true) /* override */;
 356   bool can_elide_tlab_store_barriers() const /* override */;
 357   oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */;
 358   bool can_elide_initializing_store_barrier(oop new_obj) /* override */;
 359   bool card_mark_must_follow_store() const /* override */;
 360   void collect(GCCause::Cause cause) /* override */;
 361   void do_full_collection(bool clear_all_soft_refs) /* override */;
 362   AdaptiveSizePolicy* size_policy() /* override */;
 363   CollectorPolicy* collector_policy() const /* override */;
 364   SoftRefPolicy* soft_ref_policy() /* override */;
 365   void ensure_parsability(bool retire_tlabs) /* override */;
 366   HeapWord* block_start(const void* addr) const /* override */;
 367   size_t block_size(const HeapWord* addr) const /* override */;
 368   bool block_is_obj(const HeapWord* addr) const /* override */;
 369   jlong millis_since_last_gc() /* override */;
 370   void prepare_for_verify() /* override */;
 371   void print_gc_threads_on(outputStream* st) const /* override */;
 372   void gc_threads_do(ThreadClosure* tcl) const /* override */;
 373   void print_tracing_info() const /* override */;
 374   void verify(VerifyOption vo) /* override */;
 375   bool supports_tlab_allocation() const /* override */;
 376   size_t tlab_capacity(Thread *thr) const /* override */;
 377   void object_iterate(ObjectClosure* cl) /* override */;
 378   void safe_object_iterate(ObjectClosure* cl) /* override */;
 379   size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */;
 380   size_t max_tlab_size() const /* override */;
 381   uint oop_extra_words() /* override */;
 382   size_t tlab_used(Thread* ignored) const /* override */;
 383   void stop() /* override */;
 384   virtual void safepoint_synchronize_begin();
 385   virtual void safepoint_synchronize_end();
 386 
 387   WorkGang* get_safepoint_workers() { return _safepoint_workers; }
 388 
 389 #ifndef CC_INTERP
 390   void compile_prepare_oop(MacroAssembler* masm, Register obj) /* override */;
 391 #endif
 392 
 393   void register_nmethod(nmethod* nm);
 394   void unregister_nmethod(nmethod* nm);
 395 
 396   /* override: object pinning support */
 397   bool supports_object_pinning() const { return true; }
 398   oop pin_object(JavaThread* thread, oop obj);
 399   void unpin_object(JavaThread* thread, oop obj);
 400 
 401   static ShenandoahHeap* heap();
 402   static ShenandoahHeap* heap_no_check();
 403   static address in_cset_fast_test_addr();
 404   static address cancelled_concgc_addr();
 405   static address gc_state_addr();
 406 
 407   ShenandoahCollectorPolicy *shenandoahPolicy() const { return _shenandoah_policy; }
 408   ShenandoahPhaseTimings*   phase_timings()     const { return _phase_timings; }
 409   ShenandoahAllocTracker*   alloc_tracker()     const { return _alloc_tracker; }
 410 
 411   void accumulate_statistics_all_gclabs();
 412   PLABStats* mutator_gclab_stats()   const { return _mutator_gclab_stats; }
 413   PLABStats* collector_gclab_stats() const { return _collector_gclab_stats; }
 414 
 415   inline ShenandoahHeapRegion* const heap_region_containing(const void* addr) const;
 416   inline size_t heap_region_index_containing(const void* addr) const;
 417   inline bool requires_marking(const void* entry) const;
 418 
 419   template <class T>
 420   inline oop evac_update_with_forwarded(T* p);
 421 
 422   template <class T>
 423   inline oop maybe_update_with_forwarded(T* p);
 424 
 425   template <class T>
 426   inline oop maybe_update_with_forwarded_not_null(T* p, oop obj);
 427 
 428   template <class T>
 429   inline oop update_with_forwarded_not_null(T* p, oop obj);
 430 
 431   void trash_cset_regions();
 432 
 433   void stop_concurrent_marking();
 434 
 435   void prepare_for_concurrent_evacuation();
 436   void evacuate_and_update_roots();
 437   // Fixup roots after concurrent cycle failed
 438   void fixup_roots();
 439 
 440   void update_heap_references(bool concurrent);
 441 
 442   void roots_iterate(OopClosure* cl);
 443 
 444 private:
 445   void set_gc_state_all_threads(char state);
 446   void set_gc_state_mask(uint mask, bool value);
 447 
 448 public:
 449   void set_concurrent_mark_in_progress(bool in_progress);
 450   void set_evacuation_in_progress(bool in_progress);
 451   void set_update_refs_in_progress(bool in_progress);
 452   void set_degenerated_gc_in_progress(bool in_progress);
 453   void set_full_gc_in_progress(bool in_progress);
 454   void set_full_gc_move_in_progress(bool in_progress);
 455   void set_concurrent_traversal_in_progress(bool in_progress);
 456   void set_has_forwarded_objects(bool cond);
 457 
 458   void set_process_references(bool pr);
 459   void set_unload_classes(bool uc);
 460 
 461   inline bool is_stable() const;
 462   inline bool is_idle() const;
 463   inline bool is_concurrent_mark_in_progress() const;
 464   inline bool is_update_refs_in_progress() const;
 465   inline bool is_evacuation_in_progress() const;
 466   inline bool is_degenerated_gc_in_progress() const;
 467   inline bool is_full_gc_in_progress() const;
 468   inline bool is_full_gc_move_in_progress() const;
 469   inline bool is_concurrent_traversal_in_progress() const;
 470   inline bool has_forwarded_objects() const;
 471   inline bool is_gc_in_progress_mask(uint mask) const;
 472 
 473   char gc_state() const;
 474 
 475   bool process_references() const;
 476   bool unload_classes() const;
 477 
 478   bool is_minor_gc() const;
 479   bool is_major_gc() const;
 480   void set_cycle_mode(GCCycleMode gc_cycle_mode);
 481 
 482   inline bool region_in_collection_set(size_t region_index) const;
 483 
 484   // Mainly there to avoid accidentally calling the templated
 485   // method below with ShenandoahHeapRegion* which would be *wrong*.
 486   inline bool in_collection_set(ShenandoahHeapRegion* r) const;
 487 
 488   template <class T>
 489   inline bool in_collection_set(T obj) const;
 490 
 491   inline bool allocated_after_next_mark_start(HeapWord* addr) const;
 492   void set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 493   HeapWord* next_top_at_mark_start(HeapWord* region_base);
 494 
 495   inline bool allocated_after_complete_mark_start(HeapWord* addr) const;
 496   void set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 497   HeapWord* complete_top_at_mark_start(HeapWord* region_base);
 498 
 499   // Evacuates object src. Returns the evacuated object if this thread
 500   // succeeded, otherwise rolls back the evacuation and returns the
 501   // evacuated object by the competing thread.
 502   inline oop  evacuate_object(oop src, Thread* thread);
 503   inline bool cancelled_concgc() const;
 504   inline bool check_cancelled_concgc_and_yield(bool sts_active = true);
 505   inline bool try_cancel_concgc();
 506   inline void clear_cancelled_concgc();
 507 
 508   inline ShenandoahHeapRegion* const get_region(size_t region_idx) const;
 509   void heap_region_iterate(ShenandoahHeapRegionClosure& cl) const;
 510 
 511   ShenandoahFreeSet* free_set()             const { return _free_set; }
 512   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 513 
 514   ShenandoahConnectionMatrix* connection_matrix() const;
 515 
 516   void increase_used(size_t bytes);
 517   void decrease_used(size_t bytes);
 518 
 519   void set_used(size_t bytes);
 520 
 521   void increase_committed(size_t bytes);
 522   void decrease_committed(size_t bytes);
 523 
 524   void increase_allocated(size_t bytes);
 525 
 526   void notify_alloc(size_t words, bool waste);
 527 
 528   void handle_heap_shrinkage(double shrink_before);
 529 
 530   void reset_next_mark_bitmap();
 531   void reset_next_mark_bitmap_traversal();
 532 
 533   MarkBitMap* complete_mark_bit_map();
 534   MarkBitMap* next_mark_bit_map();
 535   inline bool is_marked_complete(oop obj) const;
 536   inline bool mark_next(oop obj) const;
 537   inline bool is_marked_next(oop obj) const;
 538   bool is_next_bitmap_clear();
 539   bool is_next_bitmap_clear_range(HeapWord* start, HeapWord* end);
 540   bool is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end);
 541 
 542   bool commit_bitmap_slice(ShenandoahHeapRegion *r);
 543   bool uncommit_bitmap_slice(ShenandoahHeapRegion *r);
 544 
 545   // Hint that the bitmap slice is not needed
 546   bool idle_bitmap_slice(ShenandoahHeapRegion* r);
 547   void activate_bitmap_slice(ShenandoahHeapRegion* r);
 548 
 549   bool is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self = false);
 550 
 551   void print_heap_regions_on(outputStream* st) const;
 552 
 553   size_t bytes_allocated_since_gc_start();
 554   void reset_bytes_allocated_since_gc_start();
 555 
 556   void trash_humongous_region_at(ShenandoahHeapRegion *r);
 557 
 558   virtual GrowableArray<GCMemoryManager*> memory_managers();
 559   virtual GrowableArray<MemoryPool*> memory_pools();
 560 
 561   ShenandoahMonitoringSupport* monitoring_support();
 562   ShenandoahConcurrentMark* concurrentMark() { return _scm; }
 563   ShenandoahMarkCompact* full_gc() { return _full_gc; }
 564   ShenandoahTraversalGC* traversal_gc();
 565   ShenandoahVerifier* verifier();
 566   ShenandoahPacer* pacer() const;
 567 
 568   ReferenceProcessor* ref_processor() { return _ref_processor;}
 569 
 570   WorkGang* workers() const { return _workers;}
 571 
 572   uint max_workers();
 573 
 574   void assert_gc_workers(uint nworker) PRODUCT_RETURN;
 575 
 576   void do_evacuation();
 577   ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
 578 
 579   void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_cset_regions = false, bool skip_humongous_continuation = false) const;
 580 
 581   // Delete entries for dead interned string and clean up unreferenced symbols
 582   // in symbol table, possibly in parallel.
 583   void unload_classes_and_cleanup_tables(bool full_gc);
 584 
 585   inline size_t num_regions() const { return _num_regions; }
 586 
 587   BoolObjectClosure* is_alive_closure();
 588 
 589   // Call before starting evacuation.
 590   void enter_evacuation();
 591   // Call after finished with evacuation.
 592   void leave_evacuation();
 593 
 594 private:
 595   template<class T>
 596   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 597 
 598   template<class T>
 599   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 600 
 601 public:
 602   template<class T>
 603   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 604 
 605   template<class T>
 606   inline void marked_object_safe_iterate(ShenandoahHeapRegion* region, T* cl);
 607 
 608   template<class T>
 609   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl);
 610 
 611   template<class T>
 612   inline void marked_object_oop_safe_iterate(ShenandoahHeapRegion* region, T* cl);
 613 
 614   GCTimer* gc_timer() const;
 615 
 616   void swap_mark_bitmaps();
 617 
 618   void cancel_concgc(GCCause::Cause cause);
 619 
 620   ShenandoahHeapLock* lock() { return &_lock; }
 621   void assert_heaplock_owned_by_current_thread() PRODUCT_RETURN;
 622   void assert_heaplock_not_owned_by_current_thread() PRODUCT_RETURN;
 623   void assert_heaplock_or_safepoint() PRODUCT_RETURN;
 624 
 625 public:
 626   typedef enum {
 627     _alloc_shared,      // Allocate common, outside of TLAB
 628     _alloc_shared_gc,   // Allocate common, outside of GCLAB
 629     _alloc_tlab,        // Allocate TLAB
 630     _alloc_gclab,       // Allocate GCLAB
 631     _ALLOC_LIMIT,
 632   } AllocType;
 633 
 634   static const char* alloc_type_to_string(AllocType type) {
 635     switch (type) {
 636       case _alloc_shared:
 637         return "Shared";
 638       case _alloc_shared_gc:
 639         return "Shared GC";
 640       case _alloc_tlab:
 641         return "TLAB";
 642       case _alloc_gclab:
 643         return "GCLAB";
 644       default:
 645         ShouldNotReachHere();
 646         return "";
 647     }
 648   }
 649 private:
 650 
 651   virtual void initialize_serviceability();
 652 
 653   HeapWord* allocate_new_lab(size_t word_size, AllocType type);
 654   HeapWord* allocate_memory_under_lock(size_t word_size, AllocType type, bool &new_region);
 655   HeapWord* allocate_memory(size_t word_size, AllocType type);
 656   // Shenandoah functionality.
 657   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 658   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 659   HeapWord* allocate_new_gclab(size_t word_size);
 660 
 661   template<class T>
 662   inline void do_object_marked_complete(T* cl, oop obj);
 663 
 664   ShenandoahControlThread* control_thread() { return _control_thread; }
 665 
 666 public:
 667   inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
 668   inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
 669 
 670 private:
 671   void ref_processing_init();
 672 
 673   GCTracer* tracer();
 674 
 675 private:
 676   uint64_t _alloc_seq_at_last_gc_start;
 677   uint64_t _alloc_seq_at_last_gc_end;
 678   size_t _used_at_last_gc;
 679 
 680 public:
 681   uint64_t alloc_seq_at_last_gc_end()   const { return _alloc_seq_at_last_gc_end;  }
 682   uint64_t alloc_seq_at_last_gc_start() const { return _alloc_seq_at_last_gc_start;}
 683   size_t used_at_last_gc()              const { return _used_at_last_gc;}
 684 
 685   void set_alloc_seq_gc_start();
 686   void set_alloc_seq_gc_end();
 687 
 688   void set_used_at_last_gc() {_used_at_last_gc = used();}
 689 
 690   void make_tlabs_parsable(bool retire_tlabs) /* override */;
 691 
 692   GCMemoryManager* cycle_memory_manager() { return &_cycle_memory_manager; }
 693   GCMemoryManager* stw_memory_manager()   { return &_stw_memory_manager; }
 694 
 695 public:
 696   // Entry points to STW GC operations, these cause a related safepoint, that then
 697   // call the entry method below
 698   void vmop_entry_init_mark();
 699   void vmop_entry_final_mark();
 700   void vmop_entry_final_evac();
 701   void vmop_entry_init_updaterefs();
 702   void vmop_entry_final_updaterefs();
 703   void vmop_entry_init_traversal();
 704   void vmop_entry_final_traversal();
 705   void vmop_entry_full(GCCause::Cause cause);
 706   void vmop_degenerated(ShenandoahDegenPoint point);
 707 
 708   // Entry methods to normally STW GC operations. These set up logging, monitoring
 709   // and workers for net VM operation
 710   void entry_init_mark();
 711   void entry_final_mark();
 712   void entry_final_evac();
 713   void entry_init_updaterefs();
 714   void entry_final_updaterefs();
 715   void entry_init_traversal();
 716   void entry_final_traversal();
 717   void entry_full(GCCause::Cause cause);
 718   void entry_degenerated(int point);
 719 
 720   // Entry methods to normally concurrent GC operations. These set up logging, monitoring
 721   // for concurrent operation.
 722   void entry_mark();
 723   void entry_preclean();
 724   void entry_cleanup();
 725   void entry_cleanup_bitmaps();
 726   void entry_cleanup_traversal();
 727   void entry_evac();
 728   void entry_updaterefs();
 729   void entry_traversal();
 730 
 731 private:
 732   // Actual work for the phases
 733   void op_init_mark();
 734   void op_final_mark();
 735   void op_final_evac();
 736   void op_init_updaterefs();
 737   void op_final_updaterefs();
 738   void op_init_traversal();
 739   void op_final_traversal();
 740   void op_full(GCCause::Cause cause);
 741   void op_degenerated(ShenandoahDegenPoint point);
 742   void op_degenerated_fail();
 743   void op_degenerated_futile();
 744 
 745   void op_mark();
 746   void op_preclean();
 747   void op_cleanup();
 748   void op_evac();
 749   void op_updaterefs();
 750   void op_cleanup_bitmaps();
 751   void op_cleanup_traversal();
 752   void op_traversal();
 753 
 754 private:
 755   void try_inject_alloc_failure();
 756   bool should_inject_alloc_failure();
 757 };
 758 
 759 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP