1 /*
   2  * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/markBitMap.hpp"
  28 #include "gc/shenandoah/shenandoahHeapLock.hpp"
  29 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  30 
  31 class ConcurrentGCTimer;
  32 
  33 class ShenandoahAllocTracker;
  34 class ShenandoahCollectorPolicy;
  35 class ShenandoahConnectionMatrix;
  36 class ShenandoahPhaseTimings;
  37 class ShenandoahHeapRegion;
  38 class ShenandoahHeapRegionClosure;
  39 class ShenandoahHeapRegionSet;
  40 class ShenandoahCollectionSet;
  41 class ShenandoahFreeSet;
  42 class ShenandoahConcurrentMark;
  43 class ShenandoahPartialGC;
  44 class ShenandoahVerifier;
  45 class ShenandoahConcurrentThread;
  46 class ShenandoahMonitoringSupport;
  47 
  48 class ShenandoahUpdateRefsClosure: public OopClosure {
  49 private:
  50   ShenandoahHeap* _heap;
  51 
  52   template <class T>
  53   inline void do_oop_work(T* p);
  54 
  55 public:
  56   ShenandoahUpdateRefsClosure();
  57   inline void do_oop(oop* p);
  58   inline void do_oop(narrowOop* p);
  59 };
  60 
  61 #ifdef ASSERT
  62 class ShenandoahAssertToSpaceClosure : public OopClosure {
  63 private:
  64   template <class T>
  65   void do_oop_nv(T* p);
  66 public:
  67   void do_oop(narrowOop* p);
  68   void do_oop(oop* p);
  69 };
  70 #endif
  71 
  72 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
  73 public:
  74   bool do_object_b(oop p) { return true; }
  75 };
  76 
  77 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
  78 private:
  79   ShenandoahHeap* _heap;
  80 public:
  81   ShenandoahForwardedIsAliveClosure();
  82   void init(ShenandoahHeap* heap) {
  83     _heap = heap;
  84   }
  85   bool do_object_b(oop obj);
  86 };
  87 
  88 class ShenandoahIsAliveClosure: public BoolObjectClosure {
  89 private:
  90   ShenandoahHeap* _heap;
  91 public:
  92   ShenandoahIsAliveClosure();
  93   void init(ShenandoahHeap* heap) {
  94     _heap = heap;
  95   }
  96   bool do_object_b(oop obj);
  97 };
  98 
  99 
 100 // // A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
 101 // // It uses a new pauseless GC algorithm based on Brooks pointers.
 102 // // Derived from G1
 103 
 104 // //
 105 // // CollectedHeap
 106 // //    SharedHeap
 107 // //      ShenandoahHeap
 108 
 109 class ShenandoahHeap : public CollectedHeap {
 110   enum CancelState {
 111 
 112     // Normal state. GC has not been cancelled and is open for cancellation.
 113     // Worker threads can suspend for safepoint.
 114     CANCELLABLE,
 115 
 116     // GC has been cancelled. Worker threads can not suspend for
 117     // safepoint but must finish their work as soon as possible.
 118     CANCELLED,
 119 
 120     // GC has not been cancelled and must not be cancelled. At least
 121     // one worker thread checks for pending safepoint and may suspend
 122     // if a safepoint is pending.
 123     NOT_CANCELLED
 124 
 125   };
 126 
 127 public:
 128   enum ShenandoahCancelCause {
 129     _oom_evacuation,
 130     _vm_stop,
 131   };
 132 private:
 133   ShenandoahHeapLock _lock;
 134   ShenandoahCollectorPolicy* _shenandoah_policy;
 135   size_t _bitmap_size;
 136   size_t _bitmap_words_per_region;
 137   MemRegion _heap_region;
 138   MemRegion _bitmap_region;
 139   MemRegion _aux_bitmap_region;
 140 
 141   // Sortable array of regions
 142   ShenandoahHeapRegionSet* _ordered_regions;
 143   ShenandoahFreeSet* _free_regions;
 144   ShenandoahCollectionSet* _collection_set;
 145 
 146   ShenandoahConcurrentMark* _scm;
 147   ShenandoahPartialGC* _partial_gc;
 148   ShenandoahVerifier*  _verifier;
 149 
 150   ShenandoahConcurrentThread* _concurrent_gc_thread;
 151 
 152   ShenandoahMonitoringSupport* _monitoring_support;
 153 
 154   ShenandoahPhaseTimings*      _phase_timings;
 155   ShenandoahAllocTracker*      _alloc_tracker;
 156 
 157   size_t _num_regions;
 158   size_t _initial_size;
 159 
 160   uint _max_workers;
 161   ShenandoahWorkGang* _workers;
 162   ShenandoahWorkGang* _safepoint_workers;
 163 
 164   volatile size_t _used;
 165   volatile size_t _committed;
 166 
 167   MarkBitMap _verification_bit_map;
 168   MarkBitMap _aux_bit_map;
 169   MarkBitMap _mark_bit_map;
 170   bool _bitmap_valid;
 171 
 172   HeapWord** _top_at_mark_starts;
 173   HeapWord** _top_at_mark_starts_base;
 174 
 175   volatile jbyte _cancelled_concgc;
 176 
 177   size_t _bytes_allocated_since_cm;
 178   size_t _bytes_allocated_during_cm;
 179   size_t _allocated_last_gc;
 180   size_t _used_start_gc;
 181 
 182   char _concurrent_mark_in_progress;
 183 
 184   bool _full_gc_in_progress;
 185   bool _update_refs_in_progress;
 186   bool _concurrent_partial_in_progress;
 187 
 188   unsigned int _evacuation_in_progress;
 189   bool _need_update_refs;
 190   bool _need_reset_bitmap;
 191 
 192   ReferenceProcessor* _ref_processor;
 193 
 194   ShenandoahForwardedIsAliveClosure _forwarded_is_alive;
 195   ShenandoahIsAliveClosure _is_alive;
 196 
 197   ConcurrentGCTimer* _gc_timer;
 198 
 199   // See allocate_memory()
 200   volatile jbyte _heap_lock;
 201 
 202   ShenandoahConnectionMatrix* _connection_matrix;
 203 
 204 #ifdef ASSERT
 205   Thread* volatile _heap_lock_owner;
 206   int     _heap_expansion_count;
 207 #endif
 208 
 209 public:
 210   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 211 
 212   const char* name() const /* override */;
 213   HeapWord* allocate_new_tlab(size_t word_size) /* override */;
 214   void print_on(outputStream* st) const /* override */;
 215   void print_extended_on(outputStream *st) const /* override */;
 216 
 217   ShenandoahHeap::Name kind() const  /* override */{
 218     return CollectedHeap::ShenandoahHeap;
 219   }
 220 
 221   jint initialize() /* override */;
 222   void post_initialize() /* override */;
 223   size_t capacity() const /* override */;
 224   size_t used() const /* override */;
 225   size_t committed() const;
 226   bool is_maximal_no_gc() const /* override */;
 227   size_t max_capacity() const /* override */;
 228   size_t initial_capacity() const /* override */;
 229   bool is_in(const void* p) const /* override */;
 230   bool is_scavengable(const void* addr) /* override */;
 231   HeapWord* mem_allocate(size_t size, bool* what) /* override */;
 232   bool can_elide_tlab_store_barriers() const /* override */;
 233   oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */;
 234   bool can_elide_initializing_store_barrier(oop new_obj) /* override */;
 235   bool card_mark_must_follow_store() const /* override */;
 236   void collect(GCCause::Cause cause) /* override */;
 237   void do_full_collection(bool clear_all_soft_refs) /* override */;
 238   AdaptiveSizePolicy* size_policy() /* override */;
 239   CollectorPolicy* collector_policy() const /* override */;
 240   void ensure_parsability(bool retire_tlabs) /* override */;
 241   HeapWord* block_start(const void* addr) const /* override */;
 242   size_t block_size(const HeapWord* addr) const /* override */;
 243   bool block_is_obj(const HeapWord* addr) const /* override */;
 244   jlong millis_since_last_gc() /* override */;
 245   void prepare_for_verify() /* override */;
 246   void print_gc_threads_on(outputStream* st) const /* override */;
 247   void gc_threads_do(ThreadClosure* tcl) const /* override */;
 248   void print_tracing_info() const /* override */;
 249   void verify(VerifyOption vo) /* override */;
 250   bool supports_tlab_allocation() const /* override */;
 251   size_t tlab_capacity(Thread *thr) const /* override */;
 252   void object_iterate(ObjectClosure* cl) /* override */;
 253   void safe_object_iterate(ObjectClosure* cl) /* override */;
 254   size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */;
 255   size_t max_tlab_size() const /* override */;
 256   void resize_all_tlabs() /* override */;
 257   void accumulate_statistics_all_gclabs() /* override */;
 258   HeapWord* tlab_post_allocation_setup(HeapWord* obj) /* override */;
 259   uint oop_extra_words() /* override */;
 260   size_t tlab_used(Thread* ignored) const /* override */;
 261   void stop() /* override */;
 262 
 263   WorkGang* get_safepoint_workers() { return _safepoint_workers; }
 264 
 265 #ifndef CC_INTERP
 266   void compile_prepare_oop(MacroAssembler* masm, Register obj) /* override */;
 267 #endif
 268 
 269   void register_nmethod(nmethod* nm);
 270   void unregister_nmethod(nmethod* nm);
 271 
 272   void pin_object(oop o) /* override */;
 273   void unpin_object(oop o) /* override */;
 274 
 275   static ShenandoahHeap* heap();
 276   static ShenandoahHeap* heap_no_check();
 277   static size_t conservative_max_heap_alignment();
 278   static address in_cset_fast_test_addr();
 279   static address cancelled_concgc_addr();
 280 
 281   ShenandoahCollectorPolicy *shenandoahPolicy() const { return _shenandoah_policy; }
 282   ShenandoahPhaseTimings*   phase_timings()     const { return _phase_timings; }
 283   ShenandoahAllocTracker*   alloc_tracker()     const { return _alloc_tracker; }
 284 
 285   inline ShenandoahHeapRegion* heap_region_containing(const void* addr) const;
 286   inline size_t heap_region_index_containing(const void* addr) const;
 287   inline bool requires_marking(const void* entry) const;
 288   template <class T>
 289   inline oop maybe_update_oop_ref(T* p);
 290 
 291   template <class T>
 292   inline oop evac_update_oop_ref(T* p, bool& evac);
 293 
 294   void trash_cset_regions();
 295 
 296   void start_concurrent_marking();
 297   void stop_concurrent_marking();
 298   inline bool concurrent_mark_in_progress() const;
 299   static address concurrent_mark_in_progress_addr();
 300 
 301   void set_concurrent_partial_in_progress(bool in_progress);
 302   inline bool is_concurrent_partial_in_progress() const;
 303 
 304   void prepare_for_concurrent_evacuation();
 305   void evacuate_and_update_roots();
 306   // Fixup roots after concurrent cycle failed
 307   void fixup_roots();
 308 
 309   void update_heap_references(ShenandoahHeapRegionSet* regions, bool concurrent);
 310   void concurrent_update_heap_references();
 311   void prepare_update_refs();
 312   void finish_update_refs();
 313 
 314   void roots_iterate(OopClosure* cl);
 315 
 316 private:
 317   void set_evacuation_in_progress(bool in_progress);
 318 
 319 public:
 320   inline bool is_evacuation_in_progress() const;
 321   void set_evacuation_in_progress_concurrently(bool in_progress);
 322   void set_evacuation_in_progress_at_safepoint(bool in_progress);
 323   static address evacuation_in_progress_addr();
 324 
 325   void set_full_gc_in_progress(bool in_progress);
 326   bool is_full_gc_in_progress() const;
 327 
 328   void set_update_refs_in_progress(bool in_progress);
 329   bool is_update_refs_in_progress() const;
 330   static address update_refs_in_progress_addr();
 331 
 332   inline bool need_update_refs() const;
 333   void set_need_update_refs(bool update_refs);
 334 
 335   inline bool region_in_collection_set(size_t region_index) const;
 336 
 337   // Mainly there to avoid accidentally calling the templated
 338   // method below with ShenandoahHeapRegion* which would be *wrong*.
 339   inline bool in_collection_set(ShenandoahHeapRegion* r) const;
 340 
 341   template <class T>
 342   inline bool in_collection_set(T obj) const;
 343 
 344   inline bool allocated_after_mark_start(HeapWord* addr) const;
 345   void set_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 346   HeapWord* top_at_mark_start(HeapWord* region_base);
 347 
 348   // Evacuates object src. Returns the evacuated object if this thread
 349   // succeeded, otherwise rolls back the evacuation and returns the
 350   // evacuated object by the competing thread. 'succeeded' is an out
 351   // param and set to true if this thread succeeded, otherwise to false.
 352   inline oop  evacuate_object(oop src, Thread* thread, bool& evacuated);
 353   inline bool cancelled_concgc() const;
 354   inline bool check_cancelled_concgc_and_yield(bool sts_active = true);
 355   inline bool try_cancel_concgc();
 356   inline void clear_cancelled_concgc();
 357 
 358   ShenandoahHeapRegionSet* regions() const { return _ordered_regions;}
 359   ShenandoahFreeSet* free_regions() const  { return _free_regions; }
 360   ShenandoahCollectionSet* collection_set() const { return _collection_set; }
 361   void clear_free_regions();
 362   void add_free_region(ShenandoahHeapRegion* r);
 363 
 364   ShenandoahConnectionMatrix* connection_matrix() const;
 365 
 366   void increase_used(size_t bytes);
 367   void decrease_used(size_t bytes);
 368 
 369   void set_used(size_t bytes);
 370 
 371   void increase_committed(size_t bytes);
 372   void decrease_committed(size_t bytes);
 373 
 374   void handle_heap_shrinkage();
 375 
 376   size_t garbage();
 377 
 378   void reset_mark_bitmap(WorkGang* gang);
 379 
 380   MarkBitMap* mark_bit_map();
 381   inline bool is_marked(oop obj) const;
 382   inline bool mark(oop obj);
 383   bool is_bitmap_clear();
 384   bool is_bitmap_clear_range(HeapWord* start, HeapWord* end);
 385 
 386   bool is_bitmap_valid() const { return _bitmap_valid; }
 387   void set_bitmap_valid(bool valid) { _bitmap_valid = valid; }
 388 
 389   bool commit_bitmaps(ShenandoahHeapRegion* r);
 390   bool uncommit_bitmaps(ShenandoahHeapRegion* r);
 391 
 392   template <class T>
 393   inline oop update_oop_ref_not_null(T* p, oop obj);
 394 
 395   template <class T>
 396   inline oop maybe_update_oop_ref_not_null(T* p, oop obj);
 397 
 398   void print_heap_regions_on(outputStream* st) const;
 399 
 400   size_t bytes_allocated_since_cm();
 401   void set_bytes_allocated_since_cm(size_t bytes);
 402 
 403   size_t trash_humongous_region_at(ShenandoahHeapRegion *r);
 404 
 405   ShenandoahMonitoringSupport* monitoring_support();
 406   ShenandoahConcurrentMark* concurrentMark() { return _scm;}
 407   ShenandoahPartialGC* partial_gc();
 408   ShenandoahVerifier* verifier();
 409 
 410   ReferenceProcessor* ref_processor() { return _ref_processor;}
 411 
 412   WorkGang* workers() const { return _workers;}
 413 
 414   uint max_workers();
 415 
 416   void assert_gc_workers(uint nworker) PRODUCT_RETURN;
 417 
 418   void do_evacuation();
 419   ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
 420 
 421   void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_cset_regions = false, bool skip_humongous_continuation = false) const;
 422 
 423   // Delete entries for dead interned string and clean up unreferenced symbols
 424   // in symbol table, possibly in parallel.
 425   void unload_classes_and_cleanup_tables(bool full_gc);
 426 
 427   inline size_t num_regions() const { return _num_regions; }
 428 
 429   BoolObjectClosure* is_alive_closure();
 430 
 431 private:
 432   template<class T>
 433   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 434 
 435   template<class T>
 436   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
 437 
 438 public:
 439   template<class T>
 440   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 441 
 442   template<class T>
 443   inline void marked_object_safe_iterate(ShenandoahHeapRegion* region, T* cl);
 444 
 445   template<class T>
 446   inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl);
 447 
 448   template<class T>
 449   inline void marked_object_oop_safe_iterate(ShenandoahHeapRegion* region, T* cl);
 450 
 451   GCTimer* gc_timer() const;
 452 
 453   void cancel_concgc(GCCause::Cause cause);
 454   void cancel_concgc(ShenandoahCancelCause cause);
 455 
 456   ShenandoahHeapLock* lock() { return &_lock; }
 457   void assert_heaplock_owned_by_current_thread() PRODUCT_RETURN;
 458   void assert_heaplock_not_owned_by_current_thread() PRODUCT_RETURN;
 459   void assert_heaplock_or_safepoint() PRODUCT_RETURN;
 460 
 461 public:
 462   typedef enum {
 463     _alloc_shared,      // Allocate common, outside of TLAB
 464     _alloc_shared_gc,   // Allocate common, outside of GCLAB
 465     _alloc_tlab,        // Allocate TLAB
 466     _alloc_gclab,       // Allocate GCLAB
 467     _ALLOC_LIMIT,
 468   } AllocType;
 469 
 470   static const char* alloc_type_to_string(AllocType type) {
 471     switch (type) {
 472       case _alloc_shared:
 473         return "Shared";
 474       case _alloc_shared_gc:
 475         return "Shared GC";
 476       case _alloc_tlab:
 477         return "TLAB";
 478       case _alloc_gclab:
 479         return "GCLAB";
 480       default:
 481         ShouldNotReachHere();
 482         return "";
 483     }
 484   }
 485 private:
 486   HeapWord* allocate_new_lab(size_t word_size, AllocType type);
 487   HeapWord* allocate_memory_under_lock(size_t word_size, AllocType type, bool &new_region);
 488   HeapWord* allocate_memory(size_t word_size, AllocType type);
 489   // Shenandoah functionality.
 490   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 491   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 492   HeapWord* allocate_new_gclab(size_t word_size);
 493 
 494   template<class T>
 495   inline void do_marked_object(T* cl, oop obj);
 496 
 497   ShenandoahConcurrentThread* concurrent_thread() { return _concurrent_gc_thread; }
 498 
 499   inline bool mark_no_checks(oop obj);
 500 
 501 public:
 502   inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
 503   inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
 504 
 505 private:
 506   void ref_processing_init();
 507 
 508   GCTracer* tracer();
 509 
 510   void set_concurrent_mark_in_progress(bool in_progress);
 511 
 512   void oom_during_evacuation();
 513 
 514   HeapWord* allocate_large_memory(size_t word_size);
 515 
 516   const char* cancel_cause_to_string(ShenandoahCancelCause cause);
 517 
 518 private:
 519   size_t* _recycled_regions;
 520 
 521   uint64_t _alloc_seq_at_last_gc_start;
 522   uint64_t _alloc_seq_at_last_gc_end;
 523   size_t _used_at_last_gc;
 524 
 525 public:
 526   void recycle_trash_assist(size_t limit);
 527   void recycle_trash();
 528 
 529   uint64_t alloc_seq_at_last_gc_end()   const { return _alloc_seq_at_last_gc_end;  }
 530   uint64_t alloc_seq_at_last_gc_start() const { return _alloc_seq_at_last_gc_start;}
 531   size_t used_at_last_gc()              const { return _used_at_last_gc;}
 532 
 533   void set_alloc_seq_gc_start();
 534   void set_alloc_seq_gc_end();
 535 
 536   void set_used_at_last_gc() {_used_at_last_gc = used();}
 537 
 538   void make_tlabs_parsable(bool retire_tlabs) /* override */;
 539 };
 540 
 541 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP