1 /*
   2  * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.
   7  *
   8  * This code is distributed in the hope that it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  11  * version 2 for more details (a copy is included in the LICENSE file that
  12  * accompanied this code).
  13  *
  14  * You should have received a copy of the GNU General Public License version
  15  * 2 along with this work; if not, write to the Free Software Foundation,
  16  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  17  *
  18  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  19  * or visit www.oracle.com if you need additional information or have any
  20  * questions.
  21  *
  22  */
  23 
  24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
  26 
  27 #include "gc/shared/cmBitMap.hpp"
  28 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  29 
  30 class ConcurrentGCTimer;
  31 
  32 class ShenandoahCollectorPolicy;
  33 class ShenandoahConnectionMatrix;
  34 class ShenandoahHeapRegion;
  35 class ShenandoahHeapRegionClosure;
  36 class ShenandoahHeapRegionSet;
  37 class ShenandoahCollectionSet;
  38 class ShenandoahFreeSet;
  39 class ShenandoahConcurrentMark;
  40 class ShenandoahPartialGC;
  41 class ShenandoahConcurrentThread;
  42 class ShenandoahMonitoringSupport;
  43 
  44 class SCMUpdateRefsClosure: public OopClosure {
  45 private:
  46   ShenandoahHeap* _heap;
  47 
  48   template <class T>
  49   inline void do_oop_work(T* p);
  50 
  51 public:
  52   SCMUpdateRefsClosure();
  53 
  54 public:
  55   inline void do_oop(oop* p);
  56   inline void do_oop(narrowOop* p);
  57 };
  58 
  59 #ifdef ASSERT
  60 class AssertToSpaceClosure : public OopClosure {
  61 private:
  62   template <class T>
  63   void do_oop_nv(T* p);
  64 public:
  65   void do_oop(narrowOop* p);
  66   void do_oop(oop* p);
  67 };
  68 #endif
  69 
  70 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
  71 public:
  72   bool do_object_b(oop p) { return true; }
  73 };
  74 
  75 
  76 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
  77 
  78 private:
  79   ShenandoahHeap* _heap;
  80 public:
  81   ShenandoahForwardedIsAliveClosure();
  82   void init(ShenandoahHeap* heap);
  83   bool do_object_b(oop obj);
  84 };
  85 
  86 
  87 // // A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
  88 // // It uses a new pauseless GC algorithm based on Brooks pointers.
  89 // // Derived from G1
  90 
  91 // //
  92 // // CollectedHeap
  93 // //    SharedHeap
  94 // //      ShenandoahHeap
  95 
  96 class ShenandoahHeap : public CollectedHeap {
  97   enum LockState { unlocked = 0, locked = 1 };
  98 
  99 public:
 100   class ShenandoahHeapLock : public StackObj {
 101   private:
 102     ShenandoahHeap* _heap;
 103 
 104   public:
 105     ShenandoahHeapLock(ShenandoahHeap* heap) : _heap(heap) {
 106       while (OrderAccess::load_acquire(& _heap->_heap_lock) == locked || Atomic::cmpxchg(locked, &_heap->_heap_lock, unlocked) == locked) {
 107         SpinPause();
 108       }
 109       assert(_heap->_heap_lock == locked, "sanity");
 110 
 111 #ifdef ASSERT
 112       assert(_heap->_heap_lock_owner == NULL, "must not be owned");
 113       _heap->_heap_lock_owner = Thread::current();
 114 #endif
 115     }
 116 
 117     ~ShenandoahHeapLock() {
 118 #ifdef ASSERT
 119       _heap->assert_heaplock_owned_by_current_thread();
 120       _heap->_heap_lock_owner = NULL;
 121 #endif
 122       OrderAccess::release_store_fence(&_heap->_heap_lock, unlocked);
 123     }
 124 
 125   };
 126 
 127 public:
 128   enum ShenandoahCancelCause {
 129     _oom_evacuation,
 130     _vm_stop,
 131   };
 132 private:
 133 
 134   static ShenandoahHeap* _pgc;
 135   ShenandoahCollectorPolicy* _shenandoah_policy;
 136   VirtualSpace _storage;
 137   ShenandoahHeapRegion* _first_region;
 138   HeapWord* _first_region_bottom;
 139 
 140   // Sortable array of regions
 141   ShenandoahHeapRegionSet* _ordered_regions;
 142   ShenandoahHeapRegionSet* _sorted_regions;
 143   ShenandoahFreeSet* _free_regions;
 144   ShenandoahCollectionSet* _collection_set;
 145 
 146   ShenandoahHeapRegion* _currentAllocationRegion;
 147   ShenandoahConcurrentMark* _scm;
 148   ShenandoahPartialGC* _partial_gc;
 149 
 150   ShenandoahConcurrentThread* _concurrent_gc_thread;
 151 
 152   ShenandoahMonitoringSupport* _monitoring_support;
 153 
 154   size_t _num_regions;
 155   size_t _max_regions;
 156   size_t _initialSize;
 157 #ifndef NDEBUG
 158   uint _numAllocs;
 159 #endif
 160   uint _max_workers;
 161 
 162   ShenandoahWorkGang* _workers;
 163 
 164 
 165   volatile size_t _used;
 166 
 167   CMBitMap _mark_bit_map0;
 168   CMBitMap _mark_bit_map1;
 169   CMBitMap* _complete_mark_bit_map;
 170   CMBitMap* _next_mark_bit_map;
 171 
 172   bool* _in_cset_fast_test;
 173   bool* _in_cset_fast_test_base;
 174   size_t _in_cset_fast_test_length;
 175 
 176   HeapWord** _complete_top_at_mark_starts;
 177   HeapWord** _complete_top_at_mark_starts_base;
 178 
 179   HeapWord** _next_top_at_mark_starts;
 180   HeapWord** _next_top_at_mark_starts_base;
 181 
 182   volatile jbyte _cancelled_concgc;
 183 
 184   size_t _bytes_allocated_since_cm;
 185   size_t _bytes_allocated_during_cm;
 186   size_t _bytes_allocated_during_cm_start;
 187   size_t _max_allocated_gc;
 188   size_t _allocated_last_gc;
 189   size_t _used_start_gc;
 190 
 191   unsigned int _concurrent_mark_in_progress;
 192 
 193   bool _full_gc_in_progress;
 194 
 195   unsigned int _evacuation_in_progress;
 196   bool _need_update_refs;
 197   bool _need_reset_bitmaps;
 198 
 199   ReferenceProcessor* _ref_processor;
 200 
 201   ShenandoahForwardedIsAliveClosure isAlive;
 202 
 203   ConcurrentGCTimer* _gc_timer;
 204 
 205   // See allocate_memory()
 206   volatile jbyte _heap_lock;
 207 
 208   ShenandoahConnectionMatrix* _connection_matrix;
 209 
 210 #ifdef ASSERT
 211   volatile Thread* _heap_lock_owner;
 212 #endif
 213 
 214 public:
 215   ShenandoahHeap(ShenandoahCollectorPolicy* policy);
 216 
 217   HeapWord *first_region_bottom() { return _first_region_bottom; }
 218 
 219   const char* name() const /* override */;
 220   HeapWord* allocate_new_tlab(size_t word_size) /* override */;
 221   void print_on(outputStream* st) const /* override */;
 222 
 223   ShenandoahHeap::Name kind() const  /* override */{
 224     return CollectedHeap::ShenandoahHeap;
 225   }
 226 
 227   jint initialize() /* override */;
 228   void post_initialize() /* override */;
 229   size_t capacity() const /* override */;
 230   size_t used() const /* override */;
 231   bool is_maximal_no_gc() const /* override */;
 232   size_t max_capacity() const /* override */;
 233   size_t min_capacity() const /* override */;
 234   bool is_in(const void* p) const /* override */;
 235   bool is_scavengable(const void* addr) /* override */;
 236   HeapWord* mem_allocate(size_t size, bool* what) /* override */;
 237   bool can_elide_tlab_store_barriers() const /* override */;
 238   oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */;
 239   bool can_elide_initializing_store_barrier(oop new_obj) /* override */;
 240   bool card_mark_must_follow_store() const /* override */;
 241   void collect(GCCause::Cause) /* override */;
 242   void do_full_collection(bool clear_all_soft_refs) /* override */;
 243   AdaptiveSizePolicy* size_policy() /* override */;
 244   CollectorPolicy* collector_policy() const /* override */;
 245   void ensure_parsability(bool retire_tlabs) /* override */;
 246   HeapWord* block_start(const void* addr) const /* override */;
 247   size_t block_size(const HeapWord* addr) const /* override */;
 248   bool block_is_obj(const HeapWord* addr) const /* override */;
 249   jlong millis_since_last_gc() /* override */;
 250   void prepare_for_verify() /* override */;
 251   void print_gc_threads_on(outputStream* st) const /* override */;
 252   void gc_threads_do(ThreadClosure* tcl) const /* override */;
 253   void print_tracing_info() const /* override */;
 254   void verify(VerifyOption vo) /* override */;
 255   bool supports_tlab_allocation() const /* override */;
 256   size_t tlab_capacity(Thread *thr) const /* override */;
 257   void object_iterate(ObjectClosure* cl) /* override */;
 258   void safe_object_iterate(ObjectClosure* cl) /* override */;
 259   size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */;
 260   size_t max_tlab_size() const /* override */;
 261   void resize_all_tlabs() /* override */;
 262   void accumulate_statistics_all_gclabs() /* override */;
 263   HeapWord* tlab_post_allocation_setup(HeapWord* obj) /* override */;
 264   uint oop_extra_words() /* override */;
 265   size_t tlab_used(Thread* ignored) const /* override */;
 266   void stop() /* override */;
 267 
 268 #ifndef CC_INTERP
 269   void compile_prepare_oop(MacroAssembler* masm, Register obj) /* override */;
 270 #endif
 271 
 272   void register_nmethod(nmethod* nm);
 273   void unregister_nmethod(nmethod* nm);
 274 
 275   void pin_object(oop o) /* override */;
 276   void unpin_object(oop o) /* override */;
 277 
 278   static ShenandoahHeap* heap();
 279   static ShenandoahHeap* heap_no_check();
 280   static size_t conservative_max_heap_alignment();
 281   static address in_cset_fast_test_addr();
 282   static address cancelled_concgc_addr();
 283 
 284   ShenandoahCollectorPolicy *shenandoahPolicy() { return _shenandoah_policy;}
 285 
 286   inline ShenandoahHeapRegion* heap_region_containing(const void* addr) const;
 287   inline uint heap_region_index_containing(const void* addr) const;
 288   inline bool requires_marking(const void* entry) const;
 289   template <class T>
 290   inline oop maybe_update_oop_ref(T* p);
 291 
 292   void recycle_dirty_regions();
 293 
 294   void start_concurrent_marking();
 295   void stop_concurrent_marking();
 296   inline bool concurrent_mark_in_progress();
 297   static address concurrent_mark_in_progress_addr();
 298 
 299   void prepare_for_concurrent_evacuation();
 300   void evacuate_and_update_roots();
 301 
 302   void do_partial_collection();
 303   void verify_matrix();
 304 private:
 305   void set_evacuation_in_progress(bool in_progress);
 306 public:
 307   inline bool is_evacuation_in_progress();
 308   void set_evacuation_in_progress_concurrently(bool in_progress);
 309   void set_evacuation_in_progress_at_safepoint(bool in_progress);
 310 
 311   void set_full_gc_in_progress(bool in_progress);
 312   bool is_full_gc_in_progress() const;
 313 
 314   inline bool need_update_refs() const;
 315   void set_need_update_refs(bool update_refs);
 316 
 317   inline bool region_in_collection_set(size_t region_index) const;
 318 
 319   void set_region_in_collection_set(size_t region_index, bool b);
 320 
 321   // Mainly there to avoid accidentally calling the templated
 322   // method below with ShenandoahHeapRegion* which would be *wrong*.
 323   inline bool in_collection_set(ShenandoahHeapRegion* r) const;
 324 
 325   template <class T>
 326   inline bool in_collection_set(T obj) const;
 327 
 328   void clear_cset_fast_test();
 329 
 330   inline bool allocated_after_next_mark_start(HeapWord* addr) const;
 331   void set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 332   HeapWord* next_top_at_mark_start(HeapWord* region_base);
 333 
 334   inline bool allocated_after_complete_mark_start(HeapWord* addr) const;
 335   void set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
 336   HeapWord* complete_top_at_mark_start(HeapWord* region_base);
 337 
 338   inline oop  evacuate_object(oop src, Thread* thread);
 339   inline bool cancelled_concgc() const;
 340   inline void set_cancelled_concgc(bool v);
 341   inline bool try_cancel_concgc() const;
 342   void clear_cancelled_concgc();
 343 
 344   ShenandoahHeapRegionSet* regions() { return _ordered_regions;}
 345   ShenandoahFreeSet* free_regions();
 346   ShenandoahCollectionSet* collection_set() { return _collection_set; }
 347   void clear_free_regions();
 348   void add_free_region(ShenandoahHeapRegion* r);
 349 
 350   ShenandoahConnectionMatrix* connection_matrix();
 351 
 352   void increase_used(size_t bytes);
 353   void decrease_used(size_t bytes);
 354 
 355   void set_used(size_t bytes);
 356   size_t calculateUsed();
 357 
 358   size_t garbage();
 359 
 360   void reset_next_mark_bitmap(WorkGang* gang);
 361   void reset_complete_mark_bitmap(WorkGang* gang);
 362 
 363   CMBitMap* complete_mark_bit_map();
 364   CMBitMap* next_mark_bit_map();
 365   inline bool is_marked_complete(oop obj) const;
 366   inline bool mark_next(oop obj) const;
 367   inline bool is_marked_next(oop obj) const;
 368   bool is_next_bitmap_clear();
 369   bool is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end);
 370 
 371   void parallel_evacuate_region(ShenandoahHeapRegion* from_region);
 372 
 373   template <class T>
 374   inline oop update_oop_ref_not_null(T* p, oop obj);
 375 
 376   template <class T>
 377   inline oop maybe_update_oop_ref_not_null(T* p, oop obj);
 378 
 379   void print_heap_regions(outputStream* st = tty) const;
 380   void print_all_refs(const char* prefix);
 381   void print_heap_locations(HeapWord* start, HeapWord* end);
 382 
 383   void calculate_matrix(int* connections);
 384   void print_matrix(int* connections);
 385 
 386   size_t bytes_allocated_since_cm();
 387   void set_bytes_allocated_since_cm(size_t bytes);
 388 
 389   size_t max_allocated_gc();
 390 
 391   void reclaim_humongous_region_at(ShenandoahHeapRegion* r);
 392 
 393   VirtualSpace* storage() const;
 394 
 395   ShenandoahMonitoringSupport* monitoring_support();
 396   ShenandoahConcurrentMark* concurrentMark() { return _scm;}
 397   ShenandoahPartialGC* partial_gc();
 398 
 399   ReferenceProcessor* ref_processor() { return _ref_processor;}
 400 
 401   WorkGang* workers() const { return _workers;}
 402 
 403   uint max_workers();
 404 
 405   void do_evacuation();
 406   ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
 407 
 408   void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions = false, bool skip_humongous_continuation = false) const;
 409 
 410   void verify_heap_after_evacuation();
 411   void verify_heap_after_marking();
 412 
 413   // Delete entries for dead interned string and clean up unreferenced symbols
 414   // in symbol table, possibly in parallel.
 415   void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true);
 416 
 417   size_t num_regions();
 418   size_t max_regions();
 419 
 420   // TODO: consider moving this into ShenandoahHeapRegion.
 421 
 422   template<class T>
 423   inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
 424 
 425   GCTimer* gc_timer() const;
 426 
 427   void swap_mark_bitmaps();
 428 
 429   void cancel_concgc(GCCause::Cause cause);
 430   void cancel_concgc(ShenandoahCancelCause cause);
 431 
 432   void assert_heaplock_owned_by_current_thread() PRODUCT_RETURN;
 433   void assert_heaplock_or_safepoint() PRODUCT_RETURN;
 434 
 435 private:
 436   HeapWord* allocate_new_tlab(size_t word_size, bool mark);
 437   HeapWord* allocate_memory_under_lock(size_t word_size);
 438   HeapWord* allocate_memory(size_t word_size, bool evacuating);
 439   // Shenandoah functionality.
 440   inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
 441   HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
 442   HeapWord* allocate_new_gclab(size_t word_size);
 443 
 444   void roots_iterate(OopClosure* cl);
 445 
 446   template<class T>
 447   inline void do_marked_object(CMBitMap* bitmap, T* cl, oop obj);
 448 
 449   ShenandoahConcurrentThread* concurrent_thread() { return _concurrent_gc_thread; }
 450 
 451   inline bool mark_next_no_checks(oop obj) const;
 452 
 453   void parallel_evacuate();
 454 
 455 public:
 456   inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
 457   inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
 458 
 459 private:
 460   void evacuate_region(ShenandoahHeapRegion* from_region, ShenandoahHeapRegion* to_region);
 461 
 462 #ifdef ASSERT
 463   void verify_evacuated_region(ShenandoahHeapRegion* from_region);
 464 #endif
 465 
 466   inline void copy_object(oop p, HeapWord* s, size_t words);
 467   void verify_copy(oop p, oop c);
 468   void verify_heap_size_consistency();
 469 private:
 470   void verify_heap_after_update_refs();
 471   void verify_regions_after_update_refs();
 472 
 473   void ref_processing_init();
 474 
 475   GCTracer* tracer();
 476 
 477   bool call_from_write_barrier(bool evacuating);
 478   void grow_heap_by(size_t num_regions);
 479   void ensure_new_regions(size_t num_new_regions);
 480 
 481   void verify_evacuation(ShenandoahHeapRegion* from_region);
 482   void set_concurrent_mark_in_progress(bool in_progress);
 483 
 484   void oom_during_evacuation();
 485 
 486   void verify_live();
 487   void verify_liveness_after_concurrent_mark();
 488 
 489   HeapWord* allocate_memory_work(size_t word_size);
 490   HeapWord* allocate_large_memory(size_t word_size);
 491 
 492 #ifdef ASSERT
 493   void set_from_region_protection(bool protect);
 494 #endif
 495 
 496   const char* cancel_cause_to_string(ShenandoahCancelCause cause);
 497 
 498 };
 499 
 500 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP