1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
26
27 #include "gc/shared/cmBitMap.hpp"
28 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
29
30 class ConcurrentGCTimer;
31
32 class ShenandoahCollectorPolicy;
33 class ShenandoahConnectionMatrix;
34 class ShenandoahHeapRegion;
35 class ShenandoahHeapRegionClosure;
36 class ShenandoahHeapRegionSet;
37 class ShenandoahCollectionSet;
38 class ShenandoahFreeSet;
39 class ShenandoahConcurrentMark;
40 class ShenandoahPartialGC;
41 class ShenandoahConcurrentThread;
42 class ShenandoahMonitoringSupport;
43
44 class SCMUpdateRefsClosure: public OopClosure {
45 private:
46 ShenandoahHeap* _heap;
47
48 template <class T>
49 inline void do_oop_work(T* p);
50
51 public:
52 SCMUpdateRefsClosure();
53
54 public:
55 inline void do_oop(oop* p);
56 inline void do_oop(narrowOop* p);
57 };
58
59 #ifdef ASSERT
60 class AssertToSpaceClosure : public OopClosure {
61 private:
62 template <class T>
63 void do_oop_nv(T* p);
64 public:
65 void do_oop(narrowOop* p);
66 void do_oop(oop* p);
67 };
68 #endif
69
70 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
71 public:
72 bool do_object_b(oop p) { return true; }
73 };
74
75
76 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
77
78 private:
79 ShenandoahHeap* _heap;
80 public:
81 ShenandoahForwardedIsAliveClosure();
82 void init(ShenandoahHeap* heap);
83 bool do_object_b(oop obj);
84 };
85
86
87 // // A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
88 // // It uses a new pauseless GC algorithm based on Brooks pointers.
89 // // Derived from G1
90
91 // //
92 // // CollectedHeap
93 // // SharedHeap
94 // // ShenandoahHeap
95
96 class ShenandoahHeap : public CollectedHeap {
97 enum LockState { unlocked = 0, locked = 1 };
98
99 public:
100 class ShenandoahHeapLock : public StackObj {
101 private:
102 ShenandoahHeap* _heap;
103
104 public:
105 ShenandoahHeapLock(ShenandoahHeap* heap) : _heap(heap) {
106 while (OrderAccess::load_acquire(& _heap->_heap_lock) == locked || Atomic::cmpxchg(locked, &_heap->_heap_lock, unlocked) == locked) {
107 SpinPause();
108 }
109 assert(_heap->_heap_lock == locked, "sanity");
110
111 #ifdef ASSERT
112 assert(_heap->_heap_lock_owner == NULL, "must not be owned");
113 _heap->_heap_lock_owner = Thread::current();
114 #endif
115 }
116
117 ~ShenandoahHeapLock() {
118 #ifdef ASSERT
119 _heap->assert_heaplock_owned_by_current_thread();
120 _heap->_heap_lock_owner = NULL;
121 #endif
122 OrderAccess::release_store_fence(&_heap->_heap_lock, unlocked);
123 }
124
125 };
126
127 public:
128 enum ShenandoahCancelCause {
129 _oom_evacuation,
130 _vm_stop,
131 };
132 private:
133
134 ShenandoahCollectorPolicy* _shenandoah_policy;
135 VirtualSpace _storage;
136 size_t _bitmap_size;
137 MemRegion _heap_region;
138
139 ShenandoahHeapRegion* _first_region;
140
141 // Sortable array of regions
142 ShenandoahHeapRegionSet* _ordered_regions;
143 ShenandoahFreeSet* _free_regions;
144 ShenandoahCollectionSet* _collection_set;
145
146 ShenandoahConcurrentMark* _scm;
147 ShenandoahPartialGC* _partial_gc;
148
149 ShenandoahConcurrentThread* _concurrent_gc_thread;
150
151 ShenandoahMonitoringSupport* _monitoring_support;
152
153 size_t _num_regions;
154 size_t _max_regions;
155 size_t _initialSize;
156
157 uint _max_workers;
158 ShenandoahWorkGang* _workers;
159
160 volatile size_t _used;
161
162 CMBitMap _verification_bit_map;
163 CMBitMap _mark_bit_map0;
164 CMBitMap _mark_bit_map1;
165 CMBitMap* _complete_mark_bit_map;
166 CMBitMap* _next_mark_bit_map;
167
168 bool* _in_cset_fast_test;
169 bool* _in_cset_fast_test_base;
170 size_t _in_cset_fast_test_length;
171
172 HeapWord** _complete_top_at_mark_starts;
173 HeapWord** _complete_top_at_mark_starts_base;
174
175 HeapWord** _next_top_at_mark_starts;
176 HeapWord** _next_top_at_mark_starts_base;
177
178 volatile jbyte _cancelled_concgc;
179
180 size_t _bytes_allocated_since_cm;
181 size_t _bytes_allocated_during_cm;
182 size_t _allocated_last_gc;
183 size_t _used_start_gc;
184
185 unsigned int _concurrent_mark_in_progress;
186
187 bool _full_gc_in_progress;
188 bool _update_refs_in_progress;
189
190 unsigned int _evacuation_in_progress;
191 bool _need_update_refs;
192 bool _need_reset_bitmaps;
193
194 ReferenceProcessor* _ref_processor;
195
196 ShenandoahForwardedIsAliveClosure isAlive;
197
198 ConcurrentGCTimer* _gc_timer;
199
200 // See allocate_memory()
201 volatile jbyte _heap_lock;
202
203 ShenandoahConnectionMatrix* _connection_matrix;
204
205 #ifdef ASSERT
206 volatile Thread* _heap_lock_owner;
207 #endif
208
209 public:
210 ShenandoahHeap(ShenandoahCollectorPolicy* policy);
211
212 const char* name() const /* override */;
213 HeapWord* allocate_new_tlab(size_t word_size) /* override */;
214 void print_on(outputStream* st) const /* override */;
215
216 ShenandoahHeap::Name kind() const /* override */{
217 return CollectedHeap::ShenandoahHeap;
218 }
219
220 jint initialize() /* override */;
221 void post_initialize() /* override */;
222 size_t capacity() const /* override */;
223 size_t used() const /* override */;
224 bool is_maximal_no_gc() const /* override */;
225 size_t max_capacity() const /* override */;
226 size_t min_capacity() const /* override */;
227 bool is_in(const void* p) const /* override */;
228 bool is_scavengable(const void* addr) /* override */;
229 HeapWord* mem_allocate(size_t size, bool* what) /* override */;
230 bool can_elide_tlab_store_barriers() const /* override */;
231 oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */;
232 bool can_elide_initializing_store_barrier(oop new_obj) /* override */;
233 bool card_mark_must_follow_store() const /* override */;
234 void collect(GCCause::Cause) /* override */;
235 void do_full_collection(bool clear_all_soft_refs) /* override */;
236 AdaptiveSizePolicy* size_policy() /* override */;
237 CollectorPolicy* collector_policy() const /* override */;
238 void ensure_parsability(bool retire_tlabs) /* override */;
239 HeapWord* block_start(const void* addr) const /* override */;
240 size_t block_size(const HeapWord* addr) const /* override */;
241 bool block_is_obj(const HeapWord* addr) const /* override */;
242 jlong millis_since_last_gc() /* override */;
243 void prepare_for_verify() /* override */;
244 void print_gc_threads_on(outputStream* st) const /* override */;
245 void gc_threads_do(ThreadClosure* tcl) const /* override */;
246 void print_tracing_info() const /* override */;
247 void verify(VerifyOption vo) /* override */;
248 bool supports_tlab_allocation() const /* override */;
249 size_t tlab_capacity(Thread *thr) const /* override */;
250 void object_iterate(ObjectClosure* cl) /* override */;
251 void safe_object_iterate(ObjectClosure* cl) /* override */;
252 size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */;
253 size_t max_tlab_size() const /* override */;
254 void resize_all_tlabs() /* override */;
255 void accumulate_statistics_all_gclabs() /* override */;
256 HeapWord* tlab_post_allocation_setup(HeapWord* obj) /* override */;
257 uint oop_extra_words() /* override */;
258 size_t tlab_used(Thread* ignored) const /* override */;
259 void stop() /* override */;
260
261 #ifndef CC_INTERP
262 void compile_prepare_oop(MacroAssembler* masm, Register obj) /* override */;
263 #endif
264
265 void register_nmethod(nmethod* nm);
266 void unregister_nmethod(nmethod* nm);
267
268 void pin_object(oop o) /* override */;
269 void unpin_object(oop o) /* override */;
270
271 static ShenandoahHeap* heap();
272 static ShenandoahHeap* heap_no_check();
273 static size_t conservative_max_heap_alignment();
274 static address in_cset_fast_test_addr();
275 static address cancelled_concgc_addr();
276
277 ShenandoahCollectorPolicy *shenandoahPolicy() { return _shenandoah_policy;}
278
279 inline ShenandoahHeapRegion* heap_region_containing(const void* addr) const;
280 inline size_t heap_region_index_containing(const void* addr) const;
281 inline bool requires_marking(const void* entry) const;
282 template <class T>
283 inline oop maybe_update_oop_ref(T* p);
284
285 void recycle_dirty_regions();
286
287 void start_concurrent_marking();
288 void stop_concurrent_marking();
289 inline bool concurrent_mark_in_progress();
290 static address concurrent_mark_in_progress_addr();
291
292 void prepare_for_concurrent_evacuation();
293 void evacuate_and_update_roots();
294
295 void do_partial_collection();
296
297 void concurrent_update_heap_references();
298 void prepare_update_refs();
299 void finish_update_refs();
300 void verify_update_refs();
301
302 private:
303 void set_evacuation_in_progress(bool in_progress);
304 public:
305 inline bool is_evacuation_in_progress();
306 void set_evacuation_in_progress_concurrently(bool in_progress);
307 void set_evacuation_in_progress_at_safepoint(bool in_progress);
308
309 void set_full_gc_in_progress(bool in_progress);
310 bool is_full_gc_in_progress() const;
311
312 void set_update_refs_in_progress(bool in_progress);
313 bool is_update_refs_in_progress() const;
314 static address update_refs_in_progress_addr();
315
316 inline bool need_update_refs() const;
317 void set_need_update_refs(bool update_refs);
318
319 inline bool region_in_collection_set(size_t region_index) const;
320
321 void set_region_in_collection_set(size_t region_index, bool b);
322
323 // Mainly there to avoid accidentally calling the templated
324 // method below with ShenandoahHeapRegion* which would be *wrong*.
325 inline bool in_collection_set(ShenandoahHeapRegion* r) const;
326
327 template <class T>
328 inline bool in_collection_set(T obj) const;
329
330 void clear_cset_fast_test();
331
332 inline bool allocated_after_next_mark_start(HeapWord* addr) const;
333 void set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
334 HeapWord* next_top_at_mark_start(HeapWord* region_base);
335
336 inline bool allocated_after_complete_mark_start(HeapWord* addr) const;
337 void set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
338 HeapWord* complete_top_at_mark_start(HeapWord* region_base);
339
340 // Evacuates object src. Returns the evacuated object if this thread
341 // succeeded, otherwise rolls back the evacuation and returns the
342 // evacuated object by the competing thread. 'succeeded' is an out
343 // param and set to true if this thread succeeded, otherwise to false.
344 inline oop evacuate_object(oop src, Thread* thread, bool& evacuated);
345 inline bool cancelled_concgc() const;
346 inline bool try_cancel_concgc();
347 inline void clear_cancelled_concgc();
348
349 ShenandoahHeapRegionSet* regions() { return _ordered_regions;}
350 ShenandoahFreeSet* free_regions();
351 ShenandoahCollectionSet* collection_set() { return _collection_set; }
352 void clear_free_regions();
353 void add_free_region(ShenandoahHeapRegion* r);
354
355 ShenandoahConnectionMatrix* connection_matrix();
356
357 void increase_used(size_t bytes);
358 void decrease_used(size_t bytes);
359
360 void set_used(size_t bytes);
361 size_t calculateUsed();
362
363 size_t garbage();
364
365 void reset_next_mark_bitmap(WorkGang* gang);
366 void reset_complete_mark_bitmap(WorkGang* gang);
367
368 CMBitMap* complete_mark_bit_map();
369 CMBitMap* next_mark_bit_map();
370 inline bool is_marked_complete(oop obj) const;
371 inline bool mark_next(oop obj) const;
372 inline bool is_marked_next(oop obj) const;
373 bool is_next_bitmap_clear();
374 bool is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end);
375
376 void parallel_evacuate_region(ShenandoahHeapRegion* from_region);
377
378 template <class T>
379 inline oop update_oop_ref_not_null(T* p, oop obj);
380
381 template <class T>
382 inline oop maybe_update_oop_ref_not_null(T* p, oop obj);
383
384 void print_heap_regions(outputStream* st = tty) const;
385 void print_all_refs(const char* prefix);
386 void print_heap_locations(HeapWord* start, HeapWord* end);
387
388 size_t bytes_allocated_since_cm();
389 void set_bytes_allocated_since_cm(size_t bytes);
390
391 void reclaim_humongous_region_at(ShenandoahHeapRegion* r);
392
393 VirtualSpace* storage() const;
394
395 ShenandoahMonitoringSupport* monitoring_support();
396 ShenandoahConcurrentMark* concurrentMark() { return _scm;}
397 ShenandoahPartialGC* partial_gc();
398
399 ReferenceProcessor* ref_processor() { return _ref_processor;}
400
401 WorkGang* workers() const { return _workers;}
402
403 uint max_workers();
404
405 void do_evacuation();
406 ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
407
408 void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions = false, bool skip_humongous_continuation = false) const;
409
410 void verify_heap_after_evacuation();
411 void verify_heap_after_marking();
412 void verify_heap_reachable_at_safepoint();
413
414 // Delete entries for dead interned string and clean up unreferenced symbols
415 // in symbol table, possibly in parallel.
416 void unload_classes_and_cleanup_tables();
417
418 size_t num_regions();
419 size_t max_regions();
420
421 // TODO: consider moving this into ShenandoahHeapRegion.
422
423 private:
424 template<class T>
425 inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
426
427 template<class T>
428 inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit);
429
430 public:
431 template<class T>
432 inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
433
434 template<class T>
435 inline void marked_object_safe_iterate(ShenandoahHeapRegion* region, T* cl);
436
437 template<class T>
438 inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl);
439
440 template<class T>
441 inline void marked_object_oop_safe_iterate(ShenandoahHeapRegion* region, T* cl);
442
443 GCTimer* gc_timer() const;
444
445 void swap_mark_bitmaps();
446
447 void cancel_concgc(GCCause::Cause cause);
448 void cancel_concgc(ShenandoahCancelCause cause);
449
450 void assert_heaplock_owned_by_current_thread() PRODUCT_RETURN;
451 void assert_heaplock_or_safepoint() PRODUCT_RETURN;
452
453 private:
454 HeapWord* allocate_new_tlab(size_t word_size, bool mark);
455 HeapWord* allocate_memory_under_lock(size_t word_size);
456 HeapWord* allocate_memory(size_t word_size, bool evacuating);
457 // Shenandoah functionality.
458 inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
459 HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
460 HeapWord* allocate_new_gclab(size_t word_size);
461
462 void roots_iterate(OopClosure* cl);
463
464 template<class T>
465 inline void do_marked_object(CMBitMap* bitmap, T* cl, oop obj);
466
467 ShenandoahConcurrentThread* concurrent_thread() { return _concurrent_gc_thread; }
468
469 inline bool mark_next_no_checks(oop obj) const;
470
471 void parallel_evacuate();
472
473 public:
474 inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
475 inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
476
477 private:
478 #ifdef ASSERT
479 void verify_evacuated_region(ShenandoahHeapRegion* from_region);
480 #endif
481
482 inline void copy_object(oop p, HeapWord* s, size_t words);
483 void verify_copy(oop p, oop c);
484 void verify_heap_size_consistency();
485 private:
486 void ref_processing_init();
487
488 GCTracer* tracer();
489
490 void grow_heap_by(size_t num_regions);
491 void ensure_new_regions(size_t num_new_regions);
492
493 void verify_evacuation(ShenandoahHeapRegion* from_region);
494 void set_concurrent_mark_in_progress(bool in_progress);
495
496 void oom_during_evacuation();
497
498 HeapWord* allocate_memory_work(size_t word_size);
499 HeapWord* allocate_large_memory(size_t word_size);
500
501 const char* cancel_cause_to_string(ShenandoahCancelCause cause);
502
503 };
504
505 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP