1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP
26
27 #include "gc/shared/cmBitMap.hpp"
28 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
29
30 class ConcurrentGCTimer;
31
32 class ShenandoahCollectorPolicy;
33 class ShenandoahHeapRegion;
34 class ShenandoahHeapRegionClosure;
35 class ShenandoahHeapRegionSet;
36 class ShenandoahCollectionSet;
37 class ShenandoahFreeSet;
38 class ShenandoahConcurrentMark;
39 class ShenandoahConcurrentThread;
40 class ShenandoahMonitoringSupport;
41
42 class SCMUpdateRefsClosure: public OopClosure {
43 private:
44 ShenandoahHeap* _heap;
45
46 template <class T>
47 inline void do_oop_work(T* p);
48
49 public:
50 SCMUpdateRefsClosure();
51
52 public:
53 inline void do_oop(oop* p);
54 inline void do_oop(narrowOop* p);
55 };
56
57 #ifdef ASSERT
58 class AssertToSpaceClosure : public OopClosure {
59 private:
60 template <class T>
61 void do_oop_nv(T* p);
62 public:
63 void do_oop(narrowOop* p);
64 void do_oop(oop* p);
65 };
66 #endif
67
68 class ShenandoahAlwaysTrueClosure : public BoolObjectClosure {
69 public:
70 bool do_object_b(oop p) { return true; }
71 };
72
73
74 class ShenandoahForwardedIsAliveClosure: public BoolObjectClosure {
75
76 private:
77 ShenandoahHeap* _heap;
78 public:
79 ShenandoahForwardedIsAliveClosure();
80 void init(ShenandoahHeap* heap);
81 bool do_object_b(oop obj);
82 };
83
84
85 // // A "ShenandoahHeap" is an implementation of a java heap for HotSpot.
86 // // It uses a new pauseless GC algorithm based on Brooks pointers.
87 // // Derived from G1
88
89 // //
90 // // CollectedHeap
91 // // SharedHeap
92 // // ShenandoahHeap
93
94 class ShenandoahHeap : public CollectedHeap {
95 enum LockState { unlocked = 0, locked = 1 };
96
97 public:
98 class ShenandoahHeapLock : public StackObj {
99 private:
100 ShenandoahHeap* _heap;
101
102 public:
103 ShenandoahHeapLock(ShenandoahHeap* heap) : _heap(heap) {
104 while (OrderAccess::load_acquire(& _heap->_heap_lock) == locked || Atomic::cmpxchg(locked, &_heap->_heap_lock, unlocked) == locked) {
105 SpinPause();
106 }
107 assert(_heap->_heap_lock == locked, "sanity");
108
109 #ifdef ASSERT
110 assert(_heap->_heap_lock_owner == NULL, "must not be owned");
111 _heap->_heap_lock_owner = Thread::current();
112 #endif
113 }
114
115 ~ShenandoahHeapLock() {
116 #ifdef ASSERT
117 _heap->assert_heaplock_owned_by_current_thread();
118 _heap->_heap_lock_owner = NULL;
119 #endif
120 OrderAccess::release_store_fence(&_heap->_heap_lock, unlocked);
121 }
122
123 };
124
125 public:
126 enum ShenandoahCancelCause {
127 _oom_evacuation,
128 _vm_stop,
129 };
130 private:
131
132 static ShenandoahHeap* _pgc;
133 ShenandoahCollectorPolicy* _shenandoah_policy;
134 VirtualSpace _storage;
135 ShenandoahHeapRegion* _first_region;
136 HeapWord* _first_region_bottom;
137
138 // Sortable array of regions
139 ShenandoahHeapRegionSet* _ordered_regions;
140 ShenandoahHeapRegionSet* _sorted_regions;
141 ShenandoahFreeSet* _free_regions;
142 ShenandoahCollectionSet* _collection_set;
143 ShenandoahHeapRegion* _currentAllocationRegion;
144 ShenandoahConcurrentMark* _scm;
145
146
147
148 ShenandoahConcurrentThread* _concurrent_gc_thread;
149
150 ShenandoahMonitoringSupport* _monitoring_support;
151
152 size_t _num_regions;
153 size_t _max_regions;
154 size_t _initialSize;
155 #ifndef NDEBUG
156 uint _numAllocs;
157 #endif
158 uint _max_workers;
159
160 ShenandoahWorkGang* _workers;
161
162
163 volatile size_t _used;
164
165 CMBitMap _mark_bit_map0;
166 CMBitMap _mark_bit_map1;
167 CMBitMap* _complete_mark_bit_map;
168 CMBitMap* _next_mark_bit_map;
169
170 bool* _in_cset_fast_test;
171 bool* _in_cset_fast_test_base;
172 size_t _in_cset_fast_test_length;
173
174 HeapWord** _complete_top_at_mark_starts;
175 HeapWord** _complete_top_at_mark_starts_base;
176
177 HeapWord** _next_top_at_mark_starts;
178 HeapWord** _next_top_at_mark_starts_base;
179
180 volatile jbyte _cancelled_concgc;
181
182 size_t _bytes_allocated_since_cm;
183 size_t _bytes_allocated_during_cm;
184 size_t _bytes_allocated_during_cm_start;
185 size_t _max_allocated_gc;
186 size_t _allocated_last_gc;
187 size_t _used_start_gc;
188
189 unsigned int _concurrent_mark_in_progress;
190
191 bool _full_gc_in_progress;
192
193 unsigned int _evacuation_in_progress;
194 bool _need_update_refs;
195 bool _need_reset_bitmaps;
196
197 ReferenceProcessor* _ref_processor;
198
199 ShenandoahForwardedIsAliveClosure isAlive;
200
201 ConcurrentGCTimer* _gc_timer;
202
203 // See allocate_memory()
204 volatile jbyte _heap_lock;
205
206 #ifdef ASSERT
207 volatile Thread* _heap_lock_owner;
208 #endif
209
210 public:
211 ShenandoahHeap(ShenandoahCollectorPolicy* policy);
212
213 HeapWord *first_region_bottom() { return _first_region_bottom; }
214
215 const char* name() const /* override */;
216 HeapWord* allocate_new_tlab(size_t word_size) /* override */;
217 void print_on(outputStream* st) const /* override */;
218
219 ShenandoahHeap::Name kind() const /* override */{
220 return CollectedHeap::ShenandoahHeap;
221 }
222
223 jint initialize() /* override */;
224 void post_initialize() /* override */;
225 size_t capacity() const /* override */;
226 size_t used() const /* override */;
227 bool is_maximal_no_gc() const /* override */;
228 size_t max_capacity() const /* override */;
229 size_t min_capacity() const /* override */;
230 bool is_in(const void* p) const /* override */;
231 bool is_scavengable(const void* addr) /* override */;
232 HeapWord* mem_allocate(size_t size, bool* what) /* override */;
233 bool can_elide_tlab_store_barriers() const /* override */;
234 oop new_store_pre_barrier(JavaThread* thread, oop new_obj) /* override */;
235 bool can_elide_initializing_store_barrier(oop new_obj) /* override */;
236 bool card_mark_must_follow_store() const /* override */;
237 void collect(GCCause::Cause) /* override */;
238 void do_full_collection(bool clear_all_soft_refs) /* override */;
239 AdaptiveSizePolicy* size_policy() /* override */;
240 CollectorPolicy* collector_policy() const /* override */;
241 void ensure_parsability(bool retire_tlabs) /* override */;
242 HeapWord* block_start(const void* addr) const /* override */;
243 size_t block_size(const HeapWord* addr) const /* override */;
244 bool block_is_obj(const HeapWord* addr) const /* override */;
245 jlong millis_since_last_gc() /* override */;
246 void prepare_for_verify() /* override */;
247 void print_gc_threads_on(outputStream* st) const /* override */;
248 void gc_threads_do(ThreadClosure* tcl) const /* override */;
249 void print_tracing_info() const /* override */;
250 void verify(VerifyOption vo) /* override */;
251 bool supports_tlab_allocation() const /* override */;
252 size_t tlab_capacity(Thread *thr) const /* override */;
253 void object_iterate(ObjectClosure* cl) /* override */;
254 void safe_object_iterate(ObjectClosure* cl) /* override */;
255 size_t unsafe_max_tlab_alloc(Thread *thread) const /* override */;
256 size_t max_tlab_size() const /* override */;
257 void resize_all_tlabs() /* override */;
258 void accumulate_statistics_all_gclabs() /* override */;
259 HeapWord* tlab_post_allocation_setup(HeapWord* obj) /* override */;
260 uint oop_extra_words() /* override */;
261 size_t tlab_used(Thread* ignored) const /* override */;
262 void stop() /* override */;
263
264 #ifndef CC_INTERP
265 void compile_prepare_oop(MacroAssembler* masm, Register obj) /* override */;
266 #endif
267
268 void register_nmethod(nmethod* nm);
269 void unregister_nmethod(nmethod* nm);
270
271 void pin_object(oop o) /* override */;
272 void unpin_object(oop o) /* override */;
273
274 static ShenandoahHeap* heap();
275 static ShenandoahHeap* heap_no_check();
276 static size_t conservative_max_heap_alignment();
277 static address in_cset_fast_test_addr();
278 static address cancelled_concgc_addr();
279
280 ShenandoahCollectorPolicy *shenandoahPolicy() { return _shenandoah_policy;}
281
282 inline ShenandoahHeapRegion* heap_region_containing(const void* addr) const;
283 inline uint heap_region_index_containing(const void* addr) const;
284 inline bool requires_marking(const void* entry) const;
285 template <class T>
286 inline oop maybe_update_oop_ref(T* p);
287
288 void recycle_dirty_regions();
289
290 void start_concurrent_marking();
291 void stop_concurrent_marking();
292 inline bool concurrent_mark_in_progress();
293 static address concurrent_mark_in_progress_addr();
294
295 void prepare_for_concurrent_evacuation();
296 void evacuate_and_update_roots();
297
298 private:
299 void set_evacuation_in_progress(bool in_progress);
300 public:
301 inline bool is_evacuation_in_progress();
302 void set_evacuation_in_progress_concurrently(bool in_progress);
303 void set_evacuation_in_progress_at_safepoint(bool in_progress);
304
305 void set_full_gc_in_progress(bool in_progress);
306 bool is_full_gc_in_progress() const;
307
308 inline bool need_update_refs() const;
309 void set_need_update_refs(bool update_refs);
310
311 inline bool region_in_collection_set(size_t region_index) const;
312
313 void set_region_in_collection_set(size_t region_index, bool b);
314
315 // Mainly there to avoid accidentally calling the templated
316 // method below with ShenandoahHeapRegion* which would be *wrong*.
317 inline bool in_collection_set(ShenandoahHeapRegion* r) const;
318
319 template <class T>
320 inline bool in_collection_set(T obj) const;
321
322 void clear_cset_fast_test();
323
324 inline bool allocated_after_next_mark_start(HeapWord* addr) const;
325 void set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
326 HeapWord* next_top_at_mark_start(HeapWord* region_base);
327
328 inline bool allocated_after_complete_mark_start(HeapWord* addr) const;
329 void set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr);
330 HeapWord* complete_top_at_mark_start(HeapWord* region_base);
331
332 inline oop evacuate_object(oop src, Thread* thread);
333 inline bool cancelled_concgc() const;
334 inline void set_cancelled_concgc(bool v);
335 inline bool try_cancel_concgc() const;
336 void clear_cancelled_concgc();
337
338 ShenandoahHeapRegionSet* regions() { return _ordered_regions;}
339 ShenandoahFreeSet* free_regions();
340 void clear_free_regions();
341 void add_free_region(ShenandoahHeapRegion* r);
342
343 void increase_used(size_t bytes);
344 void decrease_used(size_t bytes);
345
346 void set_used(size_t bytes);
347 size_t calculateUsed();
348
349 size_t garbage();
350
351 void reset_next_mark_bitmap(WorkGang* gang);
352 void reset_complete_mark_bitmap(WorkGang* gang);
353
354 CMBitMap* complete_mark_bit_map();
355 CMBitMap* next_mark_bit_map();
356 inline bool is_marked_complete(oop obj) const;
357 inline bool mark_next(oop obj) const;
358 inline bool is_marked_next(oop obj) const;
359 bool is_next_bitmap_clear();
360 bool is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end);
361
362 void parallel_evacuate_region(ShenandoahHeapRegion* from_region);
363
364 template <class T>
365 inline oop update_oop_ref_not_null(T* p, oop obj);
366
367 template <class T>
368 inline oop maybe_update_oop_ref_not_null(T* p, oop obj);
369
370 void print_heap_regions(outputStream* st = tty) const;
371 void print_all_refs(const char* prefix);
372 void print_heap_locations(HeapWord* start, HeapWord* end);
373
374 void calculate_matrix(int* connections);
375 void print_matrix(int* connections);
376
377 size_t bytes_allocated_since_cm();
378 void set_bytes_allocated_since_cm(size_t bytes);
379
380 size_t max_allocated_gc();
381
382 void reclaim_humongous_region_at(ShenandoahHeapRegion* r);
383
384 VirtualSpace* storage() const;
385
386 ShenandoahMonitoringSupport* monitoring_support();
387 ShenandoahConcurrentMark* concurrentMark() { return _scm;}
388
389 ReferenceProcessor* ref_processor() { return _ref_processor;}
390
391 WorkGang* workers() const { return _workers;}
392
393 uint max_workers();
394
395 void do_evacuation();
396 ShenandoahHeapRegion* next_compaction_region(const ShenandoahHeapRegion* r);
397
398 void heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions = false, bool skip_humongous_continuation = false) const;
399
400 void verify_heap_after_evacuation();
401
402 // Delete entries for dead interned string and clean up unreferenced symbols
403 // in symbol table, possibly in parallel.
404 void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true);
405
406 size_t num_regions();
407 size_t max_regions();
408
409 // TODO: consider moving this into ShenandoahHeapRegion.
410
411 template<class T>
412 inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl);
413
414 GCTimer* gc_timer() const;
415
416 void swap_mark_bitmaps();
417
418 void cancel_concgc(GCCause::Cause cause);
419 void cancel_concgc(ShenandoahCancelCause cause);
420
421 void assert_heaplock_owned_by_current_thread() PRODUCT_RETURN;
422 void assert_heaplock_or_safepoint() PRODUCT_RETURN;
423
424 private:
425 HeapWord* allocate_new_tlab(size_t word_size, bool mark);
426 HeapWord* allocate_memory_under_lock(size_t word_size);
427 HeapWord* allocate_memory(size_t word_size, bool evacuating);
428 // Shenandoah functionality.
429 inline HeapWord* allocate_from_gclab(Thread* thread, size_t size);
430 HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size);
431 HeapWord* allocate_new_gclab(size_t word_size);
432
433 void roots_iterate(OopClosure* cl);
434
435 template<class T>
436 inline void do_marked_object(CMBitMap* bitmap, T* cl, oop obj);
437
438 ShenandoahConcurrentThread* concurrent_thread() { return _concurrent_gc_thread; }
439
440 inline bool mark_next_no_checks(oop obj) const;
441
442 void parallel_evacuate();
443
444 inline oop atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c);
445 inline oop atomic_compare_exchange_oop(oop n, oop* addr, oop c);
446
447 void evacuate_region(ShenandoahHeapRegion* from_region, ShenandoahHeapRegion* to_region);
448
449 #ifdef ASSERT
450 void verify_evacuated_region(ShenandoahHeapRegion* from_region);
451 #endif
452
453 inline void copy_object(oop p, HeapWord* s, size_t words);
454 void verify_copy(oop p, oop c);
455 void verify_heap_size_consistency();
456 void verify_heap_after_marking();
457 void verify_heap_after_update_refs();
458 void verify_regions_after_update_refs();
459
460 void ref_processing_init();
461
462 GCTracer* tracer();
463 ShenandoahCollectionSet* collection_set() { return _collection_set; }
464
465 bool call_from_write_barrier(bool evacuating);
466 void grow_heap_by(size_t num_regions);
467 void ensure_new_regions(size_t num_new_regions);
468
469 void verify_evacuation(ShenandoahHeapRegion* from_region);
470 void set_concurrent_mark_in_progress(bool in_progress);
471
472 void oom_during_evacuation();
473
474 void verify_live();
475 void verify_liveness_after_concurrent_mark();
476
477 HeapWord* allocate_memory_work(size_t word_size);
478 HeapWord* allocate_large_memory(size_t word_size);
479
480 #ifdef ASSERT
481 void set_from_region_protection(bool protect);
482 #endif
483
484 const char* cancel_cause_to_string(ShenandoahCancelCause cause);
485
486 };
487
488 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_HPP