1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "memory/allocation.hpp"
26
27 #include "gc/shared/gcTimer.hpp"
28 #include "gc/shared/gcTraceTime.inline.hpp"
29 #include "gc/shared/parallelCleaning.hpp"
30
31 #include "gc/shenandoah/brooksPointer.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
34 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
35 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
36 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
37 #include "gc/shenandoah/shenandoahConcurrentThread.hpp"
38 #include "gc/shenandoah/shenandoahFreeSet.hpp"
39 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
40 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
41 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
42 #include "gc/shenandoah/shenandoahHumongous.hpp"
43 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
44 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
45 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
46 #include "gc/shenandoah/shenandoahPartialGC.hpp"
47 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
48 #include "gc/shenandoah/vm_operations_shenandoah.hpp"
49
50 #include "runtime/vmThread.hpp"
51 #include "services/mallocTracker.hpp"
52
53 SCMUpdateRefsClosure::SCMUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {}
54
55 #ifdef ASSERT
56 template <class T>
57 void AssertToSpaceClosure::do_oop_nv(T* p) {
58 T o = oopDesc::load_heap_oop(p);
59 if (! oopDesc::is_null(o)) {
60 oop obj = oopDesc::decode_heap_oop_not_null(o);
61 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
62 "need to-space object here obj: "PTR_FORMAT" , rb(obj): "PTR_FORMAT", p: "PTR_FORMAT,
63 p2i(obj), p2i(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), p2i(p));
64 }
65 }
66
67 void AssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
68 void AssertToSpaceClosure::do_oop(oop* p) { do_oop_nv(p); }
69 #endif
70
71 const char* ShenandoahHeap::name() const {
72 return "Shenandoah";
73 }
74
75 void ShenandoahHeap::print_heap_locations(HeapWord* start, HeapWord* end) {
76 HeapWord* cur = NULL;
77 for (cur = start; cur < end; cur++) {
78 tty->print_cr(PTR_FORMAT" : "PTR_FORMAT, p2i(cur), p2i(*((HeapWord**) cur)));
79 }
80 }
81
82 class ShenandoahPretouchTask : public AbstractGangTask {
83 private:
84 ShenandoahHeapRegionSet* _regions;
85 const size_t _bitmap_size;
86 const size_t _page_size;
87 char* _bitmap0_base;
88 char* _bitmap1_base;
89 public:
90 ShenandoahPretouchTask(ShenandoahHeapRegionSet* regions,
91 char* bitmap0_base, char* bitmap1_base, size_t bitmap_size,
92 size_t page_size) :
93 AbstractGangTask("Shenandoah PreTouch",
94 Universe::is_fully_initialized() ? GCId::current_raw() :
95 // During VM initialization there is
96 // no GC cycle that this task can be
97 // associated with.
98 GCId::undefined()),
99 _bitmap0_base(bitmap0_base),
100 _bitmap1_base(bitmap1_base),
101 _regions(regions),
102 _bitmap_size(bitmap_size),
103 _page_size(page_size) {
104 _regions->clear_current_index();
105 };
106
107 virtual void work(uint worker_id) {
108 ShenandoahHeapRegion* r = _regions->claim_next();
109 while (r != NULL) {
110 log_trace(gc, heap)("Pretouch region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
111 r->region_number(), p2i(r->bottom()), p2i(r->end()));
112 os::pretouch_memory(r->bottom(), r->end(), _page_size);
113
114 size_t start = r->region_number() * ShenandoahHeapRegion::region_size_bytes() / CMBitMap::heap_map_factor();
115 size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::region_size_bytes() / CMBitMap::heap_map_factor();
116 assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
117
118 log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
119 r->region_number(), p2i(_bitmap0_base + start), p2i(_bitmap0_base + end));
120 os::pretouch_memory(_bitmap0_base + start, _bitmap0_base + end, _page_size);
121
122 log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
123 r->region_number(), p2i(_bitmap1_base + start), p2i(_bitmap1_base + end));
124 os::pretouch_memory(_bitmap1_base + start, _bitmap1_base + end, _page_size);
125
126 r = _regions->claim_next();
127 }
128 }
129 };
130
131 jint ShenandoahHeap::initialize() {
132 CollectedHeap::pre_initialize();
133
134 BrooksPointer::initial_checks();
135
136 size_t init_byte_size = collector_policy()->initial_heap_byte_size();
137 size_t max_byte_size = collector_policy()->max_heap_byte_size();
138 size_t heap_alignment = collector_policy()->heap_alignment();
139
140 Universe::check_alignment(max_byte_size,
141 ShenandoahHeapRegion::region_size_bytes(),
142 "shenandoah heap");
143 Universe::check_alignment(init_byte_size,
144 ShenandoahHeapRegion::region_size_bytes(),
145 "shenandoah heap");
146
147 ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
148 heap_alignment);
149 initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
150
151 set_barrier_set(new ShenandoahBarrierSet(this));
152 ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size);
153 _storage.initialize(pgc_rs, init_byte_size);
154
155 _num_regions = init_byte_size / ShenandoahHeapRegion::region_size_bytes();
156 _max_regions = max_byte_size / ShenandoahHeapRegion::region_size_bytes();
157 _initialSize = _num_regions * ShenandoahHeapRegion::region_size_bytes();
158 size_t regionSizeWords = ShenandoahHeapRegion::region_size_bytes() / HeapWordSize;
159 assert(init_byte_size == _initialSize, "tautology");
160 _ordered_regions = new ShenandoahHeapRegionSet(_max_regions);
161 _collection_set = new ShenandoahCollectionSet(_max_regions);
162 _free_regions = new ShenandoahFreeSet(_max_regions);
163
164 // Initialize fast collection set test structure.
165 _in_cset_fast_test_length = _max_regions;
166 _in_cset_fast_test_base =
167 NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length, mtGC);
168 _in_cset_fast_test = _in_cset_fast_test_base -
169 ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_shift());
170
171 _next_top_at_mark_starts_base =
172 NEW_C_HEAP_ARRAY(HeapWord*, _max_regions, mtGC);
173 _next_top_at_mark_starts = _next_top_at_mark_starts_base -
174 ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_shift());
175
176 _complete_top_at_mark_starts_base =
177 NEW_C_HEAP_ARRAY(HeapWord*, _max_regions, mtGC);
178 _complete_top_at_mark_starts = _complete_top_at_mark_starts_base -
179 ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::region_size_shift());
180
181 size_t i = 0;
182 for (i = 0; i < _num_regions; i++) {
183 _in_cset_fast_test_base[i] = false; // Not in cset
184 HeapWord* bottom = (HeapWord*) pgc_rs.base() + regionSizeWords * i;
185 _complete_top_at_mark_starts_base[i] = bottom;
186 _next_top_at_mark_starts_base[i] = bottom;
187 }
188
189 {
190 ShenandoahHeapLock lock(this);
191 for (i = 0; i < _num_regions; i++) {
192 ShenandoahHeapRegion* current = new ShenandoahHeapRegion(this, (HeapWord*) pgc_rs.base() +
193 regionSizeWords * i, regionSizeWords, i);
194 _free_regions->add_region(current);
195 _ordered_regions->add_region(current);
196 }
197 }
198 assert(((size_t) _ordered_regions->active_regions()) == _num_regions, "");
199 assert((((size_t) base()) &
200 (ShenandoahHeapRegion::region_size_bytes() - 1)) == 0,
201 "misaligned heap: "PTR_FORMAT, p2i(base()));
202
203 if (log_is_enabled(Trace, gc, region)) {
204 ResourceMark rm;
205 outputStream* out = Log(gc, region)::trace_stream();
206 log_trace(gc, region)("All Regions");
207 _ordered_regions->print(out);
208 log_trace(gc, region)("Free Regions");
209 _free_regions->print(out);
210 }
211
212 _recycled_regions = NEW_C_HEAP_ARRAY(size_t, _max_regions, mtGC);
213 _recycled_region_count = 0;
214
215 // The call below uses stuff (the SATB* things) that are in G1, but probably
216 // belong into a shared location.
217 JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
218 SATB_Q_FL_lock,
219 20 /*G1SATBProcessCompletedThreshold */,
220 Shared_SATB_Q_lock);
221
222 // Reserve space for prev and next bitmap.
223 _bitmap_size = CMBitMap::compute_size(heap_rs.size());
224 _heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize);
225
226 size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
227
228 ReservedSpace bitmap0(_bitmap_size, page_size);
229 os::commit_memory_or_exit(bitmap0.base(), bitmap0.size(), false, "couldn't allocate mark bitmap");
230 MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
231 MemRegion bitmap_region0 = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
232
233 ReservedSpace bitmap1(_bitmap_size, page_size);
234 os::commit_memory_or_exit(bitmap1.base(), bitmap1.size(), false, "couldn't allocate mark bitmap");
235 MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
236 MemRegion bitmap_region1 = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
237
238 if (ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix)) {
239 ReservedSpace verify_bitmap(_bitmap_size, page_size);
240 os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), false,
241 "couldn't allocate verification bitmap");
242 MemTracker::record_virtual_memory_type(verify_bitmap.base(), mtGC);
243 MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
244 _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
245 }
246
247 if (ShenandoahAlwaysPreTouch) {
248 assert (!AlwaysPreTouch, "Should have been overridden");
249
250 // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
251 // before initialize() below zeroes it with initializing thread. For any given region,
252 // we touch the region and the corresponding bitmaps from the same thread.
253
254 log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
255 _ordered_regions->count(), page_size);
256 ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), _bitmap_size, page_size);
257 _workers->run_task(&cl);
258 }
259
260 _mark_bit_map0.initialize(_heap_region, bitmap_region0);
261 _complete_mark_bit_map = &_mark_bit_map0;
262
263 _mark_bit_map1.initialize(_heap_region, bitmap_region1);
264 _next_mark_bit_map = &_mark_bit_map1;
265
266 _connection_matrix = UseShenandoahMatrix ?
267 new ShenandoahConnectionMatrix(_max_regions) :
268 NULL;
269
270 _partial_gc = _shenandoah_policy->can_do_partial_gc() ?
271 new ShenandoahPartialGC(this, _max_regions) :
272 NULL;
273
274 _monitoring_support = new ShenandoahMonitoringSupport(this);
275
276 _concurrent_gc_thread = new ShenandoahConcurrentThread();
277
278 ShenandoahMarkCompact::initialize();
279
280 return JNI_OK;
281 }
282
283 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
284 CollectedHeap(),
285 _shenandoah_policy(policy),
286 _concurrent_mark_in_progress(0),
287 _evacuation_in_progress(0),
288 _full_gc_in_progress(false),
289 _update_refs_in_progress(false),
290 _free_regions(NULL),
291 _collection_set(NULL),
292 _bytes_allocated_since_cm(0),
293 _bytes_allocated_during_cm(0),
294 _allocated_last_gc(0),
295 _used_start_gc(0),
296 _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
297 _ref_processor(NULL),
298 _in_cset_fast_test(NULL),
299 _in_cset_fast_test_base(NULL),
300 _next_top_at_mark_starts(NULL),
301 _next_top_at_mark_starts_base(NULL),
302 _complete_top_at_mark_starts(NULL),
303 _complete_top_at_mark_starts_base(NULL),
304 _mark_bit_map0(),
305 _mark_bit_map1(),
306 _connection_matrix(NULL),
307 _cancelled_concgc(false),
308 _need_update_refs(false),
309 _need_reset_bitmaps(false),
310 _heap_lock(0),
311 #ifdef ASSERT
312 _heap_lock_owner(NULL),
313 #endif
314 _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer())
315
316 {
317 log_info(gc, init)("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads);
318 log_info(gc, init)("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads);
319 log_info(gc, init)("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));
320
321 _scm = new ShenandoahConcurrentMark();
322 _used = 0;
323
324 _max_workers = MAX2(_max_workers, 1U);
325 _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
326 /* are_GC_task_threads */true,
327 /* are_ConcurrentGC_threads */false);
328 if (_workers == NULL) {
329 vm_exit_during_initialization("Failed necessary allocation.");
330 } else {
331 _workers->initialize_workers();
332 }
333 }
334
335 class ResetNextBitmapTask : public AbstractGangTask {
336 private:
337 ShenandoahHeapRegionSet* _regions;
338
339 public:
340 ResetNextBitmapTask(ShenandoahHeapRegionSet* regions) :
341 AbstractGangTask("Parallel Reset Bitmap Task"),
342 _regions(regions) {
343 _regions->clear_current_index();
344 }
345
346 void work(uint worker_id) {
347 ShenandoahHeapRegion* region = _regions->claim_next();
348 ShenandoahHeap* heap = ShenandoahHeap::heap();
349 while (region != NULL) {
350 HeapWord* bottom = region->bottom();
351 HeapWord* top = heap->next_top_at_mark_start(region->bottom());
352 if (top > bottom) {
353 heap->next_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
354 }
355 region = _regions->claim_next();
356 }
357 }
358 };
359
360 void ShenandoahHeap::reset_next_mark_bitmap(WorkGang* workers) {
361 ResetNextBitmapTask task = ResetNextBitmapTask(_ordered_regions);
362 workers->run_task(&task);
363 }
364
365 class ResetCompleteBitmapTask : public AbstractGangTask {
366 private:
367 ShenandoahHeapRegionSet* _regions;
368
369 public:
370 ResetCompleteBitmapTask(ShenandoahHeapRegionSet* regions) :
371 AbstractGangTask("Parallel Reset Bitmap Task"),
372 _regions(regions) {
373 _regions->clear_current_index();
374 }
375
376 void work(uint worker_id) {
377 ShenandoahHeapRegion* region = _regions->claim_next();
378 ShenandoahHeap* heap = ShenandoahHeap::heap();
379 while (region != NULL) {
380 HeapWord* bottom = region->bottom();
381 HeapWord* top = heap->complete_top_at_mark_start(region->bottom());
382 if (top > bottom) {
383 heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
384 }
385 region = _regions->claim_next();
386 }
387 }
388 };
389
390 void ShenandoahHeap::reset_complete_mark_bitmap(WorkGang* workers) {
391 ResetCompleteBitmapTask task = ResetCompleteBitmapTask(_ordered_regions);
392 workers->run_task(&task);
393 }
394
395 bool ShenandoahHeap::is_next_bitmap_clear() {
396 HeapWord* start = _ordered_regions->bottom();
397 HeapWord* end = _ordered_regions->end();
398 return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
399 }
400
401 bool ShenandoahHeap::is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end) {
402 return _complete_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
403 }
404
405 void ShenandoahHeap::print_on(outputStream* st) const {
406 st->print("Shenandoah Heap");
407 st->print(" total = " SIZE_FORMAT " K, used " SIZE_FORMAT " K ", capacity()/ K, used() /K);
408 st->print(" [" PTR_FORMAT ", " PTR_FORMAT ") ",
409 p2i(reserved_region().start()),
410 p2i(reserved_region().end()));
411 st->print("Region size = " SIZE_FORMAT "K ", ShenandoahHeapRegion::region_size_bytes() / K);
412 if (_concurrent_mark_in_progress) {
413 st->print("marking ");
414 }
415 if (_evacuation_in_progress) {
416 st->print("evacuating ");
417 }
418 if (cancelled_concgc()) {
419 st->print("cancelled ");
420 }
421 st->print("\n");
422
423 // Adapted from VirtualSpace::print_on(), which is non-PRODUCT only
424 st->print ("Virtual space:");
425 if (_storage.special()) st->print(" (pinned in memory)");
426 st->cr();
427 st->print_cr(" - committed: " SIZE_FORMAT, _storage.committed_size());
428 st->print_cr(" - reserved: " SIZE_FORMAT, _storage.reserved_size());
429 st->print_cr(" - [low, high]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(_storage.low()), p2i(_storage.high()));
430 st->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(_storage.low_boundary()), p2i(_storage.high_boundary()));
431
432 if (Verbose) {
433 print_heap_regions(st);
434 }
435 }
436
437 class InitGCLABClosure : public ThreadClosure {
438 public:
439 void do_thread(Thread* thread) {
440 thread->gclab().initialize(true);
441 }
442 };
443
444 void ShenandoahHeap::post_initialize() {
445 if (UseTLAB) {
446 // This is a very tricky point in VM lifetime. We cannot easily call Threads::threads_do
447 // here, because some system threads (VMThread, WatcherThread, etc) are not yet available.
448 // Their initialization should be handled separately. Is we miss some threads here,
449 // then any other TLAB-related activity would fail with asserts.
450
451 InitGCLABClosure init_gclabs;
452 {
453 MutexLocker ml(Threads_lock);
454 for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
455 init_gclabs.do_thread(thread);
456 }
457 }
458 gc_threads_do(&init_gclabs);
459
460 // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
461 // Now, we will let WorkGang to initialize gclab when new worker is created.
462 _workers->set_initialize_gclab();
463 }
464
465 _scm->initialize(_max_workers);
466
467 ref_processing_init();
468 }
469
470 class CalculateUsedRegionClosure : public ShenandoahHeapRegionClosure {
471 size_t sum;
472 public:
473
474 CalculateUsedRegionClosure() {
475 sum = 0;
476 }
477
478 bool doHeapRegion(ShenandoahHeapRegion* r) {
479 sum = sum + r->used();
480 return false;
481 }
482
483 size_t getResult() { return sum;}
484 };
485
486 size_t ShenandoahHeap::calculateUsed() {
487 CalculateUsedRegionClosure cl;
488 heap_region_iterate(&cl);
489 return cl.getResult();
490 }
491
492 void ShenandoahHeap::verify_heap_size_consistency() {
493
494 assert(calculateUsed() == used(),
495 "heap used size must be consistent heap-used: "SIZE_FORMAT" regions-used: "SIZE_FORMAT, used(), calculateUsed());
496 }
497
498 size_t ShenandoahHeap::used() const {
499 OrderAccess::acquire();
500 return _used;
501 }
502
503 void ShenandoahHeap::increase_used(size_t bytes) {
504 assert_heaplock_or_safepoint();
505 _used += bytes;
506 }
507
508 void ShenandoahHeap::set_used(size_t bytes) {
509 assert_heaplock_or_safepoint();
510 _used = bytes;
511 }
512
513 void ShenandoahHeap::decrease_used(size_t bytes) {
514 assert_heaplock_or_safepoint();
515 assert(_used >= bytes, "never decrease heap size by more than we've left");
516 _used -= bytes;
517 }
518
519 size_t ShenandoahHeap::capacity() const {
520 return _num_regions * ShenandoahHeapRegion::region_size_bytes();
521 }
522
523 bool ShenandoahHeap::is_maximal_no_gc() const {
524 Unimplemented();
525 return true;
526 }
527
528 size_t ShenandoahHeap::max_capacity() const {
529 return _max_regions * ShenandoahHeapRegion::region_size_bytes();
530 }
531
532 size_t ShenandoahHeap::min_capacity() const {
533 return _initialSize;
534 }
535
536 VirtualSpace* ShenandoahHeap::storage() const {
537 return (VirtualSpace*) &_storage;
538 }
539
540 bool ShenandoahHeap::is_in(const void* p) const {
541 HeapWord* heap_base = (HeapWord*) base();
542 HeapWord* last_region_end = heap_base + (ShenandoahHeapRegion::region_size_bytes() / HeapWordSize) * _num_regions;
543 return p >= heap_base && p < last_region_end;
544 }
545
546 bool ShenandoahHeap::is_scavengable(const void* p) {
547 return true;
548 }
549
550 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
551 // Retain tlab and allocate object in shared space if
552 // the amount free in the tlab is too large to discard.
553 if (thread->gclab().free() > thread->gclab().refill_waste_limit()) {
554 thread->gclab().record_slow_allocation(size);
555 return NULL;
556 }
557
558 // Discard gclab and allocate a new one.
559 // To minimize fragmentation, the last GCLAB may be smaller than the rest.
560 size_t new_gclab_size = thread->gclab().compute_size(size);
561
562 thread->gclab().clear_before_allocation();
563
564 if (new_gclab_size == 0) {
565 return NULL;
566 }
567
568 // Allocate a new GCLAB...
569 HeapWord* obj = allocate_new_gclab(new_gclab_size);
570 if (obj == NULL) {
571 return NULL;
572 }
573
574 if (ZeroTLAB) {
575 // ..and clear it.
576 Copy::zero_to_words(obj, new_gclab_size);
577 } else {
578 // ...and zap just allocated object.
579 #ifdef ASSERT
580 // Skip mangling the space corresponding to the object header to
581 // ensure that the returned space is not considered parsable by
582 // any concurrent GC thread.
583 size_t hdr_size = oopDesc::header_size();
584 Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
585 #endif // ASSERT
586 }
587 thread->gclab().fill(obj, obj + size, new_gclab_size);
588 return obj;
589 }
590
591 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
592 #ifdef ASSERT
593 log_debug(gc, alloc)("Allocate new tlab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize);
594 #endif
595 return allocate_new_lab(word_size, _lab_thread);
596 }
597
598 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) {
599 #ifdef ASSERT
600 log_debug(gc, alloc)("Allocate new gclab, requested size = " SIZE_FORMAT " bytes", word_size * HeapWordSize);
601 #endif
602 return allocate_new_lab(word_size, _lab_gc);
603 }
604
605 HeapWord* ShenandoahHeap::allocate_new_lab(size_t word_size, LabType type) {
606
607 HeapWord* result = allocate_memory(word_size, type);
608
609 if (result != NULL) {
610 assert(! in_collection_set(result), "Never allocate in dirty region");
611 _bytes_allocated_since_cm += word_size * HeapWordSize;
612
613 log_develop_trace(gc, tlab)("allocating new tlab of size "SIZE_FORMAT" at addr "PTR_FORMAT, word_size, p2i(result));
614
615 }
616 return result;
617 }
618
619 ShenandoahHeap* ShenandoahHeap::heap() {
620 CollectedHeap* heap = Universe::heap();
621 assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
622 assert(heap->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap");
623 return (ShenandoahHeap*) heap;
624 }
625
626 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
627 CollectedHeap* heap = Universe::heap();
628 return (ShenandoahHeap*) heap;
629 }
630
631 HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size, LabType type) {
632
633 ShenandoahHeapLock heap_lock(this);
634
635 HeapWord* result = allocate_memory_under_lock(word_size, type);
636 size_t grow_by = (word_size * HeapWordSize + ShenandoahHeapRegion::region_size_bytes() - 1) / ShenandoahHeapRegion::region_size_bytes();
637
638 while (result == NULL && _num_regions + grow_by <= _max_regions) {
639 grow_heap_by(grow_by);
640 result = allocate_memory_under_lock(word_size, type);
641 }
642
643 return result;
644 }
645
646 HeapWord* ShenandoahHeap::allocate_memory(size_t word_size, LabType type) {
647 HeapWord* result = NULL;
648 result = allocate_memory_work(word_size, type);
649
650 if (type == _lab_thread) {
651 // Allocation failed, try full-GC, then retry allocation.
652 //
653 // It might happen that one of the threads requesting allocation would unblock
654 // way later after full-GC happened, only to fail the second allocation, because
655 // other threads have already depleted the free storage. In this case, a better
656 // strategy would be to try full-GC again.
657 //
658 // Lacking the way to detect progress from "collect" call, we are left with blindly
659 // retrying for some bounded number of times.
660 // TODO: Poll if Full GC made enough progress to warrant retry.
661 int tries = 0;
662 while ((result == NULL) && (tries++ < ShenandoahFullGCTries)) {
663 log_debug(gc)("[" PTR_FORMAT " Failed to allocate " SIZE_FORMAT " bytes, doing full GC, try %d",
664 p2i(Thread::current()), word_size * HeapWordSize, tries);
665 collect(GCCause::_allocation_failure);
666 result = allocate_memory_work(word_size, type);
667 }
668 }
669
670 // Only update monitoring counters when not calling from a write-barrier.
671 // Otherwise we might attempt to grab the Service_lock, which we must
672 // not do when coming from a write-barrier (because the thread might
673 // already hold the Compile_lock).
674 if (type == _lab_thread) {
675 monitoring_support()->update_counters();
676 }
677
678 log_develop_trace(gc, alloc)("allocate memory chunk of size "SIZE_FORMAT" at addr "PTR_FORMAT " by thread %d ",
679 word_size, p2i(result), Thread::current()->osthread()->thread_id());
680
681 return result;
682 }
683
684 HeapWord* ShenandoahHeap::allocate_memory_under_lock(size_t word_size, LabType type) {
685 assert_heaplock_owned_by_current_thread();
686
687 if (word_size * HeapWordSize > ShenandoahHeapRegion::region_size_bytes()) {
688 return allocate_large_memory(word_size);
689 }
690
691 // Not enough memory in free region set.
692 // Coming out of full GC, it is possible that there is not
693 // free region available, so current_index may not be valid.
694 if (word_size * HeapWordSize > _free_regions->capacity()) return NULL;
695
696 ShenandoahHeapRegion* my_current_region = _free_regions->current_no_humongous();
697
698 if (my_current_region == NULL) {
699 return NULL; // No more room to make a new region. OOM.
700 }
701 assert(my_current_region != NULL, "should have a region at this point");
702 assert(! in_collection_set(my_current_region), "never get targetted regions in free-lists");
703 assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
704
705 HeapWord* result = my_current_region->allocate_lab(word_size, type);
706
707 while (result == NULL) {
708 // 2nd attempt. Try next region.
709 #ifdef ASSERT
710 if (my_current_region->free() > 0) {
711 log_debug(gc, alloc)("Retire region with " SIZE_FORMAT " bytes free", my_current_region->free());
712 }
713 #endif
714 _free_regions->increase_used(my_current_region->free());
715 ShenandoahHeapRegion* next_region = _free_regions->next_no_humongous();
716 assert(next_region != my_current_region, "must not get current again");
717 my_current_region = next_region;
718
719 if (my_current_region == NULL) {
720 return NULL; // No more room to make a new region. OOM.
721 }
722 assert(my_current_region != NULL, "should have a region at this point");
723 assert(! in_collection_set(my_current_region), "never get targetted regions in free-lists");
724 assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
725 result = my_current_region->allocate_lab(word_size, type);
726 }
727
728 my_current_region->increase_live_data_words(word_size);
729 increase_used(word_size * HeapWordSize);
730 _free_regions->increase_used(word_size * HeapWordSize);
731 return result;
732 }
733
734 HeapWord* ShenandoahHeap::allocate_large_memory(size_t words) {
735 assert_heaplock_owned_by_current_thread();
736
737 size_t required_regions = ShenandoahHumongous::required_regions(words * HeapWordSize);
738 if (required_regions > _max_regions) return NULL;
739
740 ShenandoahHeapRegion* r = _free_regions->allocate_contiguous(required_regions);
741
742 HeapWord* result = NULL;
743
744 if (r != NULL) {
745 result = r->bottom();
746
747 log_debug(gc, humongous)("allocating humongous object of size: "SIZE_FORMAT" KB at location "PTR_FORMAT" in start region "SIZE_FORMAT,
748 (words * HeapWordSize) / K, p2i(result), r->region_number());
749 } else {
750 log_debug(gc, humongous)("allocating humongous object of size: "SIZE_FORMAT" KB at location "PTR_FORMAT" failed",
751 (words * HeapWordSize) / K, p2i(result));
752 }
753
754
755 return result;
756
757 }
758
759 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
760 bool* gc_overhead_limit_was_exceeded) {
761
762 HeapWord* filler = allocate_memory(size + BrooksPointer::word_size(), _lab_thread);
763 HeapWord* result = filler + BrooksPointer::word_size();
764 if (filler != NULL) {
765 BrooksPointer::initialize(oop(result));
766 _bytes_allocated_since_cm += size * HeapWordSize;
767
768 assert(! in_collection_set(result), "never allocate in targetted region");
769 return result;
770 } else {
771 /*
772 tty->print_cr("Out of memory. Requested number of words: "SIZE_FORMAT" used heap: "INT64_FORMAT", bytes allocated since last CM: "INT64_FORMAT,
773 size, used(), _bytes_allocated_since_cm);
774 {
775 print_heap_regions();
776 tty->print("Printing "SIZE_FORMAT" free regions:\n", _free_regions->count());
777 _free_regions->print();
778 }
779 */
780 return NULL;
781 }
782 }
783
784 class ParallelEvacuateRegionObjectClosure : public ObjectClosure {
785 private:
786 ShenandoahHeap* _heap;
787 Thread* _thread;
788 public:
789 ParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
790 _heap(heap), _thread(Thread::current()) {
791 }
792
793 void do_object(oop p) {
794
795 log_develop_trace(gc, compaction)("Calling ParallelEvacuateRegionObjectClosure on "PTR_FORMAT" of size %d\n", p2i((HeapWord*) p), p->size());
796
797 assert(_heap->is_marked_complete(p), "expect only marked objects");
798 if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p))) {
799 bool evac;
800 _heap->evacuate_object(p, _thread, evac);
801 }
802 }
803 };
804
805 #ifdef ASSERT
806 class VerifyEvacuatedObjectClosure : public ObjectClosure {
807
808 public:
809
810 void do_object(oop p) {
811 if (ShenandoahHeap::heap()->is_marked_complete(p)) {
812 oop p_prime = oopDesc::bs()->read_barrier(p);
813 assert(! oopDesc::unsafe_equals(p, p_prime), "Should point to evacuated copy");
814 if (p->klass() != p_prime->klass()) {
815 tty->print_cr("copy has different class than original:");
816 p->klass()->print_on(tty);
817 p_prime->klass()->print_on(tty);
818 }
819 assert(p->klass() == p_prime->klass(), "Should have the same class p: "PTR_FORMAT", p_prime: "PTR_FORMAT, p2i(p), p2i(p_prime));
820 // assert(p->mark() == p_prime->mark(), "Should have the same mark");
821 assert(p->size() == p_prime->size(), "Should be the same size");
822 assert(oopDesc::unsafe_equals(p_prime, oopDesc::bs()->read_barrier(p_prime)), "One forward once");
823 }
824 }
825 };
826
827 void ShenandoahHeap::verify_evacuated_region(ShenandoahHeapRegion* from_region) {
828 VerifyEvacuatedObjectClosure verify_evacuation;
829 marked_object_iterate(from_region, &verify_evacuation);
830 }
831 #endif
832
833 void ShenandoahHeap::parallel_evacuate_region(ShenandoahHeapRegion* from_region) {
834
835 assert(from_region->has_live(), "all-garbage regions are reclaimed earlier");
836
837 ParallelEvacuateRegionObjectClosure evacuate_region(this);
838
839 marked_object_iterate(from_region, &evacuate_region);
840
841 #ifdef ASSERT
842 if (ShenandoahVerify && ! cancelled_concgc()) {
843 verify_evacuated_region(from_region);
844 }
845 #endif
846 }
847
848 class ParallelEvacuationTask : public AbstractGangTask {
849 private:
850 ShenandoahHeap* _sh;
851 ShenandoahCollectionSet* _cs;
852
853 public:
854 ParallelEvacuationTask(ShenandoahHeap* sh,
855 ShenandoahCollectionSet* cs) :
856 AbstractGangTask("Parallel Evacuation Task"),
857 _cs(cs),
858 _sh(sh) {}
859
860 void work(uint worker_id) {
861
862 ShenandoahHeapRegion* from_hr = _cs->claim_next();
863
864 while (from_hr != NULL) {
865 log_develop_trace(gc, region)("Thread "INT32_FORMAT" claimed Heap Region "SIZE_FORMAT,
866 worker_id,
867 from_hr->region_number());
868
869 assert(from_hr->has_live(), "all-garbage regions are reclaimed early");
870 _sh->parallel_evacuate_region(from_hr);
871
872 if (_sh->cancelled_concgc()) {
873 log_develop_trace(gc, region)("Cancelled concgc while evacuating region " SIZE_FORMAT "\n", from_hr->region_number());
874 break;
875 }
876 from_hr = _cs->claim_next();
877 }
878 }
879 };
880
881 void ShenandoahHeap::recycle_dirty_regions() {
882 ShenandoahHeapLock lock(this);
883
884 size_t bytes_reclaimed = 0;
885
886 ShenandoahHeapRegionSet* set = regions();
887 set->clear_current_index();
888
889 start_deferred_recycling();
890
891 ShenandoahHeapRegion* r = set->claim_next();
892 while (r != NULL) {
893 if (in_collection_set(r)) {
894 decrease_used(r->used());
895 bytes_reclaimed += r->used();
896 defer_recycle(r);
897 }
898 r = set->claim_next();
899 }
900
901 finish_deferred_recycle();
902
903 _shenandoah_policy->record_bytes_reclaimed(bytes_reclaimed);
904 if (! cancelled_concgc()) {
905 clear_cset_fast_test();
906 }
907 }
908
909 ShenandoahFreeSet* ShenandoahHeap::free_regions() {
910 return _free_regions;
911 }
912
913 void ShenandoahHeap::print_heap_regions(outputStream* st) const {
914 _ordered_regions->print(st);
915 }
916
917 class PrintAllRefsOopClosure: public ExtendedOopClosure {
918 private:
919 int _index;
920 const char* _prefix;
921
922 public:
923 PrintAllRefsOopClosure(const char* prefix) : _index(0), _prefix(prefix) {}
924
925 private:
926 template <class T>
927 inline void do_oop_work(T* p) {
928 oop o = oopDesc::load_decode_heap_oop(p);
929 if (o != NULL) {
930 if (ShenandoahHeap::heap()->is_in(o) && o->is_oop()) {
931 tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT")-> "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT")",
932 _prefix, _index,
933 p2i(p), p2i(o),
934 BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_complete(o)),
935 o->klass()->internal_name(), p2i(o->klass()));
936 } else {
937 tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty)",
938 _prefix, _index,
939 p2i(p), p2i(o));
940 }
941 } else {
942 tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT") -> "PTR_FORMAT, _prefix, _index, p2i(p), p2i((HeapWord*) o));
943 }
944 _index++;
945 }
946
947 public:
948 void do_oop(oop* p) {
949 do_oop_work(p);
950 }
951
952 void do_oop(narrowOop* p) {
953 do_oop_work(p);
954 }
955
956 };
957
958 class PrintAllRefsObjectClosure : public ObjectClosure {
959 const char* _prefix;
960
961 public:
962 PrintAllRefsObjectClosure(const char* prefix) : _prefix(prefix) {}
963
964 void do_object(oop p) {
965 if (ShenandoahHeap::heap()->is_in(p)) {
966 tty->print_cr("%s object "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT") refers to:",
967 _prefix, p2i(p),
968 BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_complete(p)),
969 p->klass()->internal_name(), p2i(p->klass()));
970 PrintAllRefsOopClosure cl(_prefix);
971 p->oop_iterate(&cl);
972 }
973 }
974 };
975
976 void ShenandoahHeap::print_all_refs(const char* prefix) {
977 tty->print_cr("printing all references in the heap");
978 tty->print_cr("root references:");
979
980 ensure_parsability(false);
981
982 PrintAllRefsOopClosure cl(prefix);
983 roots_iterate(&cl);
984
985 tty->print_cr("heap references:");
986 PrintAllRefsObjectClosure cl2(prefix);
987 object_iterate(&cl2);
988 }
989
990 class VerifyAfterMarkingOopClosure: public ExtendedOopClosure {
991 private:
992 ShenandoahHeap* _heap;
993
994 public:
995 VerifyAfterMarkingOopClosure() :
996 _heap(ShenandoahHeap::heap()) { }
997
998 private:
999 template <class T>
1000 inline void do_oop_work(T* p) {
1001 oop o = oopDesc::load_decode_heap_oop(p);
1002 if (o != NULL) {
1003 if (! _heap->is_marked_complete(o)) {
1004 _heap->print_heap_regions();
1005 _heap->print_all_refs("post-mark");
1006 tty->print_cr("oop not marked, although referrer is marked: "PTR_FORMAT": in_heap: %s, is_marked: %s",
1007 p2i((HeapWord*) o), BOOL_TO_STR(_heap->is_in(o)), BOOL_TO_STR(_heap->is_marked_complete(o)));
1008 _heap->print_heap_locations((HeapWord*) o, (HeapWord*) o + o->size());
1009
1010 tty->print_cr("oop class: %s", o->klass()->internal_name());
1011 if (_heap->is_in(p)) {
1012 oop referrer = oop(_heap->heap_region_containing(p)->block_start_const(p));
1013 tty->print_cr("Referrer starts at addr "PTR_FORMAT, p2i((HeapWord*) referrer));
1014 referrer->print();
1015 _heap->print_heap_locations((HeapWord*) referrer, (HeapWord*) referrer + referrer->size());
1016 }
1017 tty->print_cr("heap region containing object:");
1018 _heap->heap_region_containing(o)->print();
1019 tty->print_cr("heap region containing referrer:");
1020 _heap->heap_region_containing(p)->print();
1021 tty->print_cr("heap region containing forwardee:");
1022 _heap->heap_region_containing(oopDesc::bs()->read_barrier(o))->print();
1023 }
1024 assert(o->is_oop(), "oop must be an oop");
1025 assert(Metaspace::contains(o->klass()), "klass pointer must go to metaspace");
1026 if (! oopDesc::unsafe_equals(o, oopDesc::bs()->read_barrier(o))) {
1027 tty->print_cr("oops has forwardee: p: "PTR_FORMAT" (%s), o = "PTR_FORMAT" (%s), new-o: "PTR_FORMAT" (%s)",
1028 p2i(p),
1029 BOOL_TO_STR(_heap->in_collection_set(p)),
1030 p2i(o),
1031 BOOL_TO_STR(_heap->in_collection_set(o)),
1032 p2i((HeapWord*) oopDesc::bs()->read_barrier(o)),
1033 BOOL_TO_STR(_heap->in_collection_set(oopDesc::bs()->read_barrier(o))));
1034 tty->print_cr("oop class: %s", o->klass()->internal_name());
1035 }
1036 assert(oopDesc::unsafe_equals(o, oopDesc::bs()->read_barrier(o)), "oops must not be forwarded");
1037 assert(! _heap->in_collection_set(o), "references must not point to dirty heap regions");
1038 assert(_heap->is_marked_complete(o), "live oops must be marked current");
1039 }
1040 }
1041
1042 public:
1043 void do_oop(oop* p) {
1044 do_oop_work(p);
1045 }
1046
1047 void do_oop(narrowOop* p) {
1048 do_oop_work(p);
1049 }
1050
1051 };
1052
1053 void ShenandoahHeap::verify_heap_after_marking() {
1054
1055 verify_heap_size_consistency();
1056
1057 log_trace(gc)("verifying heap after marking");
1058
1059 VerifyAfterMarkingOopClosure cl;
1060 roots_iterate(&cl);
1061 ObjectToOopClosure objs(&cl);
1062 object_iterate(&objs);
1063 }
1064
1065
1066 void ShenandoahHeap::reclaim_humongous_region_at(ShenandoahHeapRegion* r) {
1067 assert(r->is_humongous_start(), "reclaim regions starting with the first one");
1068
1069 oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
1070 size_t size = humongous_obj->size() + BrooksPointer::word_size();
1071 size_t required_regions = ShenandoahHumongous::required_regions(size * HeapWordSize);
1072 size_t index = r->region_number();
1073
1074
1075 assert(!r->has_live(), "liveness must be zero");
1076
1077 for(size_t i = 0; i < required_regions; i++) {
1078
1079 ShenandoahHeapRegion* region = _ordered_regions->get(index++);
1080
1081 assert((region->is_humongous_start() || region->is_humongous_continuation()),
1082 "expect correct humongous start or continuation");
1083
1084 if (log_is_enabled(Debug, gc, humongous)) {
1085 log_debug(gc, humongous)("reclaiming "SIZE_FORMAT" humongous regions for object of size: "SIZE_FORMAT" words", required_regions, size);
1086 ResourceMark rm;
1087 outputStream* out = Log(gc, humongous)::debug_stream();
1088 region->print_on(out);
1089 }
1090
1091 region->recycle();
1092 ShenandoahHeap::heap()->decrease_used(ShenandoahHeapRegion::region_size_bytes());
1093 }
1094 }
1095
1096 class ShenandoahReclaimHumongousRegionsClosure : public ShenandoahHeapRegionClosure {
1097
1098 bool doHeapRegion(ShenandoahHeapRegion* r) {
1099 ShenandoahHeap* heap = ShenandoahHeap::heap();
1100
1101 if (r->is_humongous_start()) {
1102 oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
1103 if (! heap->is_marked_complete(humongous_obj)) {
1104
1105 heap->reclaim_humongous_region_at(r);
1106 }
1107 }
1108 return false;
1109 }
1110 };
1111
1112 #ifdef ASSERT
1113 class CheckCollectionSetClosure: public ShenandoahHeapRegionClosure {
1114 bool doHeapRegion(ShenandoahHeapRegion* r) {
1115 assert(! ShenandoahHeap::heap()->in_collection_set(r), "Should have been cleared by now");
1116 return false;
1117 }
1118 };
1119 #endif
1120
1121 void ShenandoahHeap::prepare_for_concurrent_evacuation() {
1122 assert(_ordered_regions->get(0)->region_number() == 0, "FIXME CHF. FIXME CHF!");
1123
1124 log_develop_trace(gc)("Thread %d started prepare_for_concurrent_evacuation", Thread::current()->osthread()->thread_id());
1125
1126 if (!cancelled_concgc()) {
1127 // Allocations might have happened before we STWed here, record peak:
1128 shenandoahPolicy()->record_peak_occupancy();
1129
1130 recycle_dirty_regions();
1131
1132 ensure_parsability(true);
1133
1134 if (UseShenandoahMatrix && PrintShenandoahMatrix) {
1135 outputStream* log = Log(gc)::info_stream();
1136 connection_matrix()->print_on(log);
1137 }
1138
1139 if (ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix)) {
1140 verify_heap_reachable_at_safepoint();
1141 }
1142
1143 #ifdef ASSERT
1144 if (ShenandoahVerify) {
1145 verify_heap_after_marking();
1146 }
1147 #endif
1148
1149 // NOTE: This needs to be done during a stop the world pause, because
1150 // putting regions into the collection set concurrently with Java threads
1151 // will create a race. In particular, acmp could fail because when we
1152 // resolve the first operand, the containing region might not yet be in
1153 // the collection set, and thus return the original oop. When the 2nd
1154 // operand gets resolved, the region could be in the collection set
1155 // and the oop gets evacuated. If both operands have originally been
1156 // the same, we get false negatives.
1157
1158 {
1159 ShenandoahHeapLock lock(this);
1160 _collection_set->clear();
1161 _free_regions->clear();
1162
1163 ShenandoahReclaimHumongousRegionsClosure reclaim;
1164 heap_region_iterate(&reclaim);
1165
1166 #ifdef ASSERT
1167 CheckCollectionSetClosure ccsc;
1168 _ordered_regions->heap_region_iterate(&ccsc);
1169 #endif
1170
1171 _shenandoah_policy->choose_collection_set(_collection_set);
1172
1173 _shenandoah_policy->choose_free_set(_free_regions);
1174 }
1175
1176 _bytes_allocated_since_cm = 0;
1177
1178 Universe::update_heap_info_at_gc();
1179 }
1180 }
1181
1182
1183 class RetireTLABClosure : public ThreadClosure {
1184 private:
1185 bool _retire;
1186
1187 public:
1188 RetireTLABClosure(bool retire) : _retire(retire) {
1189 }
1190
1191 void do_thread(Thread* thread) {
1192 thread->gclab().make_parsable(_retire);
1193 }
1194 };
1195
1196 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1197 if (UseTLAB) {
1198 CollectedHeap::ensure_parsability(retire_tlabs);
1199 RetireTLABClosure cl(retire_tlabs);
1200 Threads::threads_do(&cl);
1201 }
1202 }
1203
1204 class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure {
1205 private:
1206 ShenandoahHeap* _heap;
1207 Thread* _thread;
1208 public:
1209 ShenandoahEvacuateUpdateRootsClosure() :
1210 _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
1211 }
1212
1213 private:
1214 template <class T>
1215 void do_oop_work(T* p) {
1216 assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
1217
1218 T o = oopDesc::load_heap_oop(p);
1219 if (! oopDesc::is_null(o)) {
1220 oop obj = oopDesc::decode_heap_oop_not_null(o);
1221 if (_heap->in_collection_set(obj)) {
1222 assert(_heap->is_marked_complete(obj), "only evacuate marked objects %d %d",
1223 _heap->is_marked_complete(obj), _heap->is_marked_complete(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)));
1224 oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
1225 if (oopDesc::unsafe_equals(resolved, obj)) {
1226 bool evac;
1227 resolved = _heap->evacuate_object(obj, _thread, evac);
1228 }
1229 oopDesc::encode_store_heap_oop(p, resolved);
1230 }
1231 }
1232 }
1233
1234 public:
1235 void do_oop(oop* p) {
1236 do_oop_work(p);
1237 }
1238 void do_oop(narrowOop* p) {
1239 do_oop_work(p);
1240 }
1241 };
1242
1243 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1244 ShenandoahRootEvacuator* _rp;
1245 public:
1246
1247 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1248 AbstractGangTask("Shenandoah evacuate and update roots"),
1249 _rp(rp)
1250 {
1251 // Nothing else to do.
1252 }
1253
1254 void work(uint worker_id) {
1255 ShenandoahEvacuateUpdateRootsClosure cl;
1256 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1257
1258 _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1259 }
1260 };
1261
1262 class ShenandoahFixRootsTask : public AbstractGangTask {
1263 ShenandoahRootEvacuator* _rp;
1264 public:
1265
1266 ShenandoahFixRootsTask(ShenandoahRootEvacuator* rp) :
1267 AbstractGangTask("Shenandoah update roots"),
1268 _rp(rp)
1269 {
1270 // Nothing else to do.
1271 }
1272
1273 void work(uint worker_id) {
1274 SCMUpdateRefsClosure cl;
1275 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1276
1277 _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1278 }
1279 };
1280 void ShenandoahHeap::evacuate_and_update_roots() {
1281
1282 COMPILER2_PRESENT(DerivedPointerTable::clear());
1283
1284 assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
1285
1286 {
1287 ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahCollectorPolicy::init_evac);
1288 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1289 workers()->run_task(&roots_task);
1290 }
1291
1292 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
1293
1294 if (cancelled_concgc()) {
1295 // If initial evacuation has been cancelled, we need to update all references
1296 // after all workers have finished. Otherwise we might run into the following problem:
1297 // GC thread 1 cannot allocate anymore, thus evacuation fails, leaves from-space ptr of object X.
1298 // GC thread 2 evacuates the same object X to to-space
1299 // which leaves a truly dangling from-space reference in the first root oop*. This must not happen.
1300 // clear() and update_pointers() must always be called in pairs,
1301 // cannot nest with above clear()/update_pointers().
1302 COMPILER2_PRESENT(DerivedPointerTable::clear());
1303 ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahCollectorPolicy::init_evac);
1304 ShenandoahFixRootsTask update_roots_task(&rp);
1305 workers()->run_task(&update_roots_task);
1306 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
1307 }
1308
1309 #ifdef ASSERT
1310 {
1311 AssertToSpaceClosure cl;
1312 CodeBlobToOopClosure code_cl(&cl, !CodeBlobToOopClosure::FixRelocations);
1313 ShenandoahRootEvacuator rp(this, 1);
1314 rp.process_evacuate_roots(&cl, &code_cl, 0);
1315 }
1316 #endif
1317 }
1318
1319
1320 void ShenandoahHeap::do_evacuation() {
1321
1322 parallel_evacuate();
1323
1324 if (ShenandoahVerify && ! cancelled_concgc()) {
1325 VM_ShenandoahVerifyHeapAfterEvacuation verify_after_evacuation;
1326 if (Thread::current()->is_VM_thread()) {
1327 verify_after_evacuation.doit();
1328 } else {
1329 VMThread::execute(&verify_after_evacuation);
1330 }
1331 }
1332
1333 }
1334
1335 void ShenandoahHeap::parallel_evacuate() {
1336 log_develop_trace(gc)("starting parallel_evacuate");
1337
1338 _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_evac);
1339
1340 if (log_is_enabled(Trace, gc, region)) {
1341 ResourceMark rm;
1342 outputStream *out = Log(gc, region)::trace_stream();
1343 out->print("Printing all available regions");
1344 print_heap_regions(out);
1345 }
1346
1347 if (log_is_enabled(Trace, gc, cset)) {
1348 ResourceMark rm;
1349 outputStream *out = Log(gc, cset)::trace_stream();
1350 out->print("Printing collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->count());
1351 _collection_set->print(out);
1352
1353 out->print("Printing free set which contains "SIZE_FORMAT" regions:\n", _free_regions->count());
1354 _free_regions->print(out);
1355 }
1356
1357 ParallelEvacuationTask evacuationTask = ParallelEvacuationTask(this, _collection_set);
1358
1359
1360 workers()->run_task(&evacuationTask);
1361
1362 if (log_is_enabled(Trace, gc, cset)) {
1363 ResourceMark rm;
1364 outputStream *out = Log(gc, cset)::trace_stream();
1365 out->print("Printing postgc collection set which contains "SIZE_FORMAT" regions:\n",
1366 _collection_set->count());
1367
1368 _collection_set->print(out);
1369
1370 out->print("Printing postgc free regions which contain "SIZE_FORMAT" free regions:\n",
1371 _free_regions->count());
1372 _free_regions->print(out);
1373
1374 }
1375
1376 if (log_is_enabled(Trace, gc, region)) {
1377 ResourceMark rm;
1378 outputStream *out = Log(gc, region)::trace_stream();
1379 out->print_cr("all regions after evacuation:");
1380 print_heap_regions(out);
1381 }
1382
1383 _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_evac);
1384 }
1385
1386 class VerifyEvacuationClosure: public ExtendedOopClosure {
1387 private:
1388 ShenandoahHeap* _heap;
1389 ShenandoahHeapRegion* _from_region;
1390
1391 public:
1392 VerifyEvacuationClosure(ShenandoahHeapRegion* from_region) :
1393 _heap(ShenandoahHeap::heap()), _from_region(from_region) { }
1394 private:
1395 template <class T>
1396 inline void do_oop_work(T* p) {
1397 oop heap_oop = oopDesc::load_decode_heap_oop(p);
1398 if (! oopDesc::is_null(heap_oop)) {
1399 guarantee(! _from_region->is_in(heap_oop), "no references to from-region allowed after evacuation: "PTR_FORMAT, p2i((HeapWord*) heap_oop));
1400 }
1401 }
1402
1403 public:
1404 void do_oop(oop* p) {
1405 do_oop_work(p);
1406 }
1407
1408 void do_oop(narrowOop* p) {
1409 do_oop_work(p);
1410 }
1411
1412 };
1413
1414 void ShenandoahHeap::roots_iterate(OopClosure* cl) {
1415
1416 assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
1417
1418 CodeBlobToOopClosure blobsCl(cl, false);
1419 CLDToOopClosure cldCl(cl);
1420
1421 ShenandoahRootProcessor rp(this, 1);
1422 rp.process_all_roots(cl, NULL, &cldCl, &blobsCl, 0);
1423 }
1424
1425 void ShenandoahHeap::verify_evacuation(ShenandoahHeapRegion* from_region) {
1426
1427 VerifyEvacuationClosure rootsCl(from_region);
1428 roots_iterate(&rootsCl);
1429
1430 }
1431
1432 bool ShenandoahHeap::supports_tlab_allocation() const {
1433 return true;
1434 }
1435
1436
1437 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1438 size_t idx = _free_regions->current_index();
1439 ShenandoahHeapRegion* current = _free_regions->get_or_null(idx);
1440 if (current == NULL) {
1441 return 0;
1442 } else if (current->free() >= MinTLABSize) {
1443 // Current region has enough space left, can use it.
1444 return current->free();
1445 } else {
1446 // No more space in current region, peek next region
1447 return _free_regions->unsafe_peek_next_no_humongous();
1448 }
1449 }
1450
1451 size_t ShenandoahHeap::max_tlab_size() const {
1452 return ShenandoahHeapRegion::region_size_bytes();
1453 }
1454
1455 class ResizeGCLABClosure : public ThreadClosure {
1456 public:
1457 void do_thread(Thread* thread) {
1458 thread->gclab().resize();
1459 }
1460 };
1461
1462 void ShenandoahHeap::resize_all_tlabs() {
1463 CollectedHeap::resize_all_tlabs();
1464
1465 ResizeGCLABClosure cl;
1466 Threads::threads_do(&cl);
1467 }
1468
1469 class AccumulateStatisticsGCLABClosure : public ThreadClosure {
1470 public:
1471 void do_thread(Thread* thread) {
1472 thread->gclab().accumulate_statistics();
1473 thread->gclab().initialize_statistics();
1474 }
1475 };
1476
1477 void ShenandoahHeap::accumulate_statistics_all_gclabs() {
1478 AccumulateStatisticsGCLABClosure cl;
1479 Threads::threads_do(&cl);
1480 }
1481
1482 bool ShenandoahHeap::can_elide_tlab_store_barriers() const {
1483 return true;
1484 }
1485
1486 oop ShenandoahHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) {
1487 // Overridden to do nothing.
1488 return new_obj;
1489 }
1490
1491 bool ShenandoahHeap::can_elide_initializing_store_barrier(oop new_obj) {
1492 return true;
1493 }
1494
1495 bool ShenandoahHeap::card_mark_must_follow_store() const {
1496 return false;
1497 }
1498
1499 void ShenandoahHeap::collect(GCCause::Cause cause) {
1500 assert(cause != GCCause::_gc_locker, "no JNI critical callback");
1501 if (GCCause::is_user_requested_gc(cause)) {
1502 if (! DisableExplicitGC) {
1503 _concurrent_gc_thread->do_full_gc(cause);
1504 }
1505 } else if (cause == GCCause::_allocation_failure) {
1506 collector_policy()->set_should_clear_all_soft_refs(true);
1507 _concurrent_gc_thread->do_full_gc(cause);
1508 }
1509 }
1510
1511 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1512 //assert(false, "Shouldn't need to do full collections");
1513 }
1514
1515 AdaptiveSizePolicy* ShenandoahHeap::size_policy() {
1516 Unimplemented();
1517 return NULL;
1518
1519 }
1520
1521 CollectorPolicy* ShenandoahHeap::collector_policy() const {
1522 return _shenandoah_policy;
1523 }
1524
1525
1526 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1527 Space* sp = heap_region_containing(addr);
1528 if (sp != NULL) {
1529 return sp->block_start(addr);
1530 }
1531 return NULL;
1532 }
1533
1534 size_t ShenandoahHeap::block_size(const HeapWord* addr) const {
1535 Space* sp = heap_region_containing(addr);
1536 assert(sp != NULL, "block_size of address outside of heap");
1537 return sp->block_size(addr);
1538 }
1539
1540 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1541 Space* sp = heap_region_containing(addr);
1542 return sp->block_is_obj(addr);
1543 }
1544
1545 jlong ShenandoahHeap::millis_since_last_gc() {
1546 return 0;
1547 }
1548
1549 void ShenandoahHeap::prepare_for_verify() {
1550 if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1551 ensure_parsability(false);
1552 }
1553 }
1554
1555 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1556 workers()->print_worker_threads_on(st);
1557 }
1558
1559 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1560 workers()->threads_do(tcl);
1561 }
1562
1563 void ShenandoahHeap::print_tracing_info() const {
1564 if (log_is_enabled(Info, gc, stats)) {
1565 ResourceMark rm;
1566 outputStream* out = Log(gc, stats)::info_stream();
1567 _shenandoah_policy->print_tracing_info(out);
1568 }
1569 }
1570
1571 class ShenandoahVerifyRootsClosure: public ExtendedOopClosure {
1572 private:
1573 ShenandoahHeap* _heap;
1574 VerifyOption _vo;
1575 bool _failures;
1576 public:
1577 // _vo == UsePrevMarking -> use "prev" marking information,
1578 // _vo == UseNextMarking -> use "next" marking information,
1579 // _vo == UseMarkWord -> use mark word from object header.
1580 ShenandoahVerifyRootsClosure(VerifyOption vo) :
1581 _heap(ShenandoahHeap::heap()),
1582 _vo(vo),
1583 _failures(false) { }
1584
1585 bool failures() { return _failures; }
1586
1587 private:
1588 template <class T>
1589 inline void do_oop_work(T* p) {
1590 oop obj = oopDesc::load_decode_heap_oop(p);
1591 if (! oopDesc::is_null(obj) && ! obj->is_oop()) {
1592 { // Just for debugging.
1593 tty->print_cr("Root location "PTR_FORMAT
1594 "verified "PTR_FORMAT, p2i(p), p2i((void*) obj));
1595 // obj->print_on(tty);
1596 }
1597 }
1598 guarantee(obj->is_oop_or_null(), "is oop or null");
1599 }
1600
1601 public:
1602 void do_oop(oop* p) {
1603 do_oop_work(p);
1604 }
1605
1606 void do_oop(narrowOop* p) {
1607 do_oop_work(p);
1608 }
1609
1610 };
1611
1612 class ShenandoahVerifyHeapClosure: public ObjectClosure {
1613 private:
1614 ShenandoahVerifyRootsClosure _rootsCl;
1615 public:
1616 ShenandoahVerifyHeapClosure(ShenandoahVerifyRootsClosure rc) :
1617 _rootsCl(rc) {};
1618
1619 void do_object(oop p) {
1620 _rootsCl.do_oop(&p);
1621 }
1622 };
1623
1624 void ShenandoahHeap::verify(VerifyOption vo) {
1625 if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1626
1627 ShenandoahVerifyRootsClosure rootsCl(vo);
1628
1629 assert(Thread::current()->is_VM_thread(),
1630 "Expected to be executed serially by the VM thread at this point");
1631
1632 roots_iterate(&rootsCl);
1633
1634 bool failures = rootsCl.failures();
1635 log_trace(gc)("verify failures: %s", BOOL_TO_STR(failures));
1636
1637 ShenandoahVerifyHeapClosure heapCl(rootsCl);
1638
1639 object_iterate(&heapCl);
1640 // TODO: Implement rest of it.
1641 } else {
1642 tty->print("(SKIPPING roots, heapRegions, remset) ");
1643 }
1644 }
1645 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1646 return _free_regions->capacity();
1647 }
1648
1649 class ShenandoahIterateObjectClosureRegionClosure: public ShenandoahHeapRegionClosure {
1650 ObjectClosure* _cl;
1651 public:
1652 ShenandoahIterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
1653 bool doHeapRegion(ShenandoahHeapRegion* r) {
1654 ShenandoahHeap::heap()->marked_object_iterate(r, _cl);
1655 return false;
1656 }
1657 };
1658
1659 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1660 ShenandoahIterateObjectClosureRegionClosure blk(cl);
1661 heap_region_iterate(&blk, false, true);
1662 }
1663
1664 class ShenandoahSafeObjectIterateAdjustPtrsClosure : public MetadataAwareOopClosure {
1665 private:
1666 ShenandoahHeap* _heap;
1667
1668 public:
1669 ShenandoahSafeObjectIterateAdjustPtrsClosure() : _heap(ShenandoahHeap::heap()) {}
1670
1671 private:
1672 template <class T>
1673 inline void do_oop_work(T* p) {
1674 T o = oopDesc::load_heap_oop(p);
1675 if (!oopDesc::is_null(o)) {
1676 oop obj = oopDesc::decode_heap_oop_not_null(o);
1677 oopDesc::encode_store_heap_oop(p, BrooksPointer::forwardee(obj));
1678 }
1679 }
1680 public:
1681 void do_oop(oop* p) {
1682 do_oop_work(p);
1683 }
1684 void do_oop(narrowOop* p) {
1685 do_oop_work(p);
1686 }
1687 };
1688
1689 class ShenandoahSafeObjectIterateAndUpdate : public ObjectClosure {
1690 private:
1691 ObjectClosure* _cl;
1692 public:
1693 ShenandoahSafeObjectIterateAndUpdate(ObjectClosure *cl) : _cl(cl) {}
1694
1695 virtual void do_object(oop obj) {
1696 assert (oopDesc::unsafe_equals(obj, BrooksPointer::forwardee(obj)),
1697 "avoid double-counting: only non-forwarded objects here");
1698
1699 // Fix up the ptrs.
1700 ShenandoahSafeObjectIterateAdjustPtrsClosure adjust_ptrs;
1701 obj->oop_iterate(&adjust_ptrs);
1702
1703 // Can reply the object now:
1704 _cl->do_object(obj);
1705 }
1706 };
1707
1708 void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
1709 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1710
1711 // Safe iteration does objects only with correct references.
1712 // This is why we skip dirty regions that have stale copies of objects,
1713 // and fix up the pointers in the returned objects.
1714
1715 ShenandoahSafeObjectIterateAndUpdate safe_cl(cl);
1716 ShenandoahIterateObjectClosureRegionClosure blk(&safe_cl);
1717 heap_region_iterate(&blk,
1718 /* skip_dirty_regions = */ true,
1719 /* skip_humongous_continuations = */ true);
1720
1721 _need_update_refs = false; // already updated the references
1722 }
1723
1724 // Apply blk->doHeapRegion() on all committed regions in address order,
1725 // terminating the iteration early if doHeapRegion() returns true.
1726 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions, bool skip_humongous_continuation) const {
1727 for (size_t i = 0; i < _num_regions; i++) {
1728 ShenandoahHeapRegion* current = _ordered_regions->get(i);
1729 if (skip_humongous_continuation && current->is_humongous_continuation()) {
1730 continue;
1731 }
1732 if (skip_dirty_regions && in_collection_set(current)) {
1733 continue;
1734 }
1735 if (blk->doHeapRegion(current)) {
1736 return;
1737 }
1738 }
1739 }
1740
1741 class ClearLivenessClosure : public ShenandoahHeapRegionClosure {
1742 ShenandoahHeap* sh;
1743 public:
1744 ClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) { }
1745
1746 bool doHeapRegion(ShenandoahHeapRegion* r) {
1747 r->clear_live_data();
1748 sh->set_next_top_at_mark_start(r->bottom(), r->top());
1749 return false;
1750 }
1751 };
1752
1753 void ShenandoahHeap::start_concurrent_marking() {
1754
1755 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::accumulate_stats);
1756 accumulate_statistics_all_tlabs();
1757 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::accumulate_stats);
1758
1759 set_concurrent_mark_in_progress(true);
1760 // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1761 if (UseTLAB) {
1762 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::make_parsable);
1763 ensure_parsability(true);
1764 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::make_parsable);
1765 }
1766
1767 _shenandoah_policy->record_bytes_allocated(_bytes_allocated_since_cm);
1768 _used_start_gc = used();
1769
1770 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::clear_liveness);
1771 ClearLivenessClosure clc(this);
1772 heap_region_iterate(&clc);
1773 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::clear_liveness);
1774
1775 if (UseShenandoahMatrix) {
1776 connection_matrix()->clear_all();
1777 }
1778
1779 // Make above changes visible to worker threads
1780 OrderAccess::fence();
1781
1782 concurrentMark()->init_mark_roots();
1783 }
1784
1785 class VerifyAfterEvacuationClosure : public ExtendedOopClosure {
1786
1787 ShenandoahHeap* _sh;
1788
1789 public:
1790 VerifyAfterEvacuationClosure() : _sh ( ShenandoahHeap::heap() ) {}
1791
1792 template<class T> void do_oop_nv(T* p) {
1793 T heap_oop = oopDesc::load_heap_oop(p);
1794 if (!oopDesc::is_null(heap_oop)) {
1795 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1796 guarantee(_sh->in_collection_set(obj) == (! oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj))),
1797 "forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s obj-klass: %s, marked: %s",
1798 BOOL_TO_STR(_sh->in_collection_set(obj)),
1799 BOOL_TO_STR(! oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj))),
1800 obj->klass()->external_name(),
1801 BOOL_TO_STR(_sh->is_marked_complete(obj))
1802 );
1803 obj = oopDesc::bs()->read_barrier(obj);
1804 guarantee(! _sh->in_collection_set(obj), "forwarded oops must not point to dirty regions");
1805 guarantee(obj->is_oop(), "is_oop");
1806 guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace");
1807 }
1808 }
1809
1810 void do_oop(oop* p) { do_oop_nv(p); }
1811 void do_oop(narrowOop* p) { do_oop_nv(p); }
1812
1813 };
1814
1815 void ShenandoahHeap::verify_heap_after_evacuation() {
1816
1817 verify_heap_size_consistency();
1818
1819 ensure_parsability(false);
1820
1821 VerifyAfterEvacuationClosure cl;
1822 roots_iterate(&cl);
1823
1824 ObjectToOopClosure objs(&cl);
1825 object_iterate(&objs);
1826
1827 }
1828
1829 void ShenandoahHeap::swap_mark_bitmaps() {
1830 // Swap bitmaps.
1831 CMBitMap* tmp1 = _complete_mark_bit_map;
1832 _complete_mark_bit_map = _next_mark_bit_map;
1833 _next_mark_bit_map = tmp1;
1834
1835 // Swap top-at-mark-start pointers
1836 HeapWord** tmp2 = _complete_top_at_mark_starts;
1837 _complete_top_at_mark_starts = _next_top_at_mark_starts;
1838 _next_top_at_mark_starts = tmp2;
1839
1840 HeapWord** tmp3 = _complete_top_at_mark_starts_base;
1841 _complete_top_at_mark_starts_base = _next_top_at_mark_starts_base;
1842 _next_top_at_mark_starts_base = tmp3;
1843 }
1844
1845 class VerifyReachableHeapClosure : public ExtendedOopClosure {
1846 private:
1847 SCMObjToScanQueue* _queue;
1848 ShenandoahHeap* _heap;
1849 CMBitMap* _map;
1850 bool _check_matrix;
1851 oop _obj;
1852 public:
1853 VerifyReachableHeapClosure(SCMObjToScanQueue* queue, CMBitMap* map, bool check_matrix) :
1854 _queue(queue), _heap(ShenandoahHeap::heap()), _map(map), _check_matrix(check_matrix) {};
1855 template <class T>
1856 void do_oop_work(T* p) {
1857 T o = oopDesc::load_heap_oop(p);
1858 if (!oopDesc::is_null(o)) {
1859 oop obj = oopDesc::decode_heap_oop_not_null(o);
1860 guarantee(check_obj_alignment(obj), "sanity");
1861
1862 guarantee(!oopDesc::is_null(obj), "sanity");
1863 guarantee(_heap->is_in(obj), "sanity");
1864
1865 oop forw = BrooksPointer::forwardee(obj);
1866 guarantee(!oopDesc::is_null(forw), "sanity");
1867 guarantee(_heap->is_in(forw), "sanity");
1868
1869 guarantee(oopDesc::unsafe_equals(obj, forw), "should not be forwarded");
1870
1871 if (_check_matrix) {
1872 size_t from_idx = _heap->heap_region_index_containing(p);
1873 size_t to_idx = _heap->heap_region_index_containing(obj);
1874 if (!_heap->connection_matrix()->is_connected(from_idx, to_idx)) {
1875 tty->print_cr("from-obj: ");
1876 _obj->print_on(tty);
1877 tty->print_cr("to-obj:");
1878 obj->print_on(tty);
1879 tty->print_cr("from-obj allocated after mark: %s", BOOL_TO_STR(_heap->allocated_after_complete_mark_start((HeapWord*) _obj)));
1880 tty->print_cr("to-obj allocated after mark: %s", BOOL_TO_STR(_heap->allocated_after_complete_mark_start((HeapWord*) obj)));
1881 tty->print_cr("from-obj marked: %s", BOOL_TO_STR(_heap->is_marked_complete(_obj)));
1882 tty->print_cr("to-obj marked: %s", BOOL_TO_STR(_heap->is_marked_complete(obj)));
1883 tty->print_cr("from-idx: " SIZE_FORMAT ", to-idx: " SIZE_FORMAT, from_idx, to_idx);
1884
1885 oop fwd_from = BrooksPointer::forwardee(_obj);
1886 oop fwd_to = BrooksPointer::forwardee(obj);
1887 tty->print_cr("from-obj forwardee: " PTR_FORMAT, p2i(fwd_from));
1888 tty->print_cr("to-obj forwardee: " PTR_FORMAT, p2i(fwd_to));
1889 tty->print_cr("forward(from-obj) marked: %s", BOOL_TO_STR(_heap->is_marked_complete(fwd_from)));
1890 tty->print_cr("forward(to-obj) marked: %s", BOOL_TO_STR(_heap->is_marked_complete(fwd_to)));
1891 size_t fwd_from_idx = _heap->heap_region_index_containing(fwd_from);
1892 size_t fwd_to_idx = _heap->heap_region_index_containing(fwd_to);
1893 tty->print_cr("forward(from-idx): " SIZE_FORMAT ", forward(to-idx): " SIZE_FORMAT, fwd_from_idx, fwd_to_idx);
1894 tty->print_cr("forward(from) connected with forward(to)? %s", BOOL_TO_STR(_heap->connection_matrix()->is_connected(fwd_from_idx, fwd_to_idx)));
1895 }
1896 guarantee(oopDesc::unsafe_equals(ShenandoahBarrierSet::resolve_oop_static_not_null(obj), obj), "polizeilich verboten");
1897 guarantee(_heap->connection_matrix()->is_connected(from_idx, to_idx), "must be connected");
1898 }
1899
1900 if (_map->parMark((HeapWord*) obj)) {
1901 _queue->push(SCMTask(obj));
1902 }
1903 }
1904 }
1905
1906 void do_oop(oop* p) { do_oop_work(p); }
1907 void do_oop(narrowOop* p) { do_oop_work(p); }
1908 void set_obj(oop o) { _obj = o; }
1909 };
1910
1911 void ShenandoahHeap::verify_heap_reachable_at_safepoint() {
1912 guarantee(SafepointSynchronize::is_at_safepoint(), "only when nothing else happens");
1913 guarantee(ShenandoahVerify || (UseShenandoahMatrix && VerifyShenandoahMatrix),
1914 "only when these are enabled, and bitmap is initialized in ShenandoahHeap::initialize");
1915
1916 OrderAccess::fence();
1917 ensure_parsability(false);
1918
1919 // Allocate temporary bitmap for storing marking wavefront:
1920 MemRegion mr = MemRegion(_verification_bit_map.startWord(), _verification_bit_map.endWord());
1921 _verification_bit_map.clear_range_large(mr);
1922
1923 // Initialize a single queue
1924 SCMObjToScanQueue* q = new SCMObjToScanQueue();
1925 q->initialize();
1926
1927 // Scan root set
1928 ShenandoahRootProcessor rp(this, 1);
1929
1930 {
1931 VerifyReachableHeapClosure cl(q, &_verification_bit_map, false);
1932 CLDToOopClosure cld_cl(&cl);
1933 CodeBlobToOopClosure code_cl(&cl, ! CodeBlobToOopClosure::FixRelocations);
1934 rp.process_all_roots(&cl, &cl, &cld_cl, &code_cl, 0);
1935 }
1936
1937 // Finish the scan
1938 {
1939 VerifyReachableHeapClosure cl(q, &_verification_bit_map, UseShenandoahMatrix && VerifyShenandoahMatrix);
1940 SCMTask task;
1941 while ((q->pop_buffer(task) ||
1942 q->pop_local(task) ||
1943 q->pop_overflow(task))) {
1944 oop obj = task.obj();
1945 assert(!oopDesc::is_null(obj), "must not be null");
1946 cl.set_obj(obj);
1947 obj->oop_iterate(&cl);
1948 }
1949 }
1950
1951 // Clean up!
1952 delete(q);
1953 }
1954
1955 void ShenandoahHeap::stop_concurrent_marking() {
1956 assert(concurrent_mark_in_progress(), "How else could we get here?");
1957 if (! cancelled_concgc()) {
1958 // If we needed to update refs, and concurrent marking has been cancelled,
1959 // we need to finish updating references.
1960 set_need_update_refs(false);
1961 swap_mark_bitmaps();
1962 }
1963 set_concurrent_mark_in_progress(false);
1964
1965 if (log_is_enabled(Trace, gc, region)) {
1966 ResourceMark rm;
1967 outputStream* out = Log(gc, region)::trace_stream();
1968 print_heap_regions(out);
1969 }
1970
1971 }
1972
1973 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
1974 _concurrent_mark_in_progress = in_progress ? 1 : 0;
1975 JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
1976 }
1977
1978 void ShenandoahHeap::set_evacuation_in_progress_concurrently(bool in_progress) {
1979 // Note: it is important to first release the _evacuation_in_progress flag here,
1980 // so that Java threads can get out of oom_during_evacuation() and reach a safepoint,
1981 // in case a VM task is pending.
1982 set_evacuation_in_progress(in_progress);
1983 MutexLocker mu(Threads_lock);
1984 JavaThread::set_evacuation_in_progress_all_threads(in_progress);
1985 }
1986
1987 void ShenandoahHeap::set_evacuation_in_progress_at_safepoint(bool in_progress) {
1988 assert(SafepointSynchronize::is_at_safepoint(), "Only call this at safepoint");
1989 set_evacuation_in_progress(in_progress);
1990 JavaThread::set_evacuation_in_progress_all_threads(in_progress);
1991 }
1992
1993 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
1994 _evacuation_in_progress = in_progress ? 1 : 0;
1995 OrderAccess::fence();
1996 }
1997
1998 void ShenandoahHeap::verify_copy(oop p,oop c){
1999 assert(! oopDesc::unsafe_equals(p, oopDesc::bs()->read_barrier(p)), "forwarded correctly");
2000 assert(oopDesc::unsafe_equals(oopDesc::bs()->read_barrier(p), c), "verify pointer is correct");
2001 if (p->klass() != c->klass()) {
2002 print_heap_regions();
2003 }
2004 assert(p->klass() == c->klass(), "verify class p-size: "INT32_FORMAT" c-size: "INT32_FORMAT, p->size(), c->size());
2005 assert(p->size() == c->size(), "verify size");
2006 // Object may have been locked between copy and verification
2007 // assert(p->mark() == c->mark(), "verify mark");
2008 assert(oopDesc::unsafe_equals(c, oopDesc::bs()->read_barrier(c)), "verify only forwarded once");
2009 }
2010
2011 void ShenandoahHeap::oom_during_evacuation() {
2012 log_develop_trace(gc)("Out of memory during evacuation, cancel evacuation, schedule full GC by thread %d",
2013 Thread::current()->osthread()->thread_id());
2014
2015 // We ran out of memory during evacuation. Cancel evacuation, and schedule a full-GC.
2016 collector_policy()->set_should_clear_all_soft_refs(true);
2017 concurrent_thread()->try_set_full_gc();
2018 cancel_concgc(_oom_evacuation);
2019
2020 if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) {
2021 assert(! Threads_lock->owned_by_self()
2022 || SafepointSynchronize::is_at_safepoint(), "must not hold Threads_lock here");
2023 log_warning(gc)("OOM during evacuation. Let Java thread wait until evacuation finishes.");
2024 while (_evacuation_in_progress) { // wait.
2025 Thread::current()->_ParkEvent->park(1);
2026 }
2027 }
2028
2029 }
2030
2031 HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
2032 // Initialize Brooks pointer for the next object
2033 HeapWord* result = obj + BrooksPointer::word_size();
2034 BrooksPointer::initialize(oop(result));
2035 return result;
2036 }
2037
2038 uint ShenandoahHeap::oop_extra_words() {
2039 return BrooksPointer::word_size();
2040 }
2041
2042 void ShenandoahHeap::grow_heap_by(size_t num_regions) {
2043 size_t old_num_regions = _num_regions;
2044 ensure_new_regions(num_regions);
2045 for (size_t i = 0; i < num_regions; i++) {
2046 size_t new_region_index = i + old_num_regions;
2047 HeapWord* start = ((HeapWord*) base()) + (ShenandoahHeapRegion::region_size_bytes() / HeapWordSize) * new_region_index;
2048 ShenandoahHeapRegion* new_region = new ShenandoahHeapRegion(this, start, ShenandoahHeapRegion::region_size_bytes() / HeapWordSize, new_region_index);
2049
2050 if (log_is_enabled(Trace, gc, region)) {
2051 ResourceMark rm;
2052 outputStream* out = Log(gc, region)::trace_stream();
2053 out->print_cr("allocating new region at index: "SIZE_FORMAT, new_region_index);
2054 new_region->print_on(out);
2055 }
2056
2057 assert(_ordered_regions->active_regions() == new_region->region_number(), "must match");
2058 _ordered_regions->add_region(new_region);
2059 _in_cset_fast_test_base[new_region_index] = false; // Not in cset
2060 _next_top_at_mark_starts_base[new_region_index] = new_region->bottom();
2061 _complete_top_at_mark_starts_base[new_region_index] = new_region->bottom();
2062
2063 _free_regions->add_region(new_region);
2064 }
2065 }
2066
2067 void ShenandoahHeap::ensure_new_regions(size_t new_regions) {
2068
2069 size_t num_regions = _num_regions;
2070 size_t new_num_regions = num_regions + new_regions;
2071 assert(new_num_regions <= _max_regions, "we checked this earlier");
2072
2073 size_t expand_size = new_regions * ShenandoahHeapRegion::region_size_bytes();
2074 log_trace(gc, region)("expanding storage by "SIZE_FORMAT_HEX" bytes, for "SIZE_FORMAT" new regions", expand_size, new_regions);
2075 bool success = _storage.expand_by(expand_size, ShenandoahAlwaysPreTouch);
2076 assert(success, "should always be able to expand by requested size");
2077
2078 _num_regions = new_num_regions;
2079
2080 }
2081
2082 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
2083 _heap(ShenandoahHeap::heap_no_check()) {
2084 }
2085
2086 void ShenandoahForwardedIsAliveClosure::init(ShenandoahHeap* heap) {
2087 _heap = heap;
2088 }
2089
2090 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
2091
2092 assert(_heap != NULL, "sanity");
2093 obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
2094 #ifdef ASSERT
2095 if (_heap->concurrent_mark_in_progress()) {
2096 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
2097 }
2098 #endif
2099 assert(!oopDesc::is_null(obj), "null");
2100 return _heap->is_marked_next(obj);
2101 }
2102
2103 void ShenandoahHeap::ref_processing_init() {
2104 MemRegion mr = reserved_region();
2105
2106 isAlive.init(ShenandoahHeap::heap());
2107 assert(_max_workers > 0, "Sanity");
2108
2109 _ref_processor =
2110 new ReferenceProcessor(mr, // span
2111 ParallelRefProcEnabled,
2112 // mt processing
2113 _max_workers,
2114 // degree of mt processing
2115 true,
2116 // mt discovery
2117 _max_workers,
2118 // degree of mt discovery
2119 false,
2120 // Reference discovery is not atomic
2121 &isAlive);
2122 }
2123
2124 size_t ShenandoahHeap::num_regions() {
2125 return _num_regions;
2126 }
2127
2128 size_t ShenandoahHeap::max_regions() {
2129 return _max_regions;
2130 }
2131
2132 GCTracer* ShenandoahHeap::tracer() {
2133 return shenandoahPolicy()->tracer();
2134 }
2135
2136 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2137 return _free_regions->used();
2138 }
2139
2140 void ShenandoahHeap::cancel_concgc(GCCause::Cause cause) {
2141 if (try_cancel_concgc()) {
2142 log_info(gc)("Cancelling concurrent GC: %s", GCCause::to_string(cause));
2143 _shenandoah_policy->report_concgc_cancelled();
2144 }
2145 }
2146
2147 void ShenandoahHeap::cancel_concgc(ShenandoahCancelCause cause) {
2148 if (try_cancel_concgc()) {
2149 log_info(gc)("Cancelling concurrent GC: %s", cancel_cause_to_string(cause));
2150 _shenandoah_policy->report_concgc_cancelled();
2151 }
2152 }
2153
2154 const char* ShenandoahHeap::cancel_cause_to_string(ShenandoahCancelCause cause) {
2155 switch (cause) {
2156 case _oom_evacuation:
2157 return "Out of memory for evacuation";
2158 case _vm_stop:
2159 return "Stopping VM";
2160 default:
2161 return "Unknown";
2162 }
2163 }
2164
2165 uint ShenandoahHeap::max_workers() {
2166 return _max_workers;
2167 }
2168
2169 void ShenandoahHeap::stop() {
2170 // The shutdown sequence should be able to terminate when GC is running.
2171
2172 // Step 1. Notify control thread that we are in shutdown.
2173 // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2174 // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2175 _concurrent_gc_thread->prepare_for_graceful_shutdown();
2176
2177 // Step 2. Notify GC workers that we are cancelling GC.
2178 cancel_concgc(_vm_stop);
2179
2180 // Step 3. Wait until GC worker exits normally.
2181 _concurrent_gc_thread->stop();
2182 }
2183
2184 void ShenandoahHeap::unload_classes_and_cleanup_tables() {
2185 ShenandoahForwardedIsAliveClosure is_alive;
2186 // Unload classes and purge SystemDictionary.
2187 bool purged_class = SystemDictionary::do_unloading(&is_alive, true);
2188 ParallelCleaningTask unlink_task(&is_alive, true, true, _workers->active_workers(), purged_class);
2189 _workers->run_task(&unlink_task);
2190 ClassLoaderDataGraph::purge();
2191 }
2192
2193 void ShenandoahHeap::set_need_update_refs(bool need_update_refs) {
2194 _need_update_refs = need_update_refs;
2195 }
2196
2197 //fixme this should be in heapregionset
2198 ShenandoahHeapRegion* ShenandoahHeap::next_compaction_region(const ShenandoahHeapRegion* r) {
2199 size_t region_idx = r->region_number() + 1;
2200 ShenandoahHeapRegion* next = _ordered_regions->get(region_idx);
2201 guarantee(next->region_number() == region_idx, "region number must match");
2202 while (next->is_humongous()) {
2203 region_idx = next->region_number() + 1;
2204 next = _ordered_regions->get(region_idx);
2205 guarantee(next->region_number() == region_idx, "region number must match");
2206 }
2207 return next;
2208 }
2209
2210 void ShenandoahHeap::set_region_in_collection_set(size_t region_index, bool b) {
2211 _in_cset_fast_test_base[region_index] = b;
2212 }
2213
2214 ShenandoahMonitoringSupport* ShenandoahHeap::monitoring_support() {
2215 return _monitoring_support;
2216 }
2217
2218 CMBitMap* ShenandoahHeap::complete_mark_bit_map() {
2219 return _complete_mark_bit_map;
2220 }
2221
2222 CMBitMap* ShenandoahHeap::next_mark_bit_map() {
2223 return _next_mark_bit_map;
2224 }
2225
2226 void ShenandoahHeap::add_free_region(ShenandoahHeapRegion* r) {
2227 _free_regions->add_region(r);
2228 }
2229
2230 void ShenandoahHeap::clear_free_regions() {
2231 _free_regions->clear();
2232 }
2233
2234 address ShenandoahHeap::in_cset_fast_test_addr() {
2235 return (address) (ShenandoahHeap::heap()->_in_cset_fast_test);
2236 }
2237
2238 address ShenandoahHeap::cancelled_concgc_addr() {
2239 return (address) &(ShenandoahHeap::heap()->_cancelled_concgc);
2240 }
2241
2242 void ShenandoahHeap::clear_cset_fast_test() {
2243 assert(_in_cset_fast_test_base != NULL, "sanity");
2244 memset(_in_cset_fast_test_base, false,
2245 _in_cset_fast_test_length * sizeof(bool));
2246 }
2247
2248 size_t ShenandoahHeap::conservative_max_heap_alignment() {
2249 return ShenandoahMaxRegionSize;
2250 }
2251
2252 size_t ShenandoahHeap::bytes_allocated_since_cm() {
2253 return _bytes_allocated_since_cm;
2254 }
2255
2256 void ShenandoahHeap::set_bytes_allocated_since_cm(size_t bytes) {
2257 _bytes_allocated_since_cm = bytes;
2258 }
2259
2260 void ShenandoahHeap::set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
2261 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_shift();
2262 _next_top_at_mark_starts[index] = addr;
2263 }
2264
2265 HeapWord* ShenandoahHeap::next_top_at_mark_start(HeapWord* region_base) {
2266 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_shift();
2267 return _next_top_at_mark_starts[index];
2268 }
2269
2270 void ShenandoahHeap::set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
2271 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_shift();
2272 _complete_top_at_mark_starts[index] = addr;
2273 }
2274
2275 HeapWord* ShenandoahHeap::complete_top_at_mark_start(HeapWord* region_base) {
2276 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::region_size_shift();
2277 return _complete_top_at_mark_starts[index];
2278 }
2279
2280 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2281 _full_gc_in_progress = in_progress;
2282 }
2283
2284 bool ShenandoahHeap::is_full_gc_in_progress() const {
2285 return _full_gc_in_progress;
2286 }
2287
2288 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2289 _update_refs_in_progress = in_progress;
2290 }
2291
2292 bool ShenandoahHeap::is_update_refs_in_progress() const {
2293 return _update_refs_in_progress;
2294 }
2295
2296 class NMethodOopInitializer : public OopClosure {
2297 private:
2298 ShenandoahHeap* _heap;
2299 public:
2300 NMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {
2301 }
2302
2303 private:
2304 template <class T>
2305 inline void do_oop_work(T* p) {
2306 T o = oopDesc::load_heap_oop(p);
2307 if (! oopDesc::is_null(o)) {
2308 oop obj1 = oopDesc::decode_heap_oop_not_null(o);
2309 oop obj2 = oopDesc::bs()->write_barrier(obj1);
2310 if (! oopDesc::unsafe_equals(obj1, obj2)) {
2311 oopDesc::encode_store_heap_oop(p, obj2);
2312 }
2313 }
2314 }
2315
2316 public:
2317 void do_oop(oop* o) {
2318 do_oop_work(o);
2319 }
2320 void do_oop(narrowOop* o) {
2321 do_oop_work(o);
2322 }
2323 };
2324
2325 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2326 NMethodOopInitializer init;
2327 nm->oops_do(&init);
2328 nm->fix_oop_relocations();
2329 }
2330
2331 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2332 }
2333
2334 void ShenandoahHeap::pin_object(oop o) {
2335 heap_region_containing(o)->pin();
2336 }
2337
2338 void ShenandoahHeap::unpin_object(oop o) {
2339 heap_region_containing(o)->unpin();
2340 }
2341
2342
2343 GCTimer* ShenandoahHeap::gc_timer() const {
2344 return _gc_timer;
2345 }
2346
2347 class ShenandoahCountGarbageClosure : public ShenandoahHeapRegionClosure {
2348 private:
2349 size_t _garbage;
2350 public:
2351 ShenandoahCountGarbageClosure() : _garbage(0) {
2352 }
2353
2354 bool doHeapRegion(ShenandoahHeapRegion* r) {
2355 if (! r->is_humongous() && ! r->is_pinned() && ! r->in_collection_set()) {
2356 _garbage += r->garbage();
2357 }
2358 return false;
2359 }
2360
2361 size_t garbage() {
2362 return _garbage;
2363 }
2364 };
2365
2366 size_t ShenandoahHeap::garbage() {
2367 ShenandoahCountGarbageClosure cl;
2368 heap_region_iterate(&cl);
2369 return cl.garbage();
2370 }
2371
2372 ShenandoahConnectionMatrix* ShenandoahHeap::connection_matrix() {
2373 return _connection_matrix;
2374 }
2375
2376 ShenandoahPartialGC* ShenandoahHeap::partial_gc() {
2377 return _partial_gc;
2378 }
2379
2380 void ShenandoahHeap::do_partial_collection() {
2381 partial_gc()->do_partial_collection();
2382 }
2383
2384 template<class T>
2385 class ShenandoahUpdateHeapRefsTask : public AbstractGangTask {
2386 private:
2387 T cl;
2388 ShenandoahHeap* _heap;
2389 ShenandoahHeapRegionSet* _regions;
2390
2391 public:
2392 ShenandoahUpdateHeapRefsTask(ShenandoahHeapRegionSet* regions) :
2393 AbstractGangTask("Concurrent Update References Task"),
2394 cl(T()),
2395 _heap(ShenandoahHeap::heap()),
2396 _regions(regions) {
2397 }
2398
2399 void work(uint worker_id) {
2400 ShenandoahHeapRegion* r = _regions->claim_next();
2401 while (r != NULL) {
2402 if (_heap->in_collection_set(r)) {
2403 HeapWord* bottom = r->bottom();
2404 HeapWord* top = _heap->complete_top_at_mark_start(r->bottom());
2405 if (top > bottom) {
2406 _heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
2407 }
2408 } else {
2409 if (!r->is_empty()) {
2410 _heap->marked_object_oop_safe_iterate(r, &cl);
2411 }
2412 }
2413 if (_heap->cancelled_concgc()) {
2414 return;
2415 }
2416 r = _regions->claim_next();
2417 }
2418 }
2419 };
2420
2421 void ShenandoahHeap::update_heap_references(ShenandoahHeapRegionSet* update_regions) {
2422 if (UseShenandoahMatrix) {
2423 ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsMatrixClosure> task(update_regions);
2424 workers()->run_task(&task);
2425 } else {
2426 ShenandoahUpdateHeapRefsTask<ShenandoahUpdateHeapRefsClosure> task(update_regions);
2427 workers()->run_task(&task);
2428 }
2429 }
2430
2431 void ShenandoahHeap::concurrent_update_heap_references() {
2432 _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_update_refs);
2433 ShenandoahHeapRegionSet* update_regions = regions();
2434 update_regions->clear_current_index();
2435 update_heap_references(update_regions);
2436 _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_update_refs);
2437 }
2438
2439 void ShenandoahHeap::prepare_update_refs() {
2440 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2441 set_evacuation_in_progress_at_safepoint(false);
2442 set_update_refs_in_progress(true);
2443 ensure_parsability(true);
2444 if (UseShenandoahMatrix) {
2445 connection_matrix()->clear_all();
2446 }
2447 for (uint i = 0; i < _num_regions; i++) {
2448 ShenandoahHeapRegion* r = _ordered_regions->get(i);
2449 r->set_concurrent_iteration_safe_limit(r->top());
2450 }
2451 }
2452
2453 void ShenandoahHeap::finish_update_refs() {
2454 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint");
2455
2456 if (cancelled_concgc()) {
2457 // Finish updating references where we left off.
2458 clear_cancelled_concgc();
2459 ShenandoahHeapRegionSet* update_regions = regions();
2460 update_heap_references(update_regions);
2461 }
2462
2463 assert(! cancelled_concgc(), "Should have been done right before");
2464 concurrentMark()->update_roots(ShenandoahCollectorPolicy::final_update_refs_roots);
2465
2466 // Allocations might have happened before we STWed here, record peak:
2467 shenandoahPolicy()->record_peak_occupancy();
2468
2469 recycle_dirty_regions();
2470 set_need_update_refs(false);
2471
2472 if (ShenandoahVerify) {
2473 verify_update_refs();
2474 }
2475
2476 {
2477 // Rebuild the free set
2478 ShenandoahHeapLock hl(this);
2479 _free_regions->clear();
2480 size_t end = _ordered_regions->active_regions();
2481 for (size_t i = 0; i < end; i++) {
2482 ShenandoahHeapRegion* r = _ordered_regions->get(i);
2483 if (!r->is_humongous()) {
2484 assert (!in_collection_set(r), "collection set should be clear");
2485 _free_regions->add_region(r);
2486 }
2487 }
2488 }
2489 set_update_refs_in_progress(false);
2490 }
2491
2492 class ShenandoahVerifyUpdateRefsClosure : public ExtendedOopClosure {
2493 private:
2494 template <class T>
2495 void do_oop_work(T* p) {
2496 T o = oopDesc::load_heap_oop(p);
2497 if (! oopDesc::is_null(o)) {
2498 oop obj = oopDesc::decode_heap_oop_not_null(o);
2499 guarantee(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
2500 "must not be forwarded");
2501 }
2502 }
2503 public:
2504 void do_oop(oop* p) { do_oop_work(p); }
2505 void do_oop(narrowOop* p) { do_oop_work(p); }
2506 };
2507
2508 void ShenandoahHeap::verify_update_refs() {
2509
2510 ensure_parsability(false);
2511
2512 ShenandoahVerifyUpdateRefsClosure cl;
2513
2514 // Verify roots.
2515 {
2516 CodeBlobToOopClosure blobsCl(&cl, false);
2517 CLDToOopClosure cldCl(&cl);
2518 ShenandoahRootProcessor rp(this, 1);
2519 rp.process_all_roots(&cl, &cl, &cldCl, &blobsCl, 0);
2520 }
2521
2522 // Verify heap.
2523 for (uint i = 0; i < num_regions(); i++) {
2524 ShenandoahHeapRegion* r = regions()->get(i);
2525 marked_object_oop_iterate(r, &cl);
2526 }
2527 }
2528
2529 #ifdef ASSERT
2530 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2531 assert(_heap_lock == locked, "must be locked");
2532 assert(_heap_lock_owner == Thread::current(), "must be owned by current thread");
2533 }
2534
2535 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2536 Thread* thr = Thread::current();
2537 assert((_heap_lock == locked && _heap_lock_owner == thr) ||
2538 (SafepointSynchronize::is_at_safepoint() && thr->is_VM_thread()),
2539 "must own heap lock or by VM thread at safepoint");
2540 }
2541
2542 #endif
2543
2544 void ShenandoahHeap::start_deferred_recycling() {
2545 assert_heaplock_owned_by_current_thread();
2546 _recycled_region_count = 0;
2547 }
2548
2549 void ShenandoahHeap::defer_recycle(ShenandoahHeapRegion* r) {
2550 assert_heaplock_owned_by_current_thread();
2551 _recycled_regions[_recycled_region_count++] = r->region_number();
2552 }
2553
2554 void ShenandoahHeap::finish_deferred_recycle() {
2555 assert_heaplock_owned_by_current_thread();
2556 if (UseShenandoahMatrix) {
2557 for (size_t i = 0; i < _recycled_region_count; i++) {
2558 regions()->get(_recycled_regions[i])->recycle_no_matrix();
2559 }
2560 connection_matrix()->clear_batched(_recycled_regions, _recycled_region_count);
2561 } else {
2562 for (size_t i = 0; i < _recycled_region_count; i++) {
2563 regions()->get(_recycled_regions[i])->recycle();
2564 }
2565 }
2566 }
2567
2568 void ShenandoahHeap::deflate_idle_monitors_all_threads() {
2569 parallel_deflate_idle_monitors(workers());
2570 }