1 /*
2 * Copyright (c) 2013, 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "memory/allocation.hpp"
25
26 #include "gc/shared/gcTimer.hpp"
27 #include "gc/shared/gcTraceTime.inline.hpp"
28 #include "gc/shared/parallelCleaning.hpp"
29
30 #include "gc/shenandoah/brooksPointer.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
32 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
34 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
35 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
36 #include "gc/shenandoah/shenandoahConcurrentThread.hpp"
37 #include "gc/shenandoah/shenandoahFreeSet.hpp"
38 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
39 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
40 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
41 #include "gc/shenandoah/shenandoahHumongous.hpp"
42 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
43 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
44 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
45 #include "gc/shenandoah/vm_operations_shenandoah.hpp"
46
47 #include "runtime/vmThread.hpp"
48 #include "services/mallocTracker.hpp"
49
50 SCMUpdateRefsClosure::SCMUpdateRefsClosure() : _heap(ShenandoahHeap::heap()) {}
51
52 #ifdef ASSERT
53 template <class T>
54 void AssertToSpaceClosure::do_oop_nv(T* p) {
55 T o = oopDesc::load_heap_oop(p);
56 if (! oopDesc::is_null(o)) {
57 oop obj = oopDesc::decode_heap_oop_not_null(o);
58 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)),
59 "need to-space object here obj: "PTR_FORMAT" , rb(obj): "PTR_FORMAT", p: "PTR_FORMAT,
60 p2i(obj), p2i(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), p2i(p));
61 }
62 }
63
64 void AssertToSpaceClosure::do_oop(narrowOop* p) { do_oop_nv(p); }
65 void AssertToSpaceClosure::do_oop(oop* p) { do_oop_nv(p); }
66 #endif
67
68 const char* ShenandoahHeap::name() const {
69 return "Shenandoah";
70 }
71
72 void ShenandoahHeap::print_heap_locations(HeapWord* start, HeapWord* end) {
73 HeapWord* cur = NULL;
74 for (cur = start; cur < end; cur++) {
75 tty->print_cr(PTR_FORMAT" : "PTR_FORMAT, p2i(cur), p2i(*((HeapWord**) cur)));
76 }
77 }
78
79 class PrintHeapRegionsClosure : public
80 ShenandoahHeapRegionClosure {
81 private:
82 outputStream* _st;
83 public:
84 PrintHeapRegionsClosure() : _st(tty) {}
85 PrintHeapRegionsClosure(outputStream* st) : _st(st) {}
86
87 bool doHeapRegion(ShenandoahHeapRegion* r) {
88 r->print_on(_st);
89 return false;
90 }
91 };
92
93 class ShenandoahPretouchTask : public AbstractGangTask {
94 private:
95 ShenandoahHeapRegionSet* _regions;
96 const size_t _bitmap_size;
97 const size_t _page_size;
98 char* _bitmap0_base;
99 char* _bitmap1_base;
100 public:
101 ShenandoahPretouchTask(ShenandoahHeapRegionSet* regions,
102 char* bitmap0_base, char* bitmap1_base, size_t bitmap_size,
103 size_t page_size) :
104 AbstractGangTask("Shenandoah PreTouch",
105 Universe::is_fully_initialized() ? GCId::current_raw() :
106 // During VM initialization there is
107 // no GC cycle that this task can be
108 // associated with.
109 GCId::undefined()),
110 _bitmap0_base(bitmap0_base),
111 _bitmap1_base(bitmap1_base),
112 _regions(regions),
113 _bitmap_size(bitmap_size),
114 _page_size(page_size) {
115 _regions->clear_current_index();
116 };
117
118 virtual void work(uint worker_id) {
119 ShenandoahHeapRegion* r = _regions->claim_next();
120 while (r != NULL) {
121 log_trace(gc, heap)("Pretouch region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
122 r->region_number(), p2i(r->bottom()), p2i(r->end()));
123 os::pretouch_memory(r->bottom(), r->end(), _page_size);
124
125 size_t start = r->region_number() * ShenandoahHeapRegion::RegionSizeBytes / CMBitMap::heap_map_factor();
126 size_t end = (r->region_number() + 1) * ShenandoahHeapRegion::RegionSizeBytes / CMBitMap::heap_map_factor();
127 assert (end <= _bitmap_size, "end is sane: " SIZE_FORMAT " < " SIZE_FORMAT, end, _bitmap_size);
128
129 log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
130 r->region_number(), p2i(_bitmap0_base + start), p2i(_bitmap0_base + end));
131 os::pretouch_memory(_bitmap0_base + start, _bitmap0_base + end, _page_size);
132
133 log_trace(gc, heap)("Pretouch bitmap under region " SIZE_FORMAT ": " PTR_FORMAT " -> " PTR_FORMAT,
134 r->region_number(), p2i(_bitmap1_base + start), p2i(_bitmap1_base + end));
135 os::pretouch_memory(_bitmap1_base + start, _bitmap1_base + end, _page_size);
136
137 r = _regions->claim_next();
138 }
139 }
140 };
141
142 jint ShenandoahHeap::initialize() {
143 CollectedHeap::pre_initialize();
144
145 size_t init_byte_size = collector_policy()->initial_heap_byte_size();
146 size_t max_byte_size = collector_policy()->max_heap_byte_size();
147
148 Universe::check_alignment(max_byte_size,
149 ShenandoahHeapRegion::RegionSizeBytes,
150 "shenandoah heap");
151 Universe::check_alignment(init_byte_size,
152 ShenandoahHeapRegion::RegionSizeBytes,
153 "shenandoah heap");
154
155 ReservedSpace heap_rs = Universe::reserve_heap(max_byte_size,
156 Arguments::conservative_max_heap_alignment());
157 initialize_reserved_region((HeapWord*)heap_rs.base(), (HeapWord*) (heap_rs.base() + heap_rs.size()));
158
159 set_barrier_set(new ShenandoahBarrierSet(this));
160 ReservedSpace pgc_rs = heap_rs.first_part(max_byte_size);
161 _storage.initialize(pgc_rs, init_byte_size);
162
163 _num_regions = init_byte_size / ShenandoahHeapRegion::RegionSizeBytes;
164 _max_regions = max_byte_size / ShenandoahHeapRegion::RegionSizeBytes;
165 _initialSize = _num_regions * ShenandoahHeapRegion::RegionSizeBytes;
166 size_t regionSizeWords = ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize;
167 assert(init_byte_size == _initialSize, "tautology");
168 _ordered_regions = new ShenandoahHeapRegionSet(_max_regions);
169 _collection_set = new ShenandoahCollectionSet(_max_regions);
170 _free_regions = new ShenandoahFreeSet(_max_regions);
171
172 // Initialize fast collection set test structure.
173 _in_cset_fast_test_length = _max_regions;
174 _in_cset_fast_test_base =
175 NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length, mtGC);
176 _in_cset_fast_test = _in_cset_fast_test_base -
177 ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
178
179 _next_top_at_mark_starts_base =
180 NEW_C_HEAP_ARRAY(HeapWord*, _max_regions, mtGC);
181 _next_top_at_mark_starts = _next_top_at_mark_starts_base -
182 ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
183
184 _complete_top_at_mark_starts_base =
185 NEW_C_HEAP_ARRAY(HeapWord*, _max_regions, mtGC);
186 _complete_top_at_mark_starts = _complete_top_at_mark_starts_base -
187 ((uintx) pgc_rs.base() >> ShenandoahHeapRegion::RegionSizeShift);
188
189 size_t i = 0;
190 for (i = 0; i < _num_regions; i++) {
191 _in_cset_fast_test_base[i] = false; // Not in cset
192 HeapWord* bottom = (HeapWord*) pgc_rs.base() + regionSizeWords * i;
193 _complete_top_at_mark_starts_base[i] = bottom;
194 _next_top_at_mark_starts_base[i] = bottom;
195 }
196
197 {
198 ShenandoahHeapLock lock(this);
199 for (i = 0; i < _num_regions; i++) {
200 ShenandoahHeapRegion* current = new ShenandoahHeapRegion(this, (HeapWord*) pgc_rs.base() +
201 regionSizeWords * i, regionSizeWords, i);
202 _free_regions->add_region(current);
203 _ordered_regions->add_region(current);
204 }
205 }
206 assert(((size_t) _ordered_regions->active_regions()) == _num_regions, "");
207 _first_region = _ordered_regions->get(0);
208 _first_region_bottom = _first_region->bottom();
209 assert((((size_t) _first_region_bottom) &
210 (ShenandoahHeapRegion::RegionSizeBytes - 1)) == 0,
211 "misaligned heap: "PTR_FORMAT, p2i(_first_region_bottom));
212
213 _numAllocs = 0;
214
215 if (log_is_enabled(Trace, gc, region)) {
216 ResourceMark rm;
217 outputStream* out = Log(gc, region)::trace_stream();
218 log_trace(gc, region)("All Regions");
219 _ordered_regions->print(out);
220 log_trace(gc, region)("Free Regions");
221 _free_regions->print(out);
222 }
223
224 // The call below uses stuff (the SATB* things) that are in G1, but probably
225 // belong into a shared location.
226 JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon,
227 SATB_Q_FL_lock,
228 20 /*G1SATBProcessCompletedThreshold */,
229 Shared_SATB_Q_lock);
230
231 // Reserve space for prev and next bitmap.
232 size_t bitmap_size = CMBitMap::compute_size(heap_rs.size());
233 MemRegion heap_region = MemRegion((HeapWord*) heap_rs.base(), heap_rs.size() / HeapWordSize);
234
235 size_t page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size();
236
237 ReservedSpace bitmap0(bitmap_size, page_size);
238 os::commit_memory_or_exit(bitmap0.base(), bitmap0.size(), false, "couldn't allocate mark bitmap");
239 MemTracker::record_virtual_memory_type(bitmap0.base(), mtGC);
240 MemRegion bitmap_region0 = MemRegion((HeapWord*) bitmap0.base(), bitmap0.size() / HeapWordSize);
241
242 ReservedSpace bitmap1(bitmap_size, page_size);
243 os::commit_memory_or_exit(bitmap1.base(), bitmap1.size(), false, "couldn't allocate mark bitmap");
244 MemTracker::record_virtual_memory_type(bitmap1.base(), mtGC);
245 MemRegion bitmap_region1 = MemRegion((HeapWord*) bitmap1.base(), bitmap1.size() / HeapWordSize);
246
247 if (ShenandoahAlwaysPreTouch) {
248 assert (!AlwaysPreTouch, "Should have been overridden");
249
250 // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
251 // before initialize() below zeroes it with initializing thread. For any given region,
252 // we touch the region and the corresponding bitmaps from the same thread.
253
254 log_info(gc, heap)("Parallel pretouch " SIZE_FORMAT " regions with " SIZE_FORMAT " byte pages",
255 _ordered_regions->count(), page_size);
256 ShenandoahPretouchTask cl(_ordered_regions, bitmap0.base(), bitmap1.base(), bitmap_size, page_size);
257 _workers->run_task(&cl);
258 }
259
260 _mark_bit_map0.initialize(heap_region, bitmap_region0);
261 _complete_mark_bit_map = &_mark_bit_map0;
262
263 _mark_bit_map1.initialize(heap_region, bitmap_region1);
264 _next_mark_bit_map = &_mark_bit_map1;
265
266 _connection_matrix = new ShenandoahConnectionMatrix(_max_regions);
267
268 _monitoring_support = new ShenandoahMonitoringSupport(this);
269
270 _concurrent_gc_thread = new ShenandoahConcurrentThread();
271
272 ShenandoahMarkCompact::initialize();
273
274 return JNI_OK;
275 }
276
277 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
278 CollectedHeap(),
279 _shenandoah_policy(policy),
280 _concurrent_mark_in_progress(0),
281 _evacuation_in_progress(0),
282 _full_gc_in_progress(false),
283 _free_regions(NULL),
284 _collection_set(NULL),
285 _bytes_allocated_since_cm(0),
286 _bytes_allocated_during_cm(0),
287 _max_allocated_gc(0),
288 _allocated_last_gc(0),
289 _used_start_gc(0),
290 _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)),
291 _ref_processor(NULL),
292 _in_cset_fast_test(NULL),
293 _in_cset_fast_test_base(NULL),
294 _next_top_at_mark_starts(NULL),
295 _next_top_at_mark_starts_base(NULL),
296 _complete_top_at_mark_starts(NULL),
297 _complete_top_at_mark_starts_base(NULL),
298 _mark_bit_map0(),
299 _mark_bit_map1(),
300 _connection_matrix(NULL),
301 _cancelled_concgc(false),
302 _need_update_refs(false),
303 _need_reset_bitmaps(false),
304 _heap_lock(0),
305 #ifdef ASSERT
306 _heap_lock_owner(NULL),
307 #endif
308 _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer())
309
310 {
311 log_info(gc, init)("Parallel GC threads: "UINT32_FORMAT, ParallelGCThreads);
312 log_info(gc, init)("Concurrent GC threads: "UINT32_FORMAT, ConcGCThreads);
313 log_info(gc, init)("Parallel reference processing enabled: %s", BOOL_TO_STR(ParallelRefProcEnabled));
314
315 _scm = new ShenandoahConcurrentMark();
316 _used = 0;
317
318 _max_workers = MAX2(_max_workers, 1U);
319 _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers,
320 /* are_GC_task_threads */true,
321 /* are_ConcurrentGC_threads */false);
322 if (_workers == NULL) {
323 vm_exit_during_initialization("Failed necessary allocation.");
324 } else {
325 _workers->initialize_workers();
326 }
327 }
328
329 class ResetNextBitmapTask : public AbstractGangTask {
330 private:
331 ShenandoahHeapRegionSet* _regions;
332
333 public:
334 ResetNextBitmapTask(ShenandoahHeapRegionSet* regions) :
335 AbstractGangTask("Parallel Reset Bitmap Task"),
336 _regions(regions) {
337 _regions->clear_current_index();
338 }
339
340 void work(uint worker_id) {
341 ShenandoahHeapRegion* region = _regions->claim_next();
342 ShenandoahHeap* heap = ShenandoahHeap::heap();
343 while (region != NULL) {
344 HeapWord* bottom = region->bottom();
345 HeapWord* top = heap->next_top_at_mark_start(region->bottom());
346 if (top > bottom) {
347 heap->next_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
348 }
349 region = _regions->claim_next();
350 }
351 }
352 };
353
354 void ShenandoahHeap::reset_next_mark_bitmap(WorkGang* workers) {
355 ResetNextBitmapTask task = ResetNextBitmapTask(_ordered_regions);
356 workers->run_task(&task);
357 }
358
359 class ResetCompleteBitmapTask : public AbstractGangTask {
360 private:
361 ShenandoahHeapRegionSet* _regions;
362
363 public:
364 ResetCompleteBitmapTask(ShenandoahHeapRegionSet* regions) :
365 AbstractGangTask("Parallel Reset Bitmap Task"),
366 _regions(regions) {
367 _regions->clear_current_index();
368 }
369
370 void work(uint worker_id) {
371 ShenandoahHeapRegion* region = _regions->claim_next();
372 ShenandoahHeap* heap = ShenandoahHeap::heap();
373 while (region != NULL) {
374 HeapWord* bottom = region->bottom();
375 HeapWord* top = heap->complete_top_at_mark_start(region->bottom());
376 if (top > bottom) {
377 heap->complete_mark_bit_map()->clear_range_large(MemRegion(bottom, top));
378 }
379 region = _regions->claim_next();
380 }
381 }
382 };
383
384 void ShenandoahHeap::reset_complete_mark_bitmap(WorkGang* workers) {
385 ResetCompleteBitmapTask task = ResetCompleteBitmapTask(_ordered_regions);
386 workers->run_task(&task);
387 }
388
389 bool ShenandoahHeap::is_next_bitmap_clear() {
390 HeapWord* start = _ordered_regions->bottom();
391 HeapWord* end = _ordered_regions->end();
392 return _next_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
393 }
394
395 bool ShenandoahHeap::is_complete_bitmap_clear_range(HeapWord* start, HeapWord* end) {
396 return _complete_mark_bit_map->getNextMarkedWordAddress(start, end) == end;
397 }
398
399 void ShenandoahHeap::print_on(outputStream* st) const {
400 st->print("Shenandoah Heap");
401 st->print(" total = " SIZE_FORMAT " K, used " SIZE_FORMAT " K ", capacity()/ K, used() /K);
402 st->print(" [" PTR_FORMAT ", " PTR_FORMAT ") ",
403 p2i(reserved_region().start()),
404 p2i(reserved_region().end()));
405 st->print("Region size = " SIZE_FORMAT "K ", ShenandoahHeapRegion::RegionSizeBytes / K);
406 if (_concurrent_mark_in_progress) {
407 st->print("marking ");
408 }
409 if (_evacuation_in_progress) {
410 st->print("evacuating ");
411 }
412 if (cancelled_concgc()) {
413 st->print("cancelled ");
414 }
415 st->print("\n");
416
417 // Adapted from VirtualSpace::print_on(), which is non-PRODUCT only
418 st->print ("Virtual space:");
419 if (_storage.special()) st->print(" (pinned in memory)");
420 st->cr();
421 st->print_cr(" - committed: " SIZE_FORMAT, _storage.committed_size());
422 st->print_cr(" - reserved: " SIZE_FORMAT, _storage.reserved_size());
423 st->print_cr(" - [low, high]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(_storage.low()), p2i(_storage.high()));
424 st->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]", p2i(_storage.low_boundary()), p2i(_storage.high_boundary()));
425
426 if (Verbose) {
427 print_heap_regions(st);
428 }
429 }
430
431 class InitGCLABClosure : public ThreadClosure {
432 public:
433 void do_thread(Thread* thread) {
434 thread->gclab().initialize(true);
435 }
436 };
437
438 void ShenandoahHeap::post_initialize() {
439 if (UseTLAB) {
440 // This is a very tricky point in VM lifetime. We cannot easily call Threads::threads_do
441 // here, because some system threads (VMThread, WatcherThread, etc) are not yet available.
442 // Their initialization should be handled separately. Is we miss some threads here,
443 // then any other TLAB-related activity would fail with asserts.
444
445 InitGCLABClosure init_gclabs;
446 {
447 MutexLocker ml(Threads_lock);
448 for (JavaThread *thread = Threads::first(); thread != NULL; thread = thread->next()) {
449 init_gclabs.do_thread(thread);
450 }
451 }
452 gc_threads_do(&init_gclabs);
453
454 // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
455 // Now, we will let WorkGang to initialize gclab when new worker is created.
456 _workers->set_initialize_gclab();
457 }
458
459 _scm->initialize(_max_workers);
460
461 ref_processing_init();
462 }
463
464 class CalculateUsedRegionClosure : public ShenandoahHeapRegionClosure {
465 size_t sum;
466 public:
467
468 CalculateUsedRegionClosure() {
469 sum = 0;
470 }
471
472 bool doHeapRegion(ShenandoahHeapRegion* r) {
473 sum = sum + r->used();
474 return false;
475 }
476
477 size_t getResult() { return sum;}
478 };
479
480 size_t ShenandoahHeap::calculateUsed() {
481 CalculateUsedRegionClosure cl;
482 heap_region_iterate(&cl);
483 return cl.getResult();
484 }
485
486 void ShenandoahHeap::verify_heap_size_consistency() {
487
488 assert(calculateUsed() == used(),
489 "heap used size must be consistent heap-used: "SIZE_FORMAT" regions-used: "SIZE_FORMAT, used(), calculateUsed());
490 }
491
492 size_t ShenandoahHeap::used() const {
493 OrderAccess::acquire();
494 return _used;
495 }
496
497 void ShenandoahHeap::increase_used(size_t bytes) {
498 assert_heaplock_or_safepoint();
499 _used += bytes;
500 }
501
502 void ShenandoahHeap::set_used(size_t bytes) {
503 assert_heaplock_or_safepoint();
504 _used = bytes;
505 }
506
507 void ShenandoahHeap::decrease_used(size_t bytes) {
508 assert_heaplock_or_safepoint();
509 assert(_used >= bytes, "never decrease heap size by more than we've left");
510 _used -= bytes;
511 }
512
513 size_t ShenandoahHeap::capacity() const {
514 return _num_regions * ShenandoahHeapRegion::RegionSizeBytes;
515 }
516
517 bool ShenandoahHeap::is_maximal_no_gc() const {
518 Unimplemented();
519 return true;
520 }
521
522 size_t ShenandoahHeap::max_capacity() const {
523 return _max_regions * ShenandoahHeapRegion::RegionSizeBytes;
524 }
525
526 size_t ShenandoahHeap::min_capacity() const {
527 return _initialSize;
528 }
529
530 VirtualSpace* ShenandoahHeap::storage() const {
531 return (VirtualSpace*) &_storage;
532 }
533
534 bool ShenandoahHeap::is_in(const void* p) const {
535 HeapWord* first_region_bottom = _first_region->bottom();
536 HeapWord* last_region_end = first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * _num_regions;
537 return p >= _first_region_bottom && p < last_region_end;
538 }
539
540 bool ShenandoahHeap::is_scavengable(const void* p) {
541 return true;
542 }
543
544 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
545 // Retain tlab and allocate object in shared space if
546 // the amount free in the tlab is too large to discard.
547 if (thread->gclab().free() > thread->gclab().refill_waste_limit()) {
548 thread->gclab().record_slow_allocation(size);
549 return NULL;
550 }
551
552 // Discard gclab and allocate a new one.
553 // To minimize fragmentation, the last GCLAB may be smaller than the rest.
554 size_t new_gclab_size = thread->gclab().compute_size(size);
555
556 thread->gclab().clear_before_allocation();
557
558 if (new_gclab_size == 0) {
559 return NULL;
560 }
561
562 // Allocate a new GCLAB...
563 HeapWord* obj = allocate_new_gclab(new_gclab_size);
564 if (obj == NULL) {
565 return NULL;
566 }
567
568 if (ZeroTLAB) {
569 // ..and clear it.
570 Copy::zero_to_words(obj, new_gclab_size);
571 } else {
572 // ...and zap just allocated object.
573 #ifdef ASSERT
574 // Skip mangling the space corresponding to the object header to
575 // ensure that the returned space is not considered parsable by
576 // any concurrent GC thread.
577 size_t hdr_size = oopDesc::header_size();
578 Copy::fill_to_words(obj + hdr_size, new_gclab_size - hdr_size, badHeapWordVal);
579 #endif // ASSERT
580 }
581 thread->gclab().fill(obj, obj + size, new_gclab_size);
582 return obj;
583 }
584
585 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size) {
586 return allocate_new_tlab(word_size, false);
587 }
588
589 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t word_size) {
590 return allocate_new_tlab(word_size, true);
591 }
592
593 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t word_size, bool evacuating) {
594 HeapWord* result = allocate_memory(word_size, evacuating);
595
596 if (result != NULL) {
597 assert(! in_collection_set(result), "Never allocate in dirty region");
598 _bytes_allocated_since_cm += word_size * HeapWordSize;
599
600 log_develop_trace(gc, tlab)("allocating new tlab of size "SIZE_FORMAT" at addr "PTR_FORMAT, word_size, p2i(result));
601
602 }
603 return result;
604 }
605
606 ShenandoahHeap* ShenandoahHeap::heap() {
607 CollectedHeap* heap = Universe::heap();
608 assert(heap != NULL, "Unitialized access to ShenandoahHeap::heap()");
609 assert(heap->kind() == CollectedHeap::ShenandoahHeap, "not a shenandoah heap");
610 return (ShenandoahHeap*) heap;
611 }
612
613 ShenandoahHeap* ShenandoahHeap::heap_no_check() {
614 CollectedHeap* heap = Universe::heap();
615 return (ShenandoahHeap*) heap;
616 }
617
618 HeapWord* ShenandoahHeap::allocate_memory_work(size_t word_size) {
619
620 ShenandoahHeapLock heap_lock(this);
621
622 HeapWord* result = allocate_memory_under_lock(word_size);
623 int grow_by = (word_size * HeapWordSize + ShenandoahHeapRegion::RegionSizeBytes - 1) / ShenandoahHeapRegion::RegionSizeBytes;
624
625 while (result == NULL && _num_regions + grow_by <= _max_regions) {
626 grow_heap_by(grow_by);
627 result = allocate_memory_under_lock(word_size);
628 }
629
630 return result;
631 }
632
633 HeapWord* ShenandoahHeap::allocate_memory(size_t word_size, bool evacuating) {
634 HeapWord* result = NULL;
635 result = allocate_memory_work(word_size);
636
637 if (!evacuating) {
638 // Allocation failed, try full-GC, then retry allocation.
639 //
640 // It might happen that one of the threads requesting allocation would unblock
641 // way later after full-GC happened, only to fail the second allocation, because
642 // other threads have already depleted the free storage. In this case, a better
643 // strategy would be to try full-GC again.
644 //
645 // Lacking the way to detect progress from "collect" call, we are left with blindly
646 // retrying for some bounded number of times.
647 // TODO: Poll if Full GC made enough progress to warrant retry.
648 int tries = 0;
649 while ((result == NULL) && (tries++ < ShenandoahFullGCTries)) {
650 log_debug(gc)("[" PTR_FORMAT " Failed to allocate " SIZE_FORMAT " bytes, doing full GC, try %d",
651 p2i(Thread::current()), word_size * HeapWordSize, tries);
652 collect(GCCause::_allocation_failure);
653 result = allocate_memory_work(word_size);
654 }
655 }
656
657 // Only update monitoring counters when not calling from a write-barrier.
658 // Otherwise we might attempt to grab the Service_lock, which we must
659 // not do when coming from a write-barrier (because the thread might
660 // already hold the Compile_lock).
661 if (! evacuating) {
662 monitoring_support()->update_counters();
663 }
664
665 log_develop_trace(gc, alloc)("allocate memory chunk of size "SIZE_FORMAT" at addr "PTR_FORMAT " by thread %d ",
666 word_size, p2i(result), Thread::current()->osthread()->thread_id());
667
668 return result;
669 }
670
671 bool ShenandoahHeap::call_from_write_barrier(bool evacuating) {
672 return evacuating && Thread::current()->is_Java_thread();
673 }
674
675 HeapWord* ShenandoahHeap::allocate_memory_under_lock(size_t word_size) {
676 assert_heaplock_owned_by_current_thread();
677
678 if (word_size * HeapWordSize > ShenandoahHeapRegion::RegionSizeBytes) {
679 return allocate_large_memory(word_size);
680 }
681
682 // Not enough memory in free region set.
683 // Coming out of full GC, it is possible that there is not
684 // free region available, so current_index may not be valid.
685 if (word_size * HeapWordSize > _free_regions->capacity()) return NULL;
686
687 ShenandoahHeapRegion* my_current_region = _free_regions->current_no_humongous();
688
689 if (my_current_region == NULL) {
690 return NULL; // No more room to make a new region. OOM.
691 }
692 assert(my_current_region != NULL, "should have a region at this point");
693
694 #ifdef ASSERT
695 if (in_collection_set(my_current_region)) {
696 print_heap_regions();
697 }
698 #endif
699 assert(! in_collection_set(my_current_region), "never get targetted regions in free-lists");
700 assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
701
702 HeapWord* result = my_current_region->allocate(word_size);
703
704 while (result == NULL) {
705 // 2nd attempt. Try next region.
706 _free_regions->increase_used(my_current_region->free());
707 ShenandoahHeapRegion* next_region = _free_regions->next_no_humongous();
708 assert(next_region != my_current_region, "must not get current again");
709 my_current_region = next_region;
710
711 if (my_current_region == NULL) {
712 return NULL; // No more room to make a new region. OOM.
713 }
714 assert(my_current_region != NULL, "should have a region at this point");
715 assert(! in_collection_set(my_current_region), "never get targetted regions in free-lists");
716 assert(! my_current_region->is_humongous(), "never attempt to allocate from humongous object regions");
717 result = my_current_region->allocate(word_size);
718 }
719
720 my_current_region->increase_live_data_words(word_size);
721 increase_used(word_size * HeapWordSize);
722 _free_regions->increase_used(word_size * HeapWordSize);
723 return result;
724 }
725
726 HeapWord* ShenandoahHeap::allocate_large_memory(size_t words) {
727 assert_heaplock_owned_by_current_thread();
728
729 uint required_regions = ShenandoahHumongous::required_regions(words * HeapWordSize);
730 if (required_regions > _max_regions) return NULL;
731
732 ShenandoahHeapRegion* r = _free_regions->allocate_contiguous(required_regions);
733
734 HeapWord* result = NULL;
735
736 if (r != NULL) {
737 result = r->bottom();
738
739 log_debug(gc, humongous)("allocating humongous object of size: "SIZE_FORMAT" KB at location "PTR_FORMAT" in start region "SIZE_FORMAT,
740 (words * HeapWordSize) / K, p2i(result), r->region_number());
741 } else {
742 log_debug(gc, humongous)("allocating humongous object of size: "SIZE_FORMAT" KB at location "PTR_FORMAT" failed",
743 (words * HeapWordSize) / K, p2i(result));
744 }
745
746
747 return result;
748
749 }
750
751 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
752 bool* gc_overhead_limit_was_exceeded) {
753
754 #ifdef ASSERT
755 if (ShenandoahVerify && _numAllocs > 1000000) {
756 _numAllocs = 0;
757 }
758 _numAllocs++;
759 #endif
760 HeapWord* filler = allocate_memory(BrooksPointer::word_size() + size, false);
761 HeapWord* result = filler + BrooksPointer::word_size();
762 if (filler != NULL) {
763 BrooksPointer::initialize(oop(result));
764 _bytes_allocated_since_cm += size * HeapWordSize;
765
766 assert(! in_collection_set(result), "never allocate in targetted region");
767 return result;
768 } else {
769 /*
770 tty->print_cr("Out of memory. Requested number of words: "SIZE_FORMAT" used heap: "INT64_FORMAT", bytes allocated since last CM: "INT64_FORMAT,
771 size, used(), _bytes_allocated_since_cm);
772 {
773 print_heap_regions();
774 tty->print("Printing "SIZE_FORMAT" free regions:\n", _free_regions->count());
775 _free_regions->print();
776 }
777 */
778 return NULL;
779 }
780 }
781
782 class ParallelEvacuateRegionObjectClosure : public ObjectClosure {
783 private:
784 ShenandoahHeap* _heap;
785 Thread* _thread;
786 public:
787 ParallelEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
788 _heap(heap), _thread(Thread::current()) {
789 }
790
791 void do_object(oop p) {
792
793 log_develop_trace(gc, compaction)("Calling ParallelEvacuateRegionObjectClosure on "PTR_FORMAT" of size %d\n", p2i((HeapWord*) p), p->size());
794
795 assert(_heap->is_marked_complete(p), "expect only marked objects");
796 if (oopDesc::unsafe_equals(p, ShenandoahBarrierSet::resolve_oop_static_not_null(p))) {
797 _heap->evacuate_object(p, _thread);
798 }
799 }
800 };
801
802 #ifdef ASSERT
803 class VerifyEvacuatedObjectClosure : public ObjectClosure {
804
805 public:
806
807 void do_object(oop p) {
808 if (ShenandoahHeap::heap()->is_marked_complete(p)) {
809 oop p_prime = oopDesc::bs()->read_barrier(p);
810 assert(! oopDesc::unsafe_equals(p, p_prime), "Should point to evacuated copy");
811 if (p->klass() != p_prime->klass()) {
812 tty->print_cr("copy has different class than original:");
813 p->klass()->print_on(tty);
814 p_prime->klass()->print_on(tty);
815 }
816 assert(p->klass() == p_prime->klass(), "Should have the same class p: "PTR_FORMAT", p_prime: "PTR_FORMAT, p2i(p), p2i(p_prime));
817 // assert(p->mark() == p_prime->mark(), "Should have the same mark");
818 assert(p->size() == p_prime->size(), "Should be the same size");
819 assert(oopDesc::unsafe_equals(p_prime, oopDesc::bs()->read_barrier(p_prime)), "One forward once");
820 }
821 }
822 };
823
824 void ShenandoahHeap::verify_evacuated_region(ShenandoahHeapRegion* from_region) {
825 VerifyEvacuatedObjectClosure verify_evacuation;
826 marked_object_iterate(from_region, &verify_evacuation);
827 }
828 #endif
829
830 void ShenandoahHeap::parallel_evacuate_region(ShenandoahHeapRegion* from_region) {
831
832 assert(from_region->has_live(), "all-garbage regions are reclaimed earlier");
833
834 ParallelEvacuateRegionObjectClosure evacuate_region(this);
835
836 marked_object_iterate(from_region, &evacuate_region);
837
838 #ifdef ASSERT
839 if (ShenandoahVerify && ! cancelled_concgc()) {
840 verify_evacuated_region(from_region);
841 }
842 #endif
843 }
844
845 class ParallelEvacuationTask : public AbstractGangTask {
846 private:
847 ShenandoahHeap* _sh;
848 ShenandoahCollectionSet* _cs;
849
850 public:
851 ParallelEvacuationTask(ShenandoahHeap* sh,
852 ShenandoahCollectionSet* cs) :
853 AbstractGangTask("Parallel Evacuation Task"),
854 _cs(cs),
855 _sh(sh) {}
856
857 void work(uint worker_id) {
858
859 ShenandoahHeapRegion* from_hr = _cs->claim_next();
860
861 while (from_hr != NULL) {
862 log_develop_trace(gc, region)("Thread "INT32_FORMAT" claimed Heap Region "SIZE_FORMAT,
863 worker_id,
864 from_hr->region_number());
865
866 assert(from_hr->has_live(), "all-garbage regions are reclaimed early");
867 _sh->parallel_evacuate_region(from_hr);
868
869 if (_sh->cancelled_concgc()) {
870 log_develop_trace(gc, region)("Cancelled concgc while evacuating region " SIZE_FORMAT "\n", from_hr->region_number());
871 break;
872 }
873 from_hr = _cs->claim_next();
874 }
875 }
876 };
877
878 class RecycleDirtyRegionsClosure: public ShenandoahHeapRegionClosure {
879 private:
880 ShenandoahHeap* _heap;
881 size_t _bytes_reclaimed;
882 public:
883 RecycleDirtyRegionsClosure() : _heap(ShenandoahHeap::heap()) {}
884
885 bool doHeapRegion(ShenandoahHeapRegion* r) {
886
887 assert (! _heap->cancelled_concgc(), "no recycling after cancelled marking");
888
889 if (_heap->in_collection_set(r)) {
890 log_develop_trace(gc, region)("Recycling region " SIZE_FORMAT ":", r->region_number());
891 _heap->decrease_used(r->used());
892 _bytes_reclaimed += r->used();
893 r->recycle();
894 }
895
896 return false;
897 }
898 size_t bytes_reclaimed() { return _bytes_reclaimed;}
899 void clear_bytes_reclaimed() {_bytes_reclaimed = 0;}
900 };
901
902 void ShenandoahHeap::recycle_dirty_regions() {
903 RecycleDirtyRegionsClosure cl;
904 cl.clear_bytes_reclaimed();
905
906 heap_region_iterate(&cl);
907
908 _shenandoah_policy->record_bytes_reclaimed(cl.bytes_reclaimed());
909 if (! cancelled_concgc()) {
910 clear_cset_fast_test();
911 }
912 }
913
914 ShenandoahFreeSet* ShenandoahHeap::free_regions() {
915 return _free_regions;
916 }
917
918 void ShenandoahHeap::print_heap_regions(outputStream* st) const {
919 _ordered_regions->print(st);
920 }
921
922 class PrintAllRefsOopClosure: public ExtendedOopClosure {
923 private:
924 int _index;
925 const char* _prefix;
926
927 public:
928 PrintAllRefsOopClosure(const char* prefix) : _index(0), _prefix(prefix) {}
929
930 private:
931 template <class T>
932 inline void do_oop_work(T* p) {
933 oop o = oopDesc::load_decode_heap_oop(p);
934 if (o != NULL) {
935 if (ShenandoahHeap::heap()->is_in(o) && o->is_oop()) {
936 tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT")-> "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT")",
937 _prefix, _index,
938 p2i(p), p2i(o),
939 BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_complete(o)),
940 o->klass()->internal_name(), p2i(o->klass()));
941 } else {
942 tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT" dirty -> "PTR_FORMAT" (not in heap, possibly corrupted or dirty)",
943 _prefix, _index,
944 p2i(p), p2i(o));
945 }
946 } else {
947 tty->print_cr("%s "INT32_FORMAT" ("PTR_FORMAT") -> "PTR_FORMAT, _prefix, _index, p2i(p), p2i((HeapWord*) o));
948 }
949 _index++;
950 }
951
952 public:
953 void do_oop(oop* p) {
954 do_oop_work(p);
955 }
956
957 void do_oop(narrowOop* p) {
958 do_oop_work(p);
959 }
960
961 };
962
963 class PrintAllRefsObjectClosure : public ObjectClosure {
964 const char* _prefix;
965
966 public:
967 PrintAllRefsObjectClosure(const char* prefix) : _prefix(prefix) {}
968
969 void do_object(oop p) {
970 if (ShenandoahHeap::heap()->is_in(p)) {
971 tty->print_cr("%s object "PTR_FORMAT" (marked: %s) (%s "PTR_FORMAT") refers to:",
972 _prefix, p2i(p),
973 BOOL_TO_STR(ShenandoahHeap::heap()->is_marked_complete(p)),
974 p->klass()->internal_name(), p2i(p->klass()));
975 PrintAllRefsOopClosure cl(_prefix);
976 p->oop_iterate(&cl);
977 }
978 }
979 };
980
981 void ShenandoahHeap::print_all_refs(const char* prefix) {
982 tty->print_cr("printing all references in the heap");
983 tty->print_cr("root references:");
984
985 ensure_parsability(false);
986
987 PrintAllRefsOopClosure cl(prefix);
988 roots_iterate(&cl);
989
990 tty->print_cr("heap references:");
991 PrintAllRefsObjectClosure cl2(prefix);
992 object_iterate(&cl2);
993 }
994
995 class VerifyAfterMarkingOopClosure: public ExtendedOopClosure {
996 private:
997 ShenandoahHeap* _heap;
998
999 public:
1000 VerifyAfterMarkingOopClosure() :
1001 _heap(ShenandoahHeap::heap()) { }
1002
1003 private:
1004 template <class T>
1005 inline void do_oop_work(T* p) {
1006 oop o = oopDesc::load_decode_heap_oop(p);
1007 if (o != NULL) {
1008 if (! _heap->is_marked_complete(o)) {
1009 _heap->print_heap_regions();
1010 _heap->print_all_refs("post-mark");
1011 tty->print_cr("oop not marked, although referrer is marked: "PTR_FORMAT": in_heap: %s, is_marked: %s",
1012 p2i((HeapWord*) o), BOOL_TO_STR(_heap->is_in(o)), BOOL_TO_STR(_heap->is_marked_complete(o)));
1013 _heap->print_heap_locations((HeapWord*) o, (HeapWord*) o + o->size());
1014
1015 tty->print_cr("oop class: %s", o->klass()->internal_name());
1016 if (_heap->is_in(p)) {
1017 oop referrer = oop(_heap->heap_region_containing(p)->block_start_const(p));
1018 tty->print_cr("Referrer starts at addr "PTR_FORMAT, p2i((HeapWord*) referrer));
1019 referrer->print();
1020 _heap->print_heap_locations((HeapWord*) referrer, (HeapWord*) referrer + referrer->size());
1021 }
1022 tty->print_cr("heap region containing object:");
1023 _heap->heap_region_containing(o)->print();
1024 tty->print_cr("heap region containing referrer:");
1025 _heap->heap_region_containing(p)->print();
1026 tty->print_cr("heap region containing forwardee:");
1027 _heap->heap_region_containing(oopDesc::bs()->read_barrier(o))->print();
1028 }
1029 assert(o->is_oop(), "oop must be an oop");
1030 assert(Metaspace::contains(o->klass()), "klass pointer must go to metaspace");
1031 if (! oopDesc::unsafe_equals(o, oopDesc::bs()->read_barrier(o))) {
1032 tty->print_cr("oops has forwardee: p: "PTR_FORMAT" (%s), o = "PTR_FORMAT" (%s), new-o: "PTR_FORMAT" (%s)",
1033 p2i(p),
1034 BOOL_TO_STR(_heap->in_collection_set(p)),
1035 p2i(o),
1036 BOOL_TO_STR(_heap->in_collection_set(o)),
1037 p2i((HeapWord*) oopDesc::bs()->read_barrier(o)),
1038 BOOL_TO_STR(_heap->in_collection_set(oopDesc::bs()->read_barrier(o))));
1039 tty->print_cr("oop class: %s", o->klass()->internal_name());
1040 }
1041 assert(oopDesc::unsafe_equals(o, oopDesc::bs()->read_barrier(o)), "oops must not be forwarded");
1042 assert(! _heap->in_collection_set(o), "references must not point to dirty heap regions");
1043 assert(_heap->is_marked_complete(o), "live oops must be marked current");
1044 }
1045 }
1046
1047 public:
1048 void do_oop(oop* p) {
1049 do_oop_work(p);
1050 }
1051
1052 void do_oop(narrowOop* p) {
1053 do_oop_work(p);
1054 }
1055
1056 };
1057
1058 void ShenandoahHeap::verify_heap_after_marking() {
1059
1060 verify_heap_size_consistency();
1061
1062 log_trace(gc)("verifying heap after marking");
1063
1064 VerifyAfterMarkingOopClosure cl;
1065 roots_iterate(&cl);
1066 ObjectToOopClosure objs(&cl);
1067 object_iterate(&objs);
1068 }
1069
1070
1071 void ShenandoahHeap::reclaim_humongous_region_at(ShenandoahHeapRegion* r) {
1072 assert(r->is_humongous_start(), "reclaim regions starting with the first one");
1073
1074 oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
1075 size_t size = humongous_obj->size() + BrooksPointer::word_size();
1076 uint required_regions = ShenandoahHumongous::required_regions(size * HeapWordSize);
1077 uint index = r->region_number();
1078
1079
1080 assert(!r->has_live(), "liveness must be zero");
1081
1082 for(size_t i = 0; i < required_regions; i++) {
1083
1084 ShenandoahHeapRegion* region = _ordered_regions->get(index++);
1085
1086 assert((region->is_humongous_start() || region->is_humongous_continuation()),
1087 "expect correct humongous start or continuation");
1088
1089 if (log_is_enabled(Debug, gc, humongous)) {
1090 log_debug(gc, humongous)("reclaiming "UINT32_FORMAT" humongous regions for object of size: "SIZE_FORMAT" words", required_regions, size);
1091 ResourceMark rm;
1092 outputStream* out = Log(gc, humongous)::debug_stream();
1093 region->print_on(out);
1094 }
1095
1096 region->recycle();
1097 ShenandoahHeap::heap()->decrease_used(ShenandoahHeapRegion::RegionSizeBytes);
1098 }
1099 }
1100
1101 class ShenandoahReclaimHumongousRegionsClosure : public ShenandoahHeapRegionClosure {
1102
1103 bool doHeapRegion(ShenandoahHeapRegion* r) {
1104 ShenandoahHeap* heap = ShenandoahHeap::heap();
1105
1106 if (r->is_humongous_start()) {
1107 oop humongous_obj = oop(r->bottom() + BrooksPointer::word_size());
1108 if (! heap->is_marked_complete(humongous_obj)) {
1109
1110 heap->reclaim_humongous_region_at(r);
1111 }
1112 }
1113 return false;
1114 }
1115 };
1116
1117 #ifdef ASSERT
1118 class CheckCollectionSetClosure: public ShenandoahHeapRegionClosure {
1119 bool doHeapRegion(ShenandoahHeapRegion* r) {
1120 assert(! ShenandoahHeap::heap()->in_collection_set(r), "Should have been cleared by now");
1121 return false;
1122 }
1123 };
1124 #endif
1125
1126 void ShenandoahHeap::prepare_for_concurrent_evacuation() {
1127 assert(_ordered_regions->get(0)->region_number() == 0, "FIXME CHF. FIXME CHF!");
1128
1129 log_develop_trace(gc)("Thread %d started prepare_for_concurrent_evacuation", Thread::current()->osthread()->thread_id());
1130
1131 if (!cancelled_concgc()) {
1132
1133 recycle_dirty_regions();
1134
1135 ensure_parsability(true);
1136
1137 if (UseShenandoahMatrix) {
1138 if (PrintShenandoahMatrix) {
1139 outputStream* log = Log(gc)::info_stream();
1140 connection_matrix()->print_on(log);
1141 }
1142 if (VerifyShenandoahMatrix) {
1143 verify_matrix();
1144 }
1145 }
1146
1147 #ifdef ASSERT
1148 if (ShenandoahVerify) {
1149 verify_heap_after_marking();
1150 }
1151 #endif
1152
1153 // NOTE: This needs to be done during a stop the world pause, because
1154 // putting regions into the collection set concurrently with Java threads
1155 // will create a race. In particular, acmp could fail because when we
1156 // resolve the first operand, the containing region might not yet be in
1157 // the collection set, and thus return the original oop. When the 2nd
1158 // operand gets resolved, the region could be in the collection set
1159 // and the oop gets evacuated. If both operands have originally been
1160 // the same, we get false negatives.
1161
1162 {
1163 ShenandoahHeapLock lock(this);
1164 _collection_set->clear();
1165 _free_regions->clear();
1166
1167 ShenandoahReclaimHumongousRegionsClosure reclaim;
1168 heap_region_iterate(&reclaim);
1169
1170 #ifdef ASSERT
1171 CheckCollectionSetClosure ccsc;
1172 _ordered_regions->heap_region_iterate(&ccsc);
1173 #endif
1174
1175 _shenandoah_policy->choose_collection_set(_collection_set);
1176
1177 _shenandoah_policy->choose_free_set(_free_regions);
1178 }
1179
1180 _bytes_allocated_since_cm = 0;
1181
1182 Universe::update_heap_info_at_gc();
1183 }
1184 }
1185
1186
1187 class RetireTLABClosure : public ThreadClosure {
1188 private:
1189 bool _retire;
1190
1191 public:
1192 RetireTLABClosure(bool retire) : _retire(retire) {
1193 }
1194
1195 void do_thread(Thread* thread) {
1196 thread->gclab().make_parsable(_retire);
1197 }
1198 };
1199
1200 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1201 if (UseTLAB) {
1202 CollectedHeap::ensure_parsability(retire_tlabs);
1203 RetireTLABClosure cl(retire_tlabs);
1204 Threads::threads_do(&cl);
1205 }
1206 }
1207
1208 class ShenandoahEvacuateUpdateRootsClosure: public ExtendedOopClosure {
1209 private:
1210 ShenandoahHeap* _heap;
1211 Thread* _thread;
1212 public:
1213 ShenandoahEvacuateUpdateRootsClosure() :
1214 _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
1215 }
1216
1217 private:
1218 template <class T>
1219 void do_oop_work(T* p) {
1220 assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
1221
1222 T o = oopDesc::load_heap_oop(p);
1223 if (! oopDesc::is_null(o)) {
1224 oop obj = oopDesc::decode_heap_oop_not_null(o);
1225 if (_heap->in_collection_set(obj)) {
1226 assert(_heap->is_marked_complete(obj), "only evacuate marked objects %d %d",
1227 _heap->is_marked_complete(obj), _heap->is_marked_complete(ShenandoahBarrierSet::resolve_oop_static_not_null(obj)));
1228 oop resolved = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
1229 if (oopDesc::unsafe_equals(resolved, obj)) {
1230 resolved = _heap->evacuate_object(obj, _thread);
1231 }
1232 oopDesc::encode_store_heap_oop(p, resolved);
1233 }
1234 }
1235 #ifdef ASSERT
1236 else {
1237 // tty->print_cr("not updating root at: "PTR_FORMAT" with object: "PTR_FORMAT", is_in_heap: %s, is_in_cset: %s, is_marked: %s",
1238 // p2i(p),
1239 // p2i((HeapWord*) obj),
1240 // BOOL_TO_STR(_heap->is_in(obj)),
1241 // BOOL_TO_STR(_heap->in_cset_fast_test(obj)),
1242 // BOOL_TO_STR(_heap->is_marked_complete(obj)));
1243 }
1244 #endif
1245 }
1246
1247 public:
1248 void do_oop(oop* p) {
1249 do_oop_work(p);
1250 }
1251 void do_oop(narrowOop* p) {
1252 do_oop_work(p);
1253 }
1254 };
1255
1256 class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
1257 ShenandoahRootEvacuator* _rp;
1258 public:
1259
1260 ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
1261 AbstractGangTask("Shenandoah evacuate and update roots"),
1262 _rp(rp)
1263 {
1264 // Nothing else to do.
1265 }
1266
1267 void work(uint worker_id) {
1268 ShenandoahEvacuateUpdateRootsClosure cl;
1269 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1270
1271 _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1272 }
1273 };
1274
1275 class ShenandoahFixRootsTask : public AbstractGangTask {
1276 ShenandoahRootEvacuator* _rp;
1277 public:
1278
1279 ShenandoahFixRootsTask(ShenandoahRootEvacuator* rp) :
1280 AbstractGangTask("Shenandoah update roots"),
1281 _rp(rp)
1282 {
1283 // Nothing else to do.
1284 }
1285
1286 void work(uint worker_id) {
1287 SCMUpdateRefsClosure cl;
1288 MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
1289
1290 _rp->process_evacuate_roots(&cl, &blobsCl, worker_id);
1291 }
1292 };
1293 void ShenandoahHeap::evacuate_and_update_roots() {
1294
1295 COMPILER2_PRESENT(DerivedPointerTable::clear());
1296
1297 #ifdef ASSERT
1298 if (ShenandoahVerifyReadsToFromSpace) {
1299 set_from_region_protection(false);
1300 }
1301 #endif
1302
1303 assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
1304 ClassLoaderDataGraph::clear_claimed_marks();
1305
1306 {
1307 ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahCollectorPolicy::evac_thread_roots);
1308 ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
1309 workers()->run_task(&roots_task);
1310 }
1311
1312 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
1313
1314 if (cancelled_concgc()) {
1315 // If initial evacuation has been cancelled, we need to update all references
1316 // after all workers have finished. Otherwise we might run into the following problem:
1317 // GC thread 1 cannot allocate anymore, thus evacuation fails, leaves from-space ptr of object X.
1318 // GC thread 2 evacuates the same object X to to-space
1319 // which leaves a truly dangling from-space reference in the first root oop*. This must not happen.
1320 // clear() and update_pointers() must always be called in pairs,
1321 // cannot nest with above clear()/update_pointers().
1322 COMPILER2_PRESENT(DerivedPointerTable::clear());
1323 ShenandoahRootEvacuator rp(this, workers()->active_workers(), ShenandoahCollectorPolicy::evac_thread_roots);
1324 ShenandoahFixRootsTask update_roots_task(&rp);
1325 workers()->run_task(&update_roots_task);
1326 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
1327 }
1328
1329 #ifdef ASSERT
1330 if (ShenandoahVerifyReadsToFromSpace) {
1331 set_from_region_protection(true);
1332 }
1333 #endif
1334
1335 #ifdef ASSERT
1336 {
1337 AssertToSpaceClosure cl;
1338 CodeBlobToOopClosure code_cl(&cl, !CodeBlobToOopClosure::FixRelocations);
1339 ShenandoahRootEvacuator rp(this, 1);
1340 rp.process_evacuate_roots(&cl, &code_cl, 0);
1341 }
1342 #endif
1343 }
1344
1345
1346 void ShenandoahHeap::do_evacuation() {
1347
1348 parallel_evacuate();
1349
1350 if (ShenandoahVerify && ! cancelled_concgc()) {
1351 VM_ShenandoahVerifyHeapAfterEvacuation verify_after_evacuation;
1352 if (Thread::current()->is_VM_thread()) {
1353 verify_after_evacuation.doit();
1354 } else {
1355 VMThread::execute(&verify_after_evacuation);
1356 }
1357 }
1358
1359 }
1360
1361 void ShenandoahHeap::parallel_evacuate() {
1362 log_develop_trace(gc)("starting parallel_evacuate");
1363
1364 _shenandoah_policy->record_phase_start(ShenandoahCollectorPolicy::conc_evac);
1365
1366 if (log_is_enabled(Trace, gc, region)) {
1367 ResourceMark rm;
1368 outputStream *out = Log(gc, region)::trace_stream();
1369 out->print("Printing all available regions");
1370 print_heap_regions(out);
1371 }
1372
1373 if (log_is_enabled(Trace, gc, cset)) {
1374 ResourceMark rm;
1375 outputStream *out = Log(gc, cset)::trace_stream();
1376 out->print("Printing collection set which contains "SIZE_FORMAT" regions:\n", _collection_set->count());
1377 _collection_set->print(out);
1378
1379 out->print("Printing free set which contains "SIZE_FORMAT" regions:\n", _free_regions->count());
1380 _free_regions->print(out);
1381 }
1382
1383 ParallelEvacuationTask evacuationTask = ParallelEvacuationTask(this, _collection_set);
1384
1385
1386 workers()->run_task(&evacuationTask);
1387
1388 if (log_is_enabled(Trace, gc, cset)) {
1389 ResourceMark rm;
1390 outputStream *out = Log(gc, cset)::trace_stream();
1391 out->print("Printing postgc collection set which contains "SIZE_FORMAT" regions:\n",
1392 _collection_set->count());
1393
1394 _collection_set->print(out);
1395
1396 out->print("Printing postgc free regions which contain "SIZE_FORMAT" free regions:\n",
1397 _free_regions->count());
1398 _free_regions->print(out);
1399
1400 }
1401
1402 if (log_is_enabled(Trace, gc, region)) {
1403 ResourceMark rm;
1404 outputStream *out = Log(gc, region)::trace_stream();
1405 out->print_cr("all regions after evacuation:");
1406 print_heap_regions(out);
1407 }
1408
1409 _shenandoah_policy->record_phase_end(ShenandoahCollectorPolicy::conc_evac);
1410 }
1411
1412 class VerifyEvacuationClosure: public ExtendedOopClosure {
1413 private:
1414 ShenandoahHeap* _heap;
1415 ShenandoahHeapRegion* _from_region;
1416
1417 public:
1418 VerifyEvacuationClosure(ShenandoahHeapRegion* from_region) :
1419 _heap(ShenandoahHeap::heap()), _from_region(from_region) { }
1420 private:
1421 template <class T>
1422 inline void do_oop_work(T* p) {
1423 oop heap_oop = oopDesc::load_decode_heap_oop(p);
1424 if (! oopDesc::is_null(heap_oop)) {
1425 guarantee(! _from_region->is_in(heap_oop), "no references to from-region allowed after evacuation: "PTR_FORMAT, p2i((HeapWord*) heap_oop));
1426 }
1427 }
1428
1429 public:
1430 void do_oop(oop* p) {
1431 do_oop_work(p);
1432 }
1433
1434 void do_oop(narrowOop* p) {
1435 do_oop_work(p);
1436 }
1437
1438 };
1439
1440 void ShenandoahHeap::roots_iterate(OopClosure* cl) {
1441
1442 assert(SafepointSynchronize::is_at_safepoint(), "Only iterate roots while world is stopped");
1443
1444 CodeBlobToOopClosure blobsCl(cl, false);
1445 CLDToOopClosure cldCl(cl);
1446
1447 ClassLoaderDataGraph::clear_claimed_marks();
1448
1449 ShenandoahRootProcessor rp(this, 1);
1450 rp.process_all_roots(cl, NULL, &cldCl, &blobsCl, 0);
1451 }
1452
1453 void ShenandoahHeap::verify_evacuation(ShenandoahHeapRegion* from_region) {
1454
1455 VerifyEvacuationClosure rootsCl(from_region);
1456 roots_iterate(&rootsCl);
1457
1458 }
1459
1460 bool ShenandoahHeap::supports_tlab_allocation() const {
1461 return true;
1462 }
1463
1464
1465 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1466 size_t idx = _free_regions->current_index();
1467 ShenandoahHeapRegion* current = _free_regions->get(idx);
1468 if (current == NULL) {
1469 return 0;
1470 } else if (current->free() > MinTLABSize) {
1471 // Current region has enough space left, can use it.
1472 return current->free();
1473 } else {
1474 // No more space in current region, we will take next free region
1475 // on the next TLAB allocation.
1476 return ShenandoahHeapRegion::RegionSizeBytes;
1477 }
1478 }
1479
1480 size_t ShenandoahHeap::max_tlab_size() const {
1481 return ShenandoahHeapRegion::RegionSizeBytes;
1482 }
1483
1484 class ResizeGCLABClosure : public ThreadClosure {
1485 public:
1486 void do_thread(Thread* thread) {
1487 thread->gclab().resize();
1488 }
1489 };
1490
1491 void ShenandoahHeap::resize_all_tlabs() {
1492 CollectedHeap::resize_all_tlabs();
1493
1494 ResizeGCLABClosure cl;
1495 Threads::threads_do(&cl);
1496 }
1497
1498 class AccumulateStatisticsGCLABClosure : public ThreadClosure {
1499 public:
1500 void do_thread(Thread* thread) {
1501 thread->gclab().accumulate_statistics();
1502 thread->gclab().initialize_statistics();
1503 }
1504 };
1505
1506 void ShenandoahHeap::accumulate_statistics_all_gclabs() {
1507 AccumulateStatisticsGCLABClosure cl;
1508 Threads::threads_do(&cl);
1509 }
1510
1511 bool ShenandoahHeap::can_elide_tlab_store_barriers() const {
1512 return true;
1513 }
1514
1515 oop ShenandoahHeap::new_store_pre_barrier(JavaThread* thread, oop new_obj) {
1516 // Overridden to do nothing.
1517 return new_obj;
1518 }
1519
1520 bool ShenandoahHeap::can_elide_initializing_store_barrier(oop new_obj) {
1521 return true;
1522 }
1523
1524 bool ShenandoahHeap::card_mark_must_follow_store() const {
1525 return false;
1526 }
1527
1528 void ShenandoahHeap::collect(GCCause::Cause cause) {
1529 assert(cause != GCCause::_gc_locker, "no JNI critical callback");
1530 if (GCCause::is_user_requested_gc(cause)) {
1531 if (! DisableExplicitGC) {
1532 _concurrent_gc_thread->do_full_gc(cause);
1533 }
1534 } else if (cause == GCCause::_allocation_failure) {
1535 collector_policy()->set_should_clear_all_soft_refs(true);
1536 _concurrent_gc_thread->do_full_gc(cause);
1537 }
1538 }
1539
1540 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1541 //assert(false, "Shouldn't need to do full collections");
1542 }
1543
1544 AdaptiveSizePolicy* ShenandoahHeap::size_policy() {
1545 Unimplemented();
1546 return NULL;
1547
1548 }
1549
1550 CollectorPolicy* ShenandoahHeap::collector_policy() const {
1551 return _shenandoah_policy;
1552 }
1553
1554
1555 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1556 Space* sp = heap_region_containing(addr);
1557 if (sp != NULL) {
1558 return sp->block_start(addr);
1559 }
1560 return NULL;
1561 }
1562
1563 size_t ShenandoahHeap::block_size(const HeapWord* addr) const {
1564 Space* sp = heap_region_containing(addr);
1565 assert(sp != NULL, "block_size of address outside of heap");
1566 return sp->block_size(addr);
1567 }
1568
1569 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1570 Space* sp = heap_region_containing(addr);
1571 return sp->block_is_obj(addr);
1572 }
1573
1574 jlong ShenandoahHeap::millis_since_last_gc() {
1575 return 0;
1576 }
1577
1578 void ShenandoahHeap::prepare_for_verify() {
1579 if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1580 ensure_parsability(false);
1581 }
1582 }
1583
1584 void ShenandoahHeap::print_gc_threads_on(outputStream* st) const {
1585 workers()->print_worker_threads_on(st);
1586 }
1587
1588 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1589 workers()->threads_do(tcl);
1590 }
1591
1592 void ShenandoahHeap::print_tracing_info() const {
1593 if (log_is_enabled(Info, gc, stats)) {
1594 ResourceMark rm;
1595 outputStream* out = Log(gc, stats)::info_stream();
1596 _shenandoah_policy->print_tracing_info(out);
1597 }
1598 }
1599
1600 class ShenandoahVerifyRootsClosure: public ExtendedOopClosure {
1601 private:
1602 ShenandoahHeap* _heap;
1603 VerifyOption _vo;
1604 bool _failures;
1605 public:
1606 // _vo == UsePrevMarking -> use "prev" marking information,
1607 // _vo == UseNextMarking -> use "next" marking information,
1608 // _vo == UseMarkWord -> use mark word from object header.
1609 ShenandoahVerifyRootsClosure(VerifyOption vo) :
1610 _heap(ShenandoahHeap::heap()),
1611 _vo(vo),
1612 _failures(false) { }
1613
1614 bool failures() { return _failures; }
1615
1616 private:
1617 template <class T>
1618 inline void do_oop_work(T* p) {
1619 oop obj = oopDesc::load_decode_heap_oop(p);
1620 if (! oopDesc::is_null(obj) && ! obj->is_oop()) {
1621 { // Just for debugging.
1622 tty->print_cr("Root location "PTR_FORMAT
1623 "verified "PTR_FORMAT, p2i(p), p2i((void*) obj));
1624 // obj->print_on(tty);
1625 }
1626 }
1627 guarantee(obj->is_oop_or_null(), "is oop or null");
1628 }
1629
1630 public:
1631 void do_oop(oop* p) {
1632 do_oop_work(p);
1633 }
1634
1635 void do_oop(narrowOop* p) {
1636 do_oop_work(p);
1637 }
1638
1639 };
1640
1641 class ShenandoahVerifyHeapClosure: public ObjectClosure {
1642 private:
1643 ShenandoahVerifyRootsClosure _rootsCl;
1644 public:
1645 ShenandoahVerifyHeapClosure(ShenandoahVerifyRootsClosure rc) :
1646 _rootsCl(rc) {};
1647
1648 void do_object(oop p) {
1649 _rootsCl.do_oop(&p);
1650 }
1651 };
1652
1653 class ShenandoahVerifyKlassClosure: public KlassClosure {
1654 OopClosure *_oop_closure;
1655 public:
1656 ShenandoahVerifyKlassClosure(OopClosure* cl) : _oop_closure(cl) {}
1657 void do_klass(Klass* k) {
1658 k->oops_do(_oop_closure);
1659 }
1660 };
1661
1662 void ShenandoahHeap::verify(VerifyOption vo) {
1663 if (SafepointSynchronize::is_at_safepoint() || ! UseTLAB) {
1664
1665 ShenandoahVerifyRootsClosure rootsCl(vo);
1666
1667 assert(Thread::current()->is_VM_thread(),
1668 "Expected to be executed serially by the VM thread at this point");
1669
1670 roots_iterate(&rootsCl);
1671
1672 bool failures = rootsCl.failures();
1673 log_trace(gc)("verify failures: %s", BOOL_TO_STR(failures));
1674
1675 ShenandoahVerifyHeapClosure heapCl(rootsCl);
1676
1677 object_iterate(&heapCl);
1678 // TODO: Implement rest of it.
1679 } else {
1680 tty->print("(SKIPPING roots, heapRegions, remset) ");
1681 }
1682 }
1683 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1684 return _free_regions->capacity();
1685 }
1686
1687 class ShenandoahIterateObjectClosureRegionClosure: public ShenandoahHeapRegionClosure {
1688 ObjectClosure* _cl;
1689 public:
1690 ShenandoahIterateObjectClosureRegionClosure(ObjectClosure* cl) : _cl(cl) {}
1691 bool doHeapRegion(ShenandoahHeapRegion* r) {
1692 ShenandoahHeap::heap()->marked_object_iterate(r, _cl);
1693 return false;
1694 }
1695 };
1696
1697 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1698 ShenandoahIterateObjectClosureRegionClosure blk(cl);
1699 heap_region_iterate(&blk, false, true);
1700 }
1701
1702 class ShenandoahSafeObjectIterateAdjustPtrsClosure : public MetadataAwareOopClosure {
1703 private:
1704 ShenandoahHeap* _heap;
1705
1706 public:
1707 ShenandoahSafeObjectIterateAdjustPtrsClosure() : _heap(ShenandoahHeap::heap()) {}
1708
1709 private:
1710 template <class T>
1711 inline void do_oop_work(T* p) {
1712 T o = oopDesc::load_heap_oop(p);
1713 if (!oopDesc::is_null(o)) {
1714 oop obj = oopDesc::decode_heap_oop_not_null(o);
1715 oopDesc::encode_store_heap_oop(p, BrooksPointer::forwardee(obj));
1716 }
1717 }
1718 public:
1719 void do_oop(oop* p) {
1720 do_oop_work(p);
1721 }
1722 void do_oop(narrowOop* p) {
1723 do_oop_work(p);
1724 }
1725 };
1726
1727 class ShenandoahSafeObjectIterateAndUpdate : public ObjectClosure {
1728 private:
1729 ObjectClosure* _cl;
1730 public:
1731 ShenandoahSafeObjectIterateAndUpdate(ObjectClosure *cl) : _cl(cl) {}
1732
1733 virtual void do_object(oop obj) {
1734 assert (oopDesc::unsafe_equals(obj, BrooksPointer::forwardee(obj)),
1735 "avoid double-counting: only non-forwarded objects here");
1736
1737 // Fix up the ptrs.
1738 ShenandoahSafeObjectIterateAdjustPtrsClosure adjust_ptrs;
1739 obj->oop_iterate(&adjust_ptrs);
1740
1741 // Can reply the object now:
1742 _cl->do_object(obj);
1743 }
1744 };
1745
1746 void ShenandoahHeap::safe_object_iterate(ObjectClosure* cl) {
1747 assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1748
1749 // Safe iteration does objects only with correct references.
1750 // This is why we skip dirty regions that have stale copies of objects,
1751 // and fix up the pointers in the returned objects.
1752
1753 ShenandoahSafeObjectIterateAndUpdate safe_cl(cl);
1754 ShenandoahIterateObjectClosureRegionClosure blk(&safe_cl);
1755 heap_region_iterate(&blk,
1756 /* skip_dirty_regions = */ true,
1757 /* skip_humongous_continuations = */ true);
1758
1759 _need_update_refs = false; // already updated the references
1760 }
1761
1762 // Apply blk->doHeapRegion() on all committed regions in address order,
1763 // terminating the iteration early if doHeapRegion() returns true.
1764 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk, bool skip_dirty_regions, bool skip_humongous_continuation) const {
1765 for (size_t i = 0; i < _num_regions; i++) {
1766 ShenandoahHeapRegion* current = _ordered_regions->get(i);
1767 if (skip_humongous_continuation && current->is_humongous_continuation()) {
1768 continue;
1769 }
1770 if (skip_dirty_regions && in_collection_set(current)) {
1771 continue;
1772 }
1773 if (blk->doHeapRegion(current)) {
1774 return;
1775 }
1776 }
1777 }
1778
1779 class ClearLivenessClosure : public ShenandoahHeapRegionClosure {
1780 ShenandoahHeap* sh;
1781 public:
1782 ClearLivenessClosure(ShenandoahHeap* heap) : sh(heap) { }
1783
1784 bool doHeapRegion(ShenandoahHeapRegion* r) {
1785 r->clear_live_data();
1786 sh->set_next_top_at_mark_start(r->bottom(), r->top());
1787 return false;
1788 }
1789 };
1790
1791 void ShenandoahHeap::start_concurrent_marking() {
1792
1793 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::accumulate_stats);
1794 accumulate_statistics_all_tlabs();
1795 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::accumulate_stats);
1796
1797 set_concurrent_mark_in_progress(true);
1798 // We need to reset all TLABs because we'd lose marks on all objects allocated in them.
1799 if (UseTLAB) {
1800 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::make_parsable);
1801 ensure_parsability(true);
1802 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::make_parsable);
1803 }
1804
1805 _shenandoah_policy->record_bytes_allocated(_bytes_allocated_since_cm);
1806 _used_start_gc = used();
1807
1808 #ifdef ASSERT
1809 if (ShenandoahDumpHeapBeforeConcurrentMark) {
1810 ensure_parsability(false);
1811 print_all_refs("pre-mark");
1812 }
1813 #endif
1814
1815 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::clear_liveness);
1816 ClearLivenessClosure clc(this);
1817 heap_region_iterate(&clc);
1818 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::clear_liveness);
1819
1820 if (UseShenandoahMatrix) {
1821 connection_matrix()->clear_all();
1822 }
1823 // print_all_refs("pre -mark");
1824
1825 // oopDesc::_debug = true;
1826
1827 // Make above changes visible to worker threads
1828 OrderAccess::fence();
1829
1830 shenandoahPolicy()->record_phase_start(ShenandoahCollectorPolicy::scan_roots);
1831 concurrentMark()->init_mark_roots();
1832 shenandoahPolicy()->record_phase_end(ShenandoahCollectorPolicy::scan_roots);
1833
1834 // print_all_refs("pre-mark2");
1835 }
1836
1837 class VerifyAfterEvacuationClosure : public ExtendedOopClosure {
1838
1839 ShenandoahHeap* _sh;
1840
1841 public:
1842 VerifyAfterEvacuationClosure() : _sh ( ShenandoahHeap::heap() ) {}
1843
1844 template<class T> void do_oop_nv(T* p) {
1845 T heap_oop = oopDesc::load_heap_oop(p);
1846 if (!oopDesc::is_null(heap_oop)) {
1847 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
1848 guarantee(_sh->in_collection_set(obj) == (! oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj))),
1849 "forwarded objects can only exist in dirty (from-space) regions is_dirty: %s, is_forwarded: %s obj-klass: %s, marked: %s",
1850 BOOL_TO_STR(_sh->in_collection_set(obj)),
1851 BOOL_TO_STR(! oopDesc::unsafe_equals(obj, oopDesc::bs()->read_barrier(obj))),
1852 obj->klass()->external_name(),
1853 BOOL_TO_STR(_sh->is_marked_complete(obj))
1854 );
1855 obj = oopDesc::bs()->read_barrier(obj);
1856 guarantee(! _sh->in_collection_set(obj), "forwarded oops must not point to dirty regions");
1857 guarantee(obj->is_oop(), "is_oop");
1858 guarantee(Metaspace::contains(obj->klass()), "klass pointer must go to metaspace");
1859 }
1860 }
1861
1862 void do_oop(oop* p) { do_oop_nv(p); }
1863 void do_oop(narrowOop* p) { do_oop_nv(p); }
1864
1865 };
1866
1867 void ShenandoahHeap::verify_heap_after_evacuation() {
1868
1869 verify_heap_size_consistency();
1870
1871 ensure_parsability(false);
1872
1873 VerifyAfterEvacuationClosure cl;
1874 roots_iterate(&cl);
1875
1876 ObjectToOopClosure objs(&cl);
1877 object_iterate(&objs);
1878
1879 }
1880
1881 class VerifyRegionsAfterUpdateRefsClosure : public ShenandoahHeapRegionClosure {
1882 public:
1883 bool doHeapRegion(ShenandoahHeapRegion* r) {
1884 assert(! ShenandoahHeap::heap()->in_collection_set(r), "no region must be in collection set");
1885 return false;
1886 }
1887 };
1888
1889 void ShenandoahHeap::swap_mark_bitmaps() {
1890 // Swap bitmaps.
1891 CMBitMap* tmp1 = _complete_mark_bit_map;
1892 _complete_mark_bit_map = _next_mark_bit_map;
1893 _next_mark_bit_map = tmp1;
1894
1895 // Swap top-at-mark-start pointers
1896 HeapWord** tmp2 = _complete_top_at_mark_starts;
1897 _complete_top_at_mark_starts = _next_top_at_mark_starts;
1898 _next_top_at_mark_starts = tmp2;
1899
1900 HeapWord** tmp3 = _complete_top_at_mark_starts_base;
1901 _complete_top_at_mark_starts_base = _next_top_at_mark_starts_base;
1902 _next_top_at_mark_starts_base = tmp3;
1903 }
1904
1905 class ShenandoahVerifyMatrixOopClosure : public ExtendedOopClosure {
1906 private:
1907 oop _obj;
1908
1909 template <class T>
1910 inline void do_oop_nv(T* p) {
1911 T o = oopDesc::load_heap_oop(p);
1912 if (! oopDesc::is_null(o)) {
1913 oop obj = oopDesc::decode_heap_oop_not_null(o);
1914 ShenandoahHeap* heap = ShenandoahHeap::heap();
1915 guarantee(heap->is_marked_complete(obj), "must be marked");
1916
1917 uint from_idx = heap->heap_region_index_containing(p);
1918 uint to_idx = heap->heap_region_index_containing(obj);
1919 if (!heap->connection_matrix()->is_connected(from_idx, to_idx)) {
1920 tty->print_cr("from-obj: ");
1921 _obj->print_on(tty);
1922 tty->print_cr("to-obj:");
1923 obj->print_on(tty);
1924 tty->print_cr("from-obj allocated after mark: %s", BOOL_TO_STR(heap->allocated_after_complete_mark_start((HeapWord*) _obj)));
1925 tty->print_cr("to-obj allocated after mark: %s", BOOL_TO_STR(heap->allocated_after_complete_mark_start((HeapWord*) obj)));
1926 tty->print_cr("from-obj marked: %s", BOOL_TO_STR(heap->is_marked_complete(_obj)));
1927 tty->print_cr("to-obj marked: %s", BOOL_TO_STR(heap->is_marked_complete(obj)));
1928 tty->print_cr("from-idx: %u, to-idx: %u", from_idx, to_idx);
1929
1930 oop fwd_from = BrooksPointer::forwardee(_obj);
1931 oop fwd_to = BrooksPointer::forwardee(obj);
1932 tty->print_cr("from-obj forwardee: " PTR_FORMAT, p2i(fwd_from));
1933 tty->print_cr("to-obj forwardee: " PTR_FORMAT, p2i(fwd_to));
1934 tty->print_cr("forward(from-obj) marked: %s", BOOL_TO_STR(heap->is_marked_complete(fwd_from)));
1935 tty->print_cr("forward(to-obj) marked: %s", BOOL_TO_STR(heap->is_marked_complete(fwd_to)));
1936 uint fwd_from_idx = heap->heap_region_index_containing(fwd_from);
1937 uint fwd_to_idx = heap->heap_region_index_containing(fwd_to);
1938 tty->print_cr("forward(from-idx): %u, forward(to-idx): %u", fwd_from_idx, fwd_to_idx);
1939 tty->print_cr("forward(from) connected with forward(to)? %s", BOOL_TO_STR(heap->connection_matrix()->is_connected(fwd_from_idx, fwd_to_idx)));
1940 tty->print_cr("sizeof(bool): %lu", sizeof(bool));
1941 }
1942 guarantee(oopDesc::unsafe_equals(ShenandoahBarrierSet::resolve_oop_static_not_null(obj), obj), "polizeilich verboten");
1943 guarantee(heap->connection_matrix()->is_connected(from_idx, to_idx), "must be connected");
1944 }
1945 }
1946
1947 public:
1948 ShenandoahVerifyMatrixOopClosure(oop obj) : _obj(obj) {}
1949
1950 void do_oop(oop* o) {
1951 do_oop_nv(o);
1952 }
1953
1954 void do_oop(narrowOop* o) {
1955 do_oop_nv(o);
1956 }
1957 };
1958
1959 class ShenandoahVerifyMatrixObjectClosure : public ObjectClosure {
1960 public:
1961 void do_object(oop obj) {
1962 guarantee(ShenandoahHeap::heap()->is_marked_complete(obj), "must be marked");
1963 ShenandoahVerifyMatrixOopClosure cl(obj);
1964 obj->oop_iterate(&cl);
1965 }
1966
1967 };
1968
1969 class ShenandoahVerifyMatrixRegionClosure : public ShenandoahHeapRegionClosure {
1970 bool doHeapRegion(ShenandoahHeapRegion* r) {
1971 ShenandoahVerifyMatrixObjectClosure cl;
1972 ShenandoahHeap::heap()->marked_object_iterate(r, &cl);
1973 return false;
1974 }
1975 };
1976
1977 void ShenandoahHeap::verify_matrix() {
1978 OrderAccess::fence();
1979 ensure_parsability(false);
1980 ShenandoahVerifyMatrixRegionClosure cl;
1981 heap_region_iterate(&cl, true, true);
1982 }
1983
1984 void ShenandoahHeap::stop_concurrent_marking() {
1985 assert(concurrent_mark_in_progress(), "How else could we get here?");
1986 if (! cancelled_concgc()) {
1987 // If we needed to update refs, and concurrent marking has been cancelled,
1988 // we need to finish updating references.
1989 set_need_update_refs(false);
1990 swap_mark_bitmaps();
1991 }
1992 set_concurrent_mark_in_progress(false);
1993
1994 if (log_is_enabled(Trace, gc, region)) {
1995 ResourceMark rm;
1996 outputStream* out = Log(gc, region)::trace_stream();
1997 print_heap_regions(out);
1998 }
1999
2000 }
2001
2002 void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
2003 _concurrent_mark_in_progress = in_progress ? 1 : 0;
2004 JavaThread::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
2005 }
2006
2007 void ShenandoahHeap::set_evacuation_in_progress_concurrently(bool in_progress) {
2008 // Note: it is important to first release the _evacuation_in_progress flag here,
2009 // so that Java threads can get out of oom_during_evacuation() and reach a safepoint,
2010 // in case a VM task is pending.
2011 set_evacuation_in_progress(in_progress);
2012 MutexLocker mu(Threads_lock);
2013 JavaThread::set_evacuation_in_progress_all_threads(in_progress);
2014 }
2015
2016 void ShenandoahHeap::set_evacuation_in_progress_at_safepoint(bool in_progress) {
2017 assert(SafepointSynchronize::is_at_safepoint(), "Only call this at safepoint");
2018 set_evacuation_in_progress(in_progress);
2019 JavaThread::set_evacuation_in_progress_all_threads(in_progress);
2020 }
2021
2022 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2023 _evacuation_in_progress = in_progress ? 1 : 0;
2024 OrderAccess::fence();
2025 }
2026
2027 void ShenandoahHeap::verify_copy(oop p,oop c){
2028 assert(! oopDesc::unsafe_equals(p, oopDesc::bs()->read_barrier(p)), "forwarded correctly");
2029 assert(oopDesc::unsafe_equals(oopDesc::bs()->read_barrier(p), c), "verify pointer is correct");
2030 if (p->klass() != c->klass()) {
2031 print_heap_regions();
2032 }
2033 assert(p->klass() == c->klass(), "verify class p-size: "INT32_FORMAT" c-size: "INT32_FORMAT, p->size(), c->size());
2034 assert(p->size() == c->size(), "verify size");
2035 // Object may have been locked between copy and verification
2036 // assert(p->mark() == c->mark(), "verify mark");
2037 assert(oopDesc::unsafe_equals(c, oopDesc::bs()->read_barrier(c)), "verify only forwarded once");
2038 }
2039
2040 void ShenandoahHeap::oom_during_evacuation() {
2041 log_develop_trace(gc)("Out of memory during evacuation, cancel evacuation, schedule full GC by thread %d",
2042 Thread::current()->osthread()->thread_id());
2043
2044 // We ran out of memory during evacuation. Cancel evacuation, and schedule a full-GC.
2045 collector_policy()->set_should_clear_all_soft_refs(true);
2046 concurrent_thread()->try_set_full_gc();
2047 cancel_concgc(_oom_evacuation);
2048
2049 if ((! Thread::current()->is_GC_task_thread()) && (! Thread::current()->is_ConcurrentGC_thread())) {
2050 assert(! Threads_lock->owned_by_self()
2051 || SafepointSynchronize::is_at_safepoint(), "must not hold Threads_lock here");
2052 log_warning(gc)("OOM during evacuation. Let Java thread wait until evacuation finishes.");
2053 while (_evacuation_in_progress) { // wait.
2054 Thread::current()->_ParkEvent->park(1);
2055 }
2056 }
2057
2058 }
2059
2060 HeapWord* ShenandoahHeap::tlab_post_allocation_setup(HeapWord* obj) {
2061 // Initialize Brooks pointer for the next object
2062 HeapWord* result = obj + BrooksPointer::word_size();
2063 BrooksPointer::initialize(oop(result));
2064 return result;
2065 }
2066
2067 uint ShenandoahHeap::oop_extra_words() {
2068 return BrooksPointer::word_size();
2069 }
2070
2071 void ShenandoahHeap::grow_heap_by(size_t num_regions) {
2072 size_t base = _num_regions;
2073 ensure_new_regions(num_regions);
2074 for (size_t i = 0; i < num_regions; i++) {
2075 size_t new_region_index = i + base;
2076 HeapWord* start = _first_region_bottom + (ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize) * new_region_index;
2077 ShenandoahHeapRegion* new_region = new ShenandoahHeapRegion(this, start, ShenandoahHeapRegion::RegionSizeBytes / HeapWordSize, new_region_index);
2078
2079 if (log_is_enabled(Trace, gc, region)) {
2080 ResourceMark rm;
2081 outputStream* out = Log(gc, region)::trace_stream();
2082 out->print_cr("allocating new region at index: "SIZE_FORMAT, new_region_index);
2083 new_region->print_on(out);
2084 }
2085
2086 assert(_ordered_regions->active_regions() == new_region->region_number(), "must match");
2087 _ordered_regions->add_region(new_region);
2088 _in_cset_fast_test_base[new_region_index] = false; // Not in cset
2089 _next_top_at_mark_starts_base[new_region_index] = new_region->bottom();
2090 _complete_top_at_mark_starts_base[new_region_index] = new_region->bottom();
2091
2092 _free_regions->add_region(new_region);
2093 }
2094 }
2095
2096 void ShenandoahHeap::ensure_new_regions(size_t new_regions) {
2097
2098 size_t num_regions = _num_regions;
2099 size_t new_num_regions = num_regions + new_regions;
2100 assert(new_num_regions <= _max_regions, "we checked this earlier");
2101
2102 size_t expand_size = new_regions * ShenandoahHeapRegion::RegionSizeBytes;
2103 log_trace(gc, region)("expanding storage by "SIZE_FORMAT_HEX" bytes, for "SIZE_FORMAT" new regions", expand_size, new_regions);
2104 bool success = _storage.expand_by(expand_size, ShenandoahAlwaysPreTouch);
2105 assert(success, "should always be able to expand by requested size");
2106
2107 _num_regions = new_num_regions;
2108
2109 }
2110
2111 ShenandoahForwardedIsAliveClosure::ShenandoahForwardedIsAliveClosure() :
2112 _heap(ShenandoahHeap::heap_no_check()) {
2113 }
2114
2115 void ShenandoahForwardedIsAliveClosure::init(ShenandoahHeap* heap) {
2116 _heap = heap;
2117 }
2118
2119 bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) {
2120
2121 assert(_heap != NULL, "sanity");
2122 obj = ShenandoahBarrierSet::resolve_oop_static_not_null(obj);
2123 #ifdef ASSERT
2124 if (_heap->concurrent_mark_in_progress()) {
2125 assert(oopDesc::unsafe_equals(obj, ShenandoahBarrierSet::resolve_oop_static_not_null(obj)), "only query to-space");
2126 }
2127 #endif
2128 assert(!oopDesc::is_null(obj), "null");
2129 return _heap->is_marked_next(obj);
2130 }
2131
2132 void ShenandoahHeap::ref_processing_init() {
2133 MemRegion mr = reserved_region();
2134
2135 isAlive.init(ShenandoahHeap::heap());
2136 assert(_max_workers > 0, "Sanity");
2137
2138 _ref_processor =
2139 new ReferenceProcessor(mr, // span
2140 ParallelRefProcEnabled,
2141 // mt processing
2142 _max_workers,
2143 // degree of mt processing
2144 true,
2145 // mt discovery
2146 _max_workers,
2147 // degree of mt discovery
2148 false,
2149 // Reference discovery is not atomic
2150 &isAlive);
2151 }
2152
2153 #ifdef ASSERT
2154 void ShenandoahHeap::set_from_region_protection(bool protect) {
2155 for (uint i = 0; i < _num_regions; i++) {
2156 ShenandoahHeapRegion* region = _ordered_regions->get(i);
2157 if (region != NULL && in_collection_set(region)) {
2158 if (protect) {
2159 region->memProtectionOn();
2160 } else {
2161 region->memProtectionOff();
2162 }
2163 }
2164 }
2165 }
2166 #endif
2167
2168 size_t ShenandoahHeap::num_regions() {
2169 return _num_regions;
2170 }
2171
2172 size_t ShenandoahHeap::max_regions() {
2173 return _max_regions;
2174 }
2175
2176 GCTracer* ShenandoahHeap::tracer() {
2177 return shenandoahPolicy()->tracer();
2178 }
2179
2180 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2181 return _free_regions->used();
2182 }
2183
2184 void ShenandoahHeap::cancel_concgc(GCCause::Cause cause) {
2185 if (try_cancel_concgc()) {
2186 log_info(gc)("Cancelling concurrent GC: %s", GCCause::to_string(cause));
2187 _shenandoah_policy->report_concgc_cancelled();
2188 }
2189 }
2190
2191 void ShenandoahHeap::cancel_concgc(ShenandoahCancelCause cause) {
2192 if (try_cancel_concgc()) {
2193 log_info(gc)("Cancelling concurrent GC: %s", cancel_cause_to_string(cause));
2194 _shenandoah_policy->report_concgc_cancelled();
2195 }
2196 }
2197
2198 const char* ShenandoahHeap::cancel_cause_to_string(ShenandoahCancelCause cause) {
2199 switch (cause) {
2200 case _oom_evacuation:
2201 return "Out of memory for evacuation";
2202 case _vm_stop:
2203 return "Stopping VM";
2204 default:
2205 return "Unknown";
2206 }
2207 }
2208
2209 void ShenandoahHeap::clear_cancelled_concgc() {
2210 set_cancelled_concgc(false);
2211 }
2212
2213 uint ShenandoahHeap::max_workers() {
2214 return _max_workers;
2215 }
2216
2217 void ShenandoahHeap::stop() {
2218 // The shutdown sequence should be able to terminate when GC is running.
2219
2220 // Step 1. Notify control thread that we are in shutdown.
2221 // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown.
2222 // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below.
2223 _concurrent_gc_thread->prepare_for_graceful_shutdown();
2224
2225 // Step 2. Notify GC workers that we are cancelling GC.
2226 cancel_concgc(_vm_stop);
2227
2228 // Step 3. Wait until GC worker exits normally.
2229 _concurrent_gc_thread->stop();
2230 }
2231
2232 void ShenandoahHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) {
2233
2234 StringSymbolTableUnlinkTask shenandoah_unlink_task(is_alive, process_strings, process_symbols);
2235 workers()->run_task(&shenandoah_unlink_task);
2236
2237 // if (G1StringDedup::is_enabled()) {
2238 // G1StringDedup::unlink(is_alive);
2239 // }
2240 }
2241
2242 void ShenandoahHeap::set_need_update_refs(bool need_update_refs) {
2243 _need_update_refs = need_update_refs;
2244 }
2245
2246 //fixme this should be in heapregionset
2247 ShenandoahHeapRegion* ShenandoahHeap::next_compaction_region(const ShenandoahHeapRegion* r) {
2248 size_t region_idx = r->region_number() + 1;
2249 ShenandoahHeapRegion* next = _ordered_regions->get(region_idx);
2250 guarantee(next->region_number() == region_idx, "region number must match");
2251 while (next->is_humongous()) {
2252 region_idx = next->region_number() + 1;
2253 next = _ordered_regions->get(region_idx);
2254 guarantee(next->region_number() == region_idx, "region number must match");
2255 }
2256 return next;
2257 }
2258
2259 void ShenandoahHeap::set_region_in_collection_set(size_t region_index, bool b) {
2260 _in_cset_fast_test_base[region_index] = b;
2261 }
2262
2263 ShenandoahMonitoringSupport* ShenandoahHeap::monitoring_support() {
2264 return _monitoring_support;
2265 }
2266
2267 CMBitMap* ShenandoahHeap::complete_mark_bit_map() {
2268 return _complete_mark_bit_map;
2269 }
2270
2271 CMBitMap* ShenandoahHeap::next_mark_bit_map() {
2272 return _next_mark_bit_map;
2273 }
2274
2275 void ShenandoahHeap::add_free_region(ShenandoahHeapRegion* r) {
2276 _free_regions->add_region(r);
2277 }
2278
2279 void ShenandoahHeap::clear_free_regions() {
2280 _free_regions->clear();
2281 }
2282
2283 address ShenandoahHeap::in_cset_fast_test_addr() {
2284 return (address) (ShenandoahHeap::heap()->_in_cset_fast_test);
2285 }
2286
2287 address ShenandoahHeap::cancelled_concgc_addr() {
2288 return (address) &(ShenandoahHeap::heap()->_cancelled_concgc);
2289 }
2290
2291 void ShenandoahHeap::clear_cset_fast_test() {
2292 assert(_in_cset_fast_test_base != NULL, "sanity");
2293 memset(_in_cset_fast_test_base, false,
2294 _in_cset_fast_test_length * sizeof(bool));
2295 }
2296
2297 size_t ShenandoahHeap::conservative_max_heap_alignment() {
2298 return ShenandoahMaxRegionSize;
2299 }
2300
2301 size_t ShenandoahHeap::bytes_allocated_since_cm() {
2302 return _bytes_allocated_since_cm;
2303 }
2304
2305 void ShenandoahHeap::set_bytes_allocated_since_cm(size_t bytes) {
2306 _bytes_allocated_since_cm = bytes;
2307 }
2308
2309 size_t ShenandoahHeap::max_allocated_gc() {
2310 return _max_allocated_gc;
2311 }
2312
2313 void ShenandoahHeap::set_next_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
2314 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2315 _next_top_at_mark_starts[index] = addr;
2316 }
2317
2318 HeapWord* ShenandoahHeap::next_top_at_mark_start(HeapWord* region_base) {
2319 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2320 return _next_top_at_mark_starts[index];
2321 }
2322
2323 void ShenandoahHeap::set_complete_top_at_mark_start(HeapWord* region_base, HeapWord* addr) {
2324 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2325 _complete_top_at_mark_starts[index] = addr;
2326 }
2327
2328 HeapWord* ShenandoahHeap::complete_top_at_mark_start(HeapWord* region_base) {
2329 uintx index = ((uintx) region_base) >> ShenandoahHeapRegion::RegionSizeShift;
2330 return _complete_top_at_mark_starts[index];
2331 }
2332
2333 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2334 _full_gc_in_progress = in_progress;
2335 }
2336
2337 bool ShenandoahHeap::is_full_gc_in_progress() const {
2338 return _full_gc_in_progress;
2339 }
2340
2341 class NMethodOopInitializer : public OopClosure {
2342 private:
2343 ShenandoahHeap* _heap;
2344 public:
2345 NMethodOopInitializer() : _heap(ShenandoahHeap::heap()) {
2346 }
2347
2348 private:
2349 template <class T>
2350 inline void do_oop_work(T* p) {
2351 T o = oopDesc::load_heap_oop(p);
2352 if (! oopDesc::is_null(o)) {
2353 oop obj1 = oopDesc::decode_heap_oop_not_null(o);
2354 oop obj2 = oopDesc::bs()->write_barrier(obj1);
2355 if (! oopDesc::unsafe_equals(obj1, obj2)) {
2356 oopDesc::encode_store_heap_oop(p, obj2);
2357 }
2358 }
2359 }
2360
2361 public:
2362 void do_oop(oop* o) {
2363 do_oop_work(o);
2364 }
2365 void do_oop(narrowOop* o) {
2366 do_oop_work(o);
2367 }
2368 };
2369
2370 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2371 NMethodOopInitializer init;
2372 nm->oops_do(&init);
2373 nm->fix_oop_relocations();
2374 }
2375
2376 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2377 }
2378
2379 void ShenandoahHeap::pin_object(oop o) {
2380 heap_region_containing(o)->pin();
2381 }
2382
2383 void ShenandoahHeap::unpin_object(oop o) {
2384 heap_region_containing(o)->unpin();
2385 }
2386
2387
2388 GCTimer* ShenandoahHeap::gc_timer() const {
2389 return _gc_timer;
2390 }
2391
2392 class RecordAllRefsOopClosure: public ExtendedOopClosure {
2393 private:
2394 int _x;
2395 int *_matrix;
2396 int _num_regions;
2397 oop _p;
2398
2399 public:
2400 RecordAllRefsOopClosure(int *matrix, int x, size_t num_regions, oop p) :
2401 _matrix(matrix), _x(x), _num_regions(num_regions), _p(p) {}
2402
2403 template <class T>
2404 void do_oop_work(T* p) {
2405 oop o = oopDesc::load_decode_heap_oop(p);
2406 if (o != NULL) {
2407 if (ShenandoahHeap::heap()->is_in(o) && o->is_oop() ) {
2408 int y = ShenandoahHeap::heap()->heap_region_containing(o)->region_number();
2409 _matrix[_x * _num_regions + y]++;
2410 }
2411 }
2412 }
2413 void do_oop(oop* p) {
2414 do_oop_work(p);
2415 }
2416
2417 void do_oop(narrowOop* p) {
2418 do_oop_work(p);
2419 }
2420
2421 };
2422
2423 class RecordAllRefsObjectClosure : public ObjectClosure {
2424 int *_matrix;
2425 size_t _num_regions;
2426
2427 public:
2428 RecordAllRefsObjectClosure(int *matrix, size_t num_regions) :
2429 _matrix(matrix), _num_regions(num_regions) {}
2430
2431 void do_object(oop p) {
2432 if (ShenandoahHeap::heap()->is_in(p) && ShenandoahHeap::heap()->is_marked_next(p) && p->is_oop()) {
2433 int x = ShenandoahHeap::heap()->heap_region_containing(p)->region_number();
2434 RecordAllRefsOopClosure cl(_matrix, x, _num_regions, p);
2435 p->oop_iterate(&cl);
2436 }
2437 }
2438 };
2439 void ShenandoahHeap::calculate_matrix(int* connections) {
2440 log_develop_trace(gc)("calculating matrix");
2441 ensure_parsability(false);
2442 int num = num_regions();
2443
2444 for (int i = 0; i < num; i++) {
2445 for (int j = 0; j < num; j++) {
2446 connections[i * num + j] = 0;
2447 }
2448 }
2449
2450 RecordAllRefsOopClosure cl(connections, 0, num, NULL);
2451 roots_iterate(&cl);
2452
2453 RecordAllRefsObjectClosure cl2(connections, num);
2454 object_iterate(&cl2);
2455
2456 }
2457
2458 void ShenandoahHeap::print_matrix(int* connections) {
2459 int num = num_regions();
2460 int cs_regions = 0;
2461 int referenced = 0;
2462
2463 for (int i = 0; i < num; i++) {
2464 size_t liveData = ShenandoahHeap::heap()->regions()->get(i)->get_live_data_bytes();
2465
2466 int numReferencedRegions = 0;
2467 int numReferencedByRegions = 0;
2468
2469 for (int j = 0; j < num; j++) {
2470 if (connections[i * num + j] > 0)
2471 numReferencedRegions++;
2472
2473 if (connections [j * num + i] > 0)
2474 numReferencedByRegions++;
2475
2476 cs_regions++;
2477 referenced += numReferencedByRegions;
2478 }
2479
2480 if (ShenandoahHeap::heap()->regions()->get(i)->has_live()) {
2481 tty->print("Region %d is referenced by %d regions {",
2482 i, numReferencedByRegions);
2483 int col_count = 0;
2484 for (int j = 0; j < num; j++) {
2485 int foo = connections[j * num + i];
2486 if (foo > 0) {
2487 col_count++;
2488 if ((col_count % 10) == 0)
2489 tty->print("\n");
2490 tty->print("%d(%d), ", j,foo);
2491 }
2492 }
2493 tty->print("} \n");
2494 }
2495 }
2496
2497 double avg = (double)referenced / (double) cs_regions;
2498 tty->print("Average Number of regions scanned / region = %lf\n", avg);
2499 }
2500
2501 class ShenandoahCountGarbageClosure : public ShenandoahHeapRegionClosure {
2502 private:
2503 size_t _garbage;
2504 public:
2505 ShenandoahCountGarbageClosure() : _garbage(0) {
2506 }
2507
2508 bool doHeapRegion(ShenandoahHeapRegion* r) {
2509 if (! r->is_humongous() && ! r->is_pinned() && ! r->in_collection_set()) {
2510 _garbage += r->garbage();
2511 }
2512 return false;
2513 }
2514
2515 size_t garbage() {
2516 return _garbage;
2517 }
2518 };
2519
2520 size_t ShenandoahHeap::garbage() {
2521 ShenandoahCountGarbageClosure cl;
2522 heap_region_iterate(&cl);
2523 return cl.garbage();
2524 }
2525
2526 ShenandoahConnectionMatrix* ShenandoahHeap::connection_matrix() {
2527 return _connection_matrix;
2528 }
2529
2530 #ifdef ASSERT
2531 void ShenandoahHeap::assert_heaplock_owned_by_current_thread() {
2532 assert(_heap_lock == locked, "must be locked");
2533 assert(_heap_lock_owner == Thread::current(), "must be owned by current thread");
2534 }
2535
2536 void ShenandoahHeap::assert_heaplock_or_safepoint() {
2537 Thread* thr = Thread::current();
2538 assert((_heap_lock == locked && _heap_lock_owner == thr) ||
2539 (SafepointSynchronize::is_at_safepoint() && thr->is_VM_thread()),
2540 "must own heap lock or by VM thread at safepoint");
2541 }
2542
2543 #endif