1 /*
2 * Copyright (c) 2015, 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
25 #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP
26
27 #include "classfile/javaClasses.inline.hpp"
28 #include "gc/shared/markBitMap.inline.hpp"
29 #include "gc/shared/threadLocalAllocBuffer.inline.hpp"
30 #include "gc/shared/suspendibleThreadSet.hpp"
31 #include "gc/shenandoah/brooksPointer.inline.hpp"
32 #include "gc/shenandoah/shenandoahAsserts.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
34 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
35 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
36 #include "gc/shenandoah/shenandoahConnectionMatrix.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeap.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
39 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
40 #include "gc/shenandoah/shenandoahControlThread.hpp"
41 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
42 #include "gc/shenandoah/shenandoahUtils.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/atomic.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/prefetch.hpp"
47 #include "runtime/prefetch.inline.hpp"
48 #include "runtime/thread.hpp"
49 #include "utilities/copy.hpp"
50
51 template <class T>
52 void ShenandoahUpdateRefsClosure::do_oop_work(T* p) {
53 T o = RawAccess<>::oop_load(p);
54 if (!CompressedOops::is_null(o)) {
55 oop obj = CompressedOops::decode_not_null(o);
56 _heap->update_with_forwarded_not_null(p, obj);
57 }
58 }
59
60 void ShenandoahUpdateRefsClosure::do_oop(oop* p) { do_oop_work(p); }
61 void ShenandoahUpdateRefsClosure::do_oop(narrowOop* p) { do_oop_work(p); }
62
63 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
64 size_t new_index = Atomic::add((size_t) 1, &_index);
65 // get_region() provides the bounds-check and returns NULL on OOB.
66 return _heap->get_region(new_index - 1);
67 }
68
69 /*
70 * Marks the object. Returns true if the object has not been marked before and has
71 * been marked by this thread. Returns false if the object has already been marked,
72 * or if a competing thread succeeded in marking this object.
73 */
74 inline bool ShenandoahHeap::mark_next(oop obj) const {
75 shenandoah_assert_not_forwarded(NULL, obj);
76 HeapWord* addr = (HeapWord*) obj;
77 return (! allocated_after_next_mark_start(addr)) && _next_mark_bit_map->parMark(addr);
78 }
79
80 inline bool ShenandoahHeap::is_marked_next(oop obj) const {
81 HeapWord* addr = (HeapWord*) obj;
82 return allocated_after_next_mark_start(addr) || _next_mark_bit_map->isMarked(addr);
83 }
84
85 inline bool ShenandoahHeap::is_marked_complete(oop obj) const {
86 HeapWord* addr = (HeapWord*) obj;
87 return allocated_after_complete_mark_start(addr) || _complete_mark_bit_map->isMarked(addr);
88 }
89
90 inline bool ShenandoahHeap::has_forwarded_objects() const {
91 return _gc_state.is_set(HAS_FORWARDED);
92 }
93
94 inline size_t ShenandoahHeap::heap_region_index_containing(const void* addr) const {
95 uintptr_t region_start = ((uintptr_t) addr);
96 uintptr_t index = (region_start - (uintptr_t) base()) >> ShenandoahHeapRegion::region_size_bytes_shift();
97 assert(index < num_regions(), "Region index is in bounds: " PTR_FORMAT, p2i(addr));
98 return index;
99 }
100
101 inline ShenandoahHeapRegion* const ShenandoahHeap::heap_region_containing(const void* addr) const {
102 size_t index = heap_region_index_containing(addr);
103 ShenandoahHeapRegion* const result = get_region(index);
104 assert(addr >= result->bottom() && addr < result->end(), "Heap region contains the address: " PTR_FORMAT, p2i(addr));
105 return result;
106 }
107
108 template <class T>
109 inline oop ShenandoahHeap::update_with_forwarded_not_null(T* p, oop obj) {
110 if (in_collection_set(obj)) {
111 shenandoah_assert_forwarded_except(p, obj, is_full_gc_in_progress() || cancelled_concgc());
112 obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
113 RawAccess<OOP_NOT_NULL>::oop_store(p, obj);
114 }
115 #ifdef ASSERT
116 else {
117 shenandoah_assert_not_forwarded(p, obj);
118 }
119 #endif
120 return obj;
121 }
122
123 template <class T>
124 inline oop ShenandoahHeap::maybe_update_with_forwarded(T* p) {
125 T o = RawAccess<>::oop_load(p);
126 if (!CompressedOops::is_null(o)) {
127 oop obj = CompressedOops::decode_not_null(o);
128 return maybe_update_with_forwarded_not_null(p, obj);
129 } else {
130 return NULL;
131 }
132 }
133
134 template <class T>
135 inline oop ShenandoahHeap::evac_update_with_forwarded(T* p) {
136 T o = RawAccess<>::oop_load(p);
137 if (!CompressedOops::is_null(o)) {
138 oop heap_oop = CompressedOops::decode_not_null(o);
139 if (in_collection_set(heap_oop)) {
140 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
141 if (oopDesc::unsafe_equals(forwarded_oop, heap_oop)) {
142 forwarded_oop = evacuate_object(heap_oop, Thread::current());
143 }
144 oop prev = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop);
145 if (oopDesc::unsafe_equals(prev, heap_oop)) {
146 return forwarded_oop;
147 } else {
148 return NULL;
149 }
150 }
151 return heap_oop;
152 } else {
153 return NULL;
154 }
155 }
156
157 inline oop ShenandoahHeap::atomic_compare_exchange_oop(oop n, oop* addr, oop c) {
158 return (oop) Atomic::cmpxchg(n, addr, c);
159 }
160
161 inline oop ShenandoahHeap::atomic_compare_exchange_oop(oop n, narrowOop* addr, oop c) {
162 narrowOop cmp = CompressedOops::encode(c);
163 narrowOop val = CompressedOops::encode(n);
164 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, cmp));
165 }
166
167 template <class T>
168 inline oop ShenandoahHeap::maybe_update_with_forwarded_not_null(T* p, oop heap_oop) {
169 shenandoah_assert_not_in_cset_loc_except(p, !is_in(p) || is_full_gc_in_progress());
170 shenandoah_assert_correct(p, heap_oop);
171
172 if (in_collection_set(heap_oop)) {
173 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
174 if (oopDesc::unsafe_equals(forwarded_oop, heap_oop)) {
175 // E.g. during evacuation.
176 return forwarded_oop;
177 }
178
179 shenandoah_assert_forwarded_except(p, heap_oop, is_full_gc_in_progress());
180 shenandoah_assert_not_in_cset_except(p, forwarded_oop, cancelled_concgc());
181
182 log_develop_trace(gc)("Updating old ref: "PTR_FORMAT" pointing to "PTR_FORMAT" to new ref: "PTR_FORMAT,
183 p2i(p), p2i(heap_oop), p2i(forwarded_oop));
184
185 // If this fails, another thread wrote to p before us, it will be logged in SATB and the
186 // reference be updated later.
187 oop result = atomic_compare_exchange_oop(forwarded_oop, p, heap_oop);
188
189 if (oopDesc::unsafe_equals(result, heap_oop)) { // CAS successful.
190 return forwarded_oop;
191 } else {
192 // Note: we used to assert the following here. This doesn't work because sometimes, during
193 // marking/updating-refs, it can happen that a Java thread beats us with an arraycopy,
194 // which first copies the array, which potentially contains from-space refs, and only afterwards
195 // updates all from-space refs to to-space refs, which leaves a short window where the new array
196 // elements can be from-space.
197 // assert(CompressedOops::is_null(result) ||
198 // oopDesc::unsafe_equals(result, ShenandoahBarrierSet::resolve_oop_static_not_null(result)),
199 // "expect not forwarded");
200 return NULL;
201 }
202 } else {
203 shenandoah_assert_not_forwarded(p, heap_oop);
204 return heap_oop;
205 }
206 }
207
208 inline bool ShenandoahHeap::cancelled_concgc() const {
209 return _cancelled_concgc.get() == CANCELLED;
210 }
211
212 inline bool ShenandoahHeap::check_cancelled_concgc_and_yield(bool sts_active) {
213 if (! (sts_active && ShenandoahSuspendibleWorkers)) {
214 return cancelled_concgc();
215 }
216
217 jbyte prev = _cancelled_concgc.cmpxchg(NOT_CANCELLED, CANCELLABLE);
218 if (prev == CANCELLABLE || prev == NOT_CANCELLED) {
219
220 if (SuspendibleThreadSet::should_yield()) {
221 SuspendibleThreadSet::yield();
222 }
223
224 // Back to CANCELLABLE. The thread that poked NOT_CANCELLED first gets
225 // to restore to CANCELLABLE.
226 if (prev == CANCELLABLE) {
227 _cancelled_concgc.set(CANCELLABLE);
228 }
229 return false;
230 } else {
231 return true;
232 }
233 }
234
235 inline bool ShenandoahHeap::try_cancel_concgc() {
236 while (true) {
237 jbyte prev = _cancelled_concgc.cmpxchg(CANCELLED, CANCELLABLE);
238 if (prev == CANCELLABLE) return true;
239 else if (prev == CANCELLED) return false;
240 assert(ShenandoahSuspendibleWorkers, "should not get here when not using suspendible workers");
241 assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED");
242 {
243 // We need to provide a safepoint here, otherwise we might
244 // spin forever if a SP is pending.
245 ThreadBlockInVM sp(JavaThread::current());
246 SpinPause();
247 }
248 }
249 }
250
251 inline void ShenandoahHeap::clear_cancelled_concgc() {
252 _cancelled_concgc.set(CANCELLABLE);
253 _oom_evac_handler.clear();
254 }
255
256 inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) {
257 PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
258 if (gclab == NULL) {
259 assert(!thread->is_Java_thread() && !thread->is_Worker_thread(),
260 "Performance: thread should have GCLAB: %s", thread->name());
261 // No GCLABs in this thread, fallback to shared allocation
262 return NULL;
263 }
264 HeapWord* obj = gclab->allocate(size);
265 if (obj != NULL) {
266 return obj;
267 }
268 // Otherwise...
269 return allocate_from_gclab_slow(thread, size);
270 }
271
272 inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
273 if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) {
274 // This thread went through the OOM during evac protocol and it is safe to return
275 // the forward pointer. It must not attempt to evacuate any more.
276 return ShenandoahBarrierSet::resolve_forwarded(p);
277 }
278
279 size_t size_no_fwdptr = (size_t) p->size();
280 size_t size_with_fwdptr = size_no_fwdptr + BrooksPointer::word_size();
281
282 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects");
283
284 bool alloc_from_gclab = true;
285 HeapWord* filler;
286 #ifdef ASSERT
287
288 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in ShenandoahOOMDuringEvacHandler");
289
290 if (ShenandoahOOMDuringEvacALot &&
291 (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
292 filler = NULL;
293 } else {
294 #endif
295 filler = allocate_from_gclab(thread, size_with_fwdptr);
296 if (filler == NULL) {
297 filler = allocate_memory(size_with_fwdptr, _alloc_shared_gc);
298 alloc_from_gclab = false;
299 }
300 #ifdef ASSERT
301 }
302 #endif
303
304 if (filler == NULL) {
305 control_thread()->handle_alloc_failure_evac(size_with_fwdptr);
306
307 _oom_evac_handler.handle_out_of_memory_during_evacuation();
308
309 return ShenandoahBarrierSet::resolve_forwarded(p);
310 }
311
312 // Copy the object and initialize its forwarding ptr:
313 HeapWord* copy = filler + BrooksPointer::word_size();
314 oop copy_val = oop(copy);
315
316 Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr);
317 BrooksPointer::initialize(oop(copy));
318
319 log_develop_trace(gc, compaction)("Copy object: " PTR_FORMAT " -> " PTR_FORMAT,
320 p2i(p), p2i(copy));
321
322 // Try to install the new forwarding pointer.
323 oop result = BrooksPointer::try_update_forwardee(p, copy_val);
324
325 if (oopDesc::unsafe_equals(result, p)) {
326 // Successfully evacuated. Our copy is now the public one!
327 log_develop_trace(gc, compaction)("Copy object: " PTR_FORMAT " -> " PTR_FORMAT " succeeded",
328 p2i(p), p2i(copy));
329
330
331 #ifdef ASSERT
332 assert(oopDesc::is_oop(copy_val), "expect oop");
333 assert(p->klass() == copy_val->klass(), "Should have the same class p: "PTR_FORMAT", copy: "PTR_FORMAT,
334 p2i(p), p2i(copy));
335 #endif
336 return copy_val;
337 } else {
338 // Failed to evacuate. We need to deal with the object that is left behind. Since this
339 // new allocation is certainly after TAMS, it will be considered live in the next cycle.
340 // But if it happens to contain references to evacuated regions, those references would
341 // not get updated for this stale copy during this cycle, and we will crash while scanning
342 // it the next cycle.
343 //
344 // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next
345 // object will overwrite this stale copy, or the filler object on LAB retirement will
346 // do this. For non-GCLAB allocations, we have no way to retract the allocation, and
347 // have to explicitly overwrite the copy with the filler object. With that overwrite,
348 // we have to keep the fwdptr initialized and pointing to our (stale) copy.
349 if (alloc_from_gclab) {
350 ShenandoahThreadLocalData::gclab(thread)->undo_allocation(filler, size_with_fwdptr);
351 } else {
352 fill_with_object(filler, size_with_fwdptr);
353 }
354 log_develop_trace(gc, compaction)("Copy object: " PTR_FORMAT " -> " PTR_FORMAT " failed, use other: " PTR_FORMAT,
355 p2i(p), p2i(copy), p2i(result));
356 return result;
357 }
358 }
359
360 inline bool ShenandoahHeap::requires_marking(const void* entry) const {
361 return ! is_marked_next(oop(entry));
362 }
363
364 bool ShenandoahHeap::region_in_collection_set(size_t region_index) const {
365 assert(collection_set() != NULL, "Sanity");
366 return collection_set()->is_in(region_index);
367 }
368
369 bool ShenandoahHeap::in_collection_set(ShenandoahHeapRegion* r) const {
370 return region_in_collection_set(r->region_number());
371 }
372
373 template <class T>
374 inline bool ShenandoahHeap::in_collection_set(T p) const {
375 HeapWord* obj = (HeapWord*) p;
376 assert(collection_set() != NULL, "Sanity");
377 assert(is_in(obj), "should be in heap");
378
379 return collection_set()->is_in(obj);
380 }
381
382 inline bool ShenandoahHeap::is_stable() const {
383 return _gc_state.is_clear();
384 }
385
386 inline bool ShenandoahHeap::is_idle() const {
387 return _gc_state.is_unset(MARKING | EVACUATION | UPDATEREFS | TRAVERSAL);
388 }
389
390 inline bool ShenandoahHeap::is_concurrent_mark_in_progress() const {
391 return _gc_state.is_set(MARKING);
392 }
393
394 inline bool ShenandoahHeap::is_concurrent_traversal_in_progress() const {
395 return _gc_state.is_set(TRAVERSAL);
396 }
397
398 inline bool ShenandoahHeap::is_evacuation_in_progress() const {
399 return _gc_state.is_set(EVACUATION);
400 }
401
402 inline bool ShenandoahHeap::is_gc_in_progress_mask(uint mask) const {
403 return _gc_state.is_set(mask);
404 }
405
406 inline bool ShenandoahHeap::is_degenerated_gc_in_progress() const {
407 return _degenerated_gc_in_progress.is_set();
408 }
409
410 inline bool ShenandoahHeap::is_full_gc_in_progress() const {
411 return _full_gc_in_progress.is_set();
412 }
413
414 inline bool ShenandoahHeap::is_full_gc_move_in_progress() const {
415 return _full_gc_move_in_progress.is_set();
416 }
417
418 inline bool ShenandoahHeap::is_update_refs_in_progress() const {
419 return _gc_state.is_set(UPDATEREFS);
420 }
421
422 inline bool ShenandoahHeap::allocated_after_next_mark_start(HeapWord* addr) const {
423 uintx index = ((uintx) addr) >> ShenandoahHeapRegion::region_size_bytes_shift();
424 HeapWord* top_at_mark_start = _next_top_at_mark_starts[index];
425 bool alloc_after_mark_start = addr >= top_at_mark_start;
426 return alloc_after_mark_start;
427 }
428
429 inline bool ShenandoahHeap::allocated_after_complete_mark_start(HeapWord* addr) const {
430 uintx index = ((uintx) addr) >> ShenandoahHeapRegion::region_size_bytes_shift();
431 HeapWord* top_at_mark_start = _complete_top_at_mark_starts[index];
432 bool alloc_after_mark_start = addr >= top_at_mark_start;
433 return alloc_after_mark_start;
434 }
435
436 template<class T>
437 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) {
438 marked_object_iterate(region, cl, region->top());
439 }
440
441 template<class T>
442 inline void ShenandoahHeap::marked_object_safe_iterate(ShenandoahHeapRegion* region, T* cl) {
443 marked_object_iterate(region, cl, region->concurrent_iteration_safe_limit());
444 }
445
446 template<class T>
447 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) {
448 assert(BrooksPointer::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj");
449 assert(! region->is_humongous_continuation(), "no humongous continuation regions here");
450
451 MarkBitMap* mark_bit_map = _complete_mark_bit_map;
452 HeapWord* tams = complete_top_at_mark_start(region->bottom());
453
454 size_t skip_bitmap_delta = BrooksPointer::word_size() + 1;
455 size_t skip_objsize_delta = BrooksPointer::word_size() /* + actual obj.size() below */;
456 HeapWord* start = region->bottom() + BrooksPointer::word_size();
457 HeapWord* end = MIN2(tams + BrooksPointer::word_size(), region->end());
458
459 // Step 1. Scan below the TAMS based on bitmap data.
460 HeapWord* limit_bitmap = MIN2(limit, tams);
461
462 // Try to scan the initial candidate. If the candidate is above the TAMS, it would
463 // fail the subsequent "< limit_bitmap" checks, and fall through to Step 2.
464 HeapWord* cb = mark_bit_map->getNextMarkedWordAddress(start, end);
465
466 intx dist = ShenandoahMarkScanPrefetch;
467 if (dist > 0) {
468 // Batched scan that prefetches the oop data, anticipating the access to
469 // either header, oop field, or forwarding pointer. Not that we cannot
470 // touch anything in oop, while it still being prefetched to get enough
471 // time for prefetch to work. This is why we try to scan the bitmap linearly,
472 // disregarding the object size. However, since we know forwarding pointer
473 // preceeds the object, we can skip over it. Once we cannot trust the bitmap,
474 // there is no point for prefetching the oop contents, as oop->size() will
475 // touch it prematurely.
476
477 // No variable-length arrays in standard C++, have enough slots to fit
478 // the prefetch distance.
479 static const int SLOT_COUNT = 256;
480 guarantee(dist <= SLOT_COUNT, "adjust slot count");
481 HeapWord* slots[SLOT_COUNT];
482
483 int avail;
484 do {
485 avail = 0;
486 for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) {
487 Prefetch::read(cb, BrooksPointer::byte_offset());
488 slots[avail++] = cb;
489 cb += skip_bitmap_delta;
490 if (cb < limit_bitmap) {
491 cb = mark_bit_map->getNextMarkedWordAddress(cb, limit_bitmap);
492 }
493 }
494
495 for (int c = 0; c < avail; c++) {
496 assert (slots[c] < tams, "only objects below TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(slots[c]), p2i(tams));
497 assert (slots[c] < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(slots[c]), p2i(limit));
498 oop obj = oop(slots[c]);
499 do_object_marked_complete(cl, obj);
500 }
501 } while (avail > 0);
502 } else {
503 while (cb < limit_bitmap) {
504 assert (cb < tams, "only objects below TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cb), p2i(tams));
505 assert (cb < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cb), p2i(limit));
506 oop obj = oop(cb);
507 do_object_marked_complete(cl, obj);
508 cb += skip_bitmap_delta;
509 if (cb < limit_bitmap) {
510 cb = mark_bit_map->getNextMarkedWordAddress(cb, limit_bitmap);
511 }
512 }
513 }
514
515 // Step 2. Accurate size-based traversal, happens past the TAMS.
516 // This restarts the scan at TAMS, which makes sure we traverse all objects,
517 // regardless of what happened at Step 1.
518 HeapWord* cs = tams + BrooksPointer::word_size();
519 while (cs < limit) {
520 assert (cs > tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams));
521 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit));
522 oop obj = oop(cs);
523 int size = obj->size();
524 do_object_marked_complete(cl, obj);
525 cs += size + skip_objsize_delta;
526 }
527 }
528
529 template<class T>
530 inline void ShenandoahHeap::do_object_marked_complete(T* cl, oop obj) {
531 assert(oopDesc::is_oop(obj), "sanity");
532 assert(is_marked_complete(obj), "object expected to be marked");
533 cl->do_object(obj);
534 }
535
536 template <class T>
537 class ShenandoahObjectToOopClosure : public ObjectClosure {
538 T* _cl;
539 public:
540 ShenandoahObjectToOopClosure(T* cl) : _cl(cl) {}
541
542 void do_object(oop obj) {
543 obj->oop_iterate(_cl);
544 }
545 };
546
547 template <class T>
548 class ShenandoahObjectToOopBoundedClosure : public ObjectClosure {
549 T* _cl;
550 MemRegion _bounds;
551 public:
552 ShenandoahObjectToOopBoundedClosure(T* cl, HeapWord* bottom, HeapWord* top) :
553 _cl(cl), _bounds(bottom, top) {}
554
555 void do_object(oop obj) {
556 obj->oop_iterate(_cl, _bounds);
557 }
558 };
559
560 template<class T>
561 inline void ShenandoahHeap::marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* top) {
562 if (region->is_humongous()) {
563 HeapWord* bottom = region->bottom();
564 if (top > bottom) {
565 region = region->humongous_start_region();
566 ShenandoahObjectToOopBoundedClosure<T> objs(cl, bottom, top);
567 marked_object_iterate(region, &objs);
568 }
569 } else {
570 ShenandoahObjectToOopClosure<T> objs(cl);
571 marked_object_iterate(region, &objs, top);
572 }
573 }
574
575 template<class T>
576 inline void ShenandoahHeap::marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl) {
577 marked_object_oop_iterate(region, cl, region->top());
578 }
579
580 template<class T>
581 inline void ShenandoahHeap::marked_object_oop_safe_iterate(ShenandoahHeapRegion* region, T* cl) {
582 marked_object_oop_iterate(region, cl, region->concurrent_iteration_safe_limit());
583 }
584
585 inline ShenandoahHeapRegion* const ShenandoahHeap::get_region(size_t region_idx) const {
586 if (region_idx >= _num_regions) {
587 return NULL;
588 } else {
589 return _regions[region_idx];
590 }
591 }
592
593 #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP