32 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
35 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
36 #include "gc/shenandoah/shenandoahHeap.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
39 #include "gc/shenandoah/shenandoahControlThread.hpp"
40 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
41 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
42 #include "oops/compressedOops.inline.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/atomic.hpp"
45 #include "runtime/prefetch.inline.hpp"
46 #include "runtime/thread.hpp"
47 #include "utilities/copy.hpp"
48 #include "utilities/globalDefinitions.hpp"
49
50
51 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
52 size_t new_index = Atomic::add((size_t) 1, &_index);
53 // get_region() provides the bounds-check and returns NULL on OOB.
54 return _heap->get_region(new_index - 1);
55 }
56
57 inline bool ShenandoahHeap::has_forwarded_objects() const {
58 return _gc_state.is_set(HAS_FORWARDED);
59 }
60
61 inline WorkGang* ShenandoahHeap::workers() const {
62 return _workers;
63 }
64
65 inline WorkGang* ShenandoahHeap::get_safepoint_workers() {
66 return _safepoint_workers;
67 }
68
69 inline size_t ShenandoahHeap::heap_region_index_containing(const void* addr) const {
70 uintptr_t region_start = ((uintptr_t) addr);
71 uintptr_t index = (region_start - (uintptr_t) base()) >> ShenandoahHeapRegion::region_size_bytes_shift();
72 assert(index < num_regions(), "Region index is in bounds: " PTR_FORMAT, p2i(addr));
114 if (in_collection_set(heap_oop)) {
115 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
116 if (forwarded_oop == heap_oop) {
117 forwarded_oop = evacuate_object(heap_oop, Thread::current());
118 }
119 oop prev = cas_oop(forwarded_oop, p, heap_oop);
120 if (prev == heap_oop) {
121 return forwarded_oop;
122 } else {
123 return NULL;
124 }
125 }
126 return heap_oop;
127 } else {
128 return NULL;
129 }
130 }
131
132 inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) {
133 assert(is_aligned(addr, HeapWordSize), "Address should be aligned: " PTR_FORMAT, p2i(addr));
134 return (oop) Atomic::cmpxchg(n, addr, c);
135 }
136
137 inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, narrowOop c) {
138 assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr));
139 narrowOop val = CompressedOops::encode(n);
140 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, c));
141 }
142
143 inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, oop c) {
144 assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr));
145 narrowOop cmp = CompressedOops::encode(c);
146 narrowOop val = CompressedOops::encode(n);
147 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(val, addr, cmp));
148 }
149
150 template <class T>
151 inline oop ShenandoahHeap::maybe_update_with_forwarded_not_null(T* p, oop heap_oop) {
152 shenandoah_assert_not_in_cset_loc_except(p, !is_in(p) || is_full_gc_in_progress() || is_degenerated_gc_in_progress());
153 shenandoah_assert_correct(p, heap_oop);
154
155 if (in_collection_set(heap_oop)) {
156 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
157 if (forwarded_oop == heap_oop) {
158 // E.g. during evacuation.
159 return forwarded_oop;
160 }
161
162 shenandoah_assert_forwarded_except(p, heap_oop, is_full_gc_in_progress() || is_degenerated_gc_in_progress());
163 shenandoah_assert_not_forwarded(p, forwarded_oop);
164 shenandoah_assert_not_in_cset_except(p, forwarded_oop, cancelled_gc());
165
166 // If this fails, another thread wrote to p before us, it will be logged in SATB and the
167 // reference be updated later.
|
32 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
33 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.inline.hpp"
35 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
36 #include "gc/shenandoah/shenandoahHeap.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
39 #include "gc/shenandoah/shenandoahControlThread.hpp"
40 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
41 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
42 #include "oops/compressedOops.inline.hpp"
43 #include "oops/oop.inline.hpp"
44 #include "runtime/atomic.hpp"
45 #include "runtime/prefetch.inline.hpp"
46 #include "runtime/thread.hpp"
47 #include "utilities/copy.hpp"
48 #include "utilities/globalDefinitions.hpp"
49
50
51 inline ShenandoahHeapRegion* ShenandoahRegionIterator::next() {
52 size_t new_index = Atomic::add(&_index, (size_t) 1);
53 // get_region() provides the bounds-check and returns NULL on OOB.
54 return _heap->get_region(new_index - 1);
55 }
56
57 inline bool ShenandoahHeap::has_forwarded_objects() const {
58 return _gc_state.is_set(HAS_FORWARDED);
59 }
60
61 inline WorkGang* ShenandoahHeap::workers() const {
62 return _workers;
63 }
64
65 inline WorkGang* ShenandoahHeap::get_safepoint_workers() {
66 return _safepoint_workers;
67 }
68
69 inline size_t ShenandoahHeap::heap_region_index_containing(const void* addr) const {
70 uintptr_t region_start = ((uintptr_t) addr);
71 uintptr_t index = (region_start - (uintptr_t) base()) >> ShenandoahHeapRegion::region_size_bytes_shift();
72 assert(index < num_regions(), "Region index is in bounds: " PTR_FORMAT, p2i(addr));
114 if (in_collection_set(heap_oop)) {
115 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
116 if (forwarded_oop == heap_oop) {
117 forwarded_oop = evacuate_object(heap_oop, Thread::current());
118 }
119 oop prev = cas_oop(forwarded_oop, p, heap_oop);
120 if (prev == heap_oop) {
121 return forwarded_oop;
122 } else {
123 return NULL;
124 }
125 }
126 return heap_oop;
127 } else {
128 return NULL;
129 }
130 }
131
132 inline oop ShenandoahHeap::cas_oop(oop n, oop* addr, oop c) {
133 assert(is_aligned(addr, HeapWordSize), "Address should be aligned: " PTR_FORMAT, p2i(addr));
134 return (oop) Atomic::cmpxchg(addr, c, n);
135 }
136
137 inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, narrowOop c) {
138 assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr));
139 narrowOop val = CompressedOops::encode(n);
140 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(addr, c, val));
141 }
142
143 inline oop ShenandoahHeap::cas_oop(oop n, narrowOop* addr, oop c) {
144 assert(is_aligned(addr, sizeof(narrowOop)), "Address should be aligned: " PTR_FORMAT, p2i(addr));
145 narrowOop cmp = CompressedOops::encode(c);
146 narrowOop val = CompressedOops::encode(n);
147 return CompressedOops::decode((narrowOop) Atomic::cmpxchg(addr, cmp, val));
148 }
149
150 template <class T>
151 inline oop ShenandoahHeap::maybe_update_with_forwarded_not_null(T* p, oop heap_oop) {
152 shenandoah_assert_not_in_cset_loc_except(p, !is_in(p) || is_full_gc_in_progress() || is_degenerated_gc_in_progress());
153 shenandoah_assert_correct(p, heap_oop);
154
155 if (in_collection_set(heap_oop)) {
156 oop forwarded_oop = ShenandoahBarrierSet::resolve_forwarded_not_null(heap_oop);
157 if (forwarded_oop == heap_oop) {
158 // E.g. during evacuation.
159 return forwarded_oop;
160 }
161
162 shenandoah_assert_forwarded_except(p, heap_oop, is_full_gc_in_progress() || is_degenerated_gc_in_progress());
163 shenandoah_assert_not_forwarded(p, forwarded_oop);
164 shenandoah_assert_not_in_cset_except(p, forwarded_oop, cancelled_gc());
165
166 // If this fails, another thread wrote to p before us, it will be logged in SATB and the
167 // reference be updated later.
|