1 /*
2 * Copyright (c) 2015, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #ifndef SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_INLINE_HPP
25 #define SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_INLINE_HPP
26
27 #include "gc/shenandoah/brooksPointer.hpp"
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
29 #include "runtime/atomic.hpp"
30
31 inline HeapWord** BrooksPointer::brooks_ptr_addr(oop obj) {
32 return (HeapWord**)((HeapWord*) obj + word_offset());
33 }
34
35 inline void BrooksPointer::initialize(oop obj) {
36 #ifdef ASSERT
37 log_develop_trace(gc)("Init forwardee for "PTR_FORMAT" to self", p2i(obj));
38
39 assert(UseShenandoahGC, "must only be called when Shenandoah is used.");
40 assert(obj != NULL, "oop is not NULL");
41 assert(ShenandoahHeap::heap()->is_in(obj), "oop must point to a heap address");
42 #endif
43
44 *brooks_ptr_addr(obj) = (HeapWord*) obj;
45 }
46
47 inline void BrooksPointer::set_forwardee(oop holder, oop update) {
48 #ifdef ASSERT
49 log_develop_trace(gc)("Setting forwardee to "PTR_FORMAT" = "PTR_FORMAT, p2i(holder), p2i(update));
50
51 assert(UseShenandoahGC, "must only be called when Shenandoah is used.");
52 assert(holder != NULL, "holder should not be NULL");
53 assert(update != NULL, "update should not be NULL");
54 assert(ShenandoahHeap::heap()->is_in(holder), "holder must point to a heap address");
55 assert(ShenandoahHeap::heap()->is_in(update), "update must point to a heap address");
56
57 assert (holder->klass() == update->klass(), "klasses should match");
58
59 assert(!oopDesc::unsafe_equals(holder, update), "forwarding should make progress");
60 assert(ShenandoahHeap::heap()->heap_region_containing(holder) !=
61 ShenandoahHeap::heap()->heap_region_containing(update), "forwarding should be across regions");
62 assert( ShenandoahHeap::heap()->in_cset_fast_test((HeapWord*) holder), "holder should be in collection set");
63 assert(!ShenandoahHeap::heap()->in_cset_fast_test((HeapWord*) update), "update should not be in collection set");
64 #endif
65
66 *brooks_ptr_addr(holder) = (HeapWord*) update;
67 }
68
69 inline void BrooksPointer::set_raw(oop holder, HeapWord* update) {
70 log_develop_trace(gc)("Setting RAW forwardee for "PTR_FORMAT" = "PTR_FORMAT, p2i(holder), p2i(update));
71 assert(UseShenandoahGC, "must only be called when Shenandoah is used.");
72 *brooks_ptr_addr(holder) = update;
73 }
74
75 inline HeapWord* BrooksPointer::get_raw(oop holder) {
76 assert(UseShenandoahGC, "must only be called when Shenandoah is used.");
77 HeapWord* res = *brooks_ptr_addr(holder);
78 log_develop_trace(gc)("Getting RAW forwardee for "PTR_FORMAT" = "PTR_FORMAT, p2i(holder), p2i(res));
79 return res;
80 }
81
82 inline oop BrooksPointer::forwardee(oop obj) {
83 #ifdef ASSERT
84 assert(UseShenandoahGC, "must only be called when Shenandoah is used.");
85 assert(Universe::heap()->is_in(obj), "We shouldn't be calling this on objects not in the heap: "PTR_FORMAT, p2i(obj));
86
87 oop forwardee;
88 if (ShenandoahVerifyReadsToFromSpace) {
89 ShenandoahHeap* heap = (ShenandoahHeap *) Universe::heap();
90 ShenandoahHeapRegion* region = heap->heap_region_containing(obj);
91 {
92 region->memProtectionOff();
93 forwardee = oop(*brooks_ptr_addr(obj));
94 region->memProtectionOn();
95 }
96 } else {
97 forwardee = oop(*brooks_ptr_addr(obj));
98 }
99
100 assert(forwardee != NULL, "forwardee is not NULL");
101 assert(ShenandoahHeap::heap()->is_in(forwardee), "forwardee must point to a heap address");
102 assert((oopDesc::unsafe_equals(forwardee, obj) ||
103 (ShenandoahHeap::heap()->heap_region_containing(forwardee) !=
104 ShenandoahHeap::heap()->heap_region_containing(obj))), "forwardee should be self, or another region");
105
106 if (!oopDesc::unsafe_equals(obj, forwardee)) {
107 oop second_forwardee = oop(*brooks_ptr_addr(forwardee));
108 if (!oopDesc::unsafe_equals(forwardee, second_forwardee)) {
109 // We should never be forwarded more than once.
110 fatal("Multiple forwardings: "PTR_FORMAT" -> "PTR_FORMAT" -> "PTR_FORMAT, p2i(obj), p2i(forwardee), p2i(second_forwardee));
111 }
112 }
113 #else
114 oop forwardee = oop(*brooks_ptr_addr(obj));
115 #endif
116
117 log_develop_trace(gc)("Forwardee for "PTR_FORMAT" = "PTR_FORMAT, p2i(obj), p2i(forwardee));
118
119 return forwardee;
120 }
121
122 inline oop BrooksPointer::try_update_forwardee(oop holder, oop update) {
123 #ifdef ASSERT
124 assert(holder != NULL, "holder should not be NULL");
125 assert(update != NULL, "update should not be NULL");
126 assert(!oopDesc::unsafe_equals(holder, update), "forwarding should make progress");
127 assert(ShenandoahHeap::heap()->is_in(holder), "holder must point to a heap address");
128 assert(ShenandoahHeap::heap()->is_in(update), "update must point to a heap address");
129 assert(ShenandoahHeap::heap()->heap_region_containing(holder) !=
130 ShenandoahHeap::heap()->heap_region_containing(update), "forwarding should be across regions");
131 assert( ShenandoahHeap::heap()->in_cset_fast_test((HeapWord*) holder), "holder should be in collection set");
132 assert(!ShenandoahHeap::heap()->in_cset_fast_test((HeapWord*) update), "update should not be in collection set");
133 assert (holder->klass() == update->klass(), "klasses should match");
134
135 oop result;
136 if (ShenandoahVerifyWritesToFromSpace || ShenandoahVerifyReadsToFromSpace) {
137 ShenandoahHeap* sh = (ShenandoahHeap*) Universe::heap();
138 ShenandoahHeapRegion* hr = sh->heap_region_containing(holder);
139
140 {
141 hr->memProtectionOff();
142 result = (oop) Atomic::cmpxchg_ptr(update, brooks_ptr_addr(holder), holder);
143 hr->memProtectionOn();
144 }
145 } else {
146 result = (oop) Atomic::cmpxchg_ptr(update, brooks_ptr_addr(holder), holder);
147 }
148
149 assert(result != NULL, "CAS result is not NULL");
150 assert(ShenandoahHeap::heap()->is_in(result), "CAS result must point to a heap address");
151
152 if (oopDesc::unsafe_equals(result, holder)) {
153 log_develop_trace(gc)("Updated forwardee for "PTR_FORMAT" to "PTR_FORMAT, p2i(holder), p2i(update));
154 } else {
155 log_develop_trace(gc)("Failed to set forwardee for "PTR_FORMAT" to "PTR_FORMAT", was already "PTR_FORMAT, p2i(holder), p2i(update), p2i(result));
156 }
157 #else
158 oop result = (oop) Atomic::cmpxchg_ptr(update, brooks_ptr_addr(holder), holder);
159 #endif
160
161 return result;
162 }
163
164 #endif // SHARE_VM_GC_SHENANDOAH_BROOKSPOINTER_INLINE_HPP