1 /*
2 * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "c1/c1_IR.hpp"
26 #include "gc/shared/satbMarkQueue.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahBrooksPointer.hpp"
29 #include "gc/shenandoah/shenandoahHeap.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
32 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
33
34 #ifndef PATCHED_ADDR
35 #define PATCHED_ADDR (max_jint)
36 #endif
37
38 #ifdef ASSERT
39 #define __ gen->lir(__FILE__, __LINE__)->
40 #else
41 #define __ gen->lir()->
42 #endif
43
44 void ShenandoahPreBarrierStub::emit_code(LIR_Assembler* ce) {
45 ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
46 bs->gen_pre_barrier_stub(ce, this);
47 }
48
49 void ShenandoahWriteBarrierStub::emit_code(LIR_Assembler* ce) {
50 ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
51 bs->gen_write_barrier_stub(ce, this);
52 }
53
54 void ShenandoahBarrierSetC1::pre_barrier(LIRGenerator* gen, CodeEmitInfo* info, DecoratorSet decorators, LIR_Opr addr_opr, LIR_Opr pre_val) {
55 // First we test whether marking is in progress.
56 BasicType flag_type;
57 bool patch = (decorators & C1_NEEDS_PATCHING) != 0;
58 bool do_load = pre_val == LIR_OprFact::illegalOpr;
59 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
60 flag_type = T_INT;
61 } else {
62 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1,
63 "Assumption");
64 // Use unsigned type T_BOOLEAN here rather than signed T_BYTE since some platforms, eg. ARM,
65 // need to use unsigned instructions to use the large offset to load the satb_mark_queue.
66 flag_type = T_BOOLEAN;
67 }
68 LIR_Opr thrd = gen->getThreadPointer();
69 LIR_Address* mark_active_flag_addr =
70 new LIR_Address(thrd,
71 in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()),
72 flag_type);
73 // Read the marking-in-progress flag.
74 LIR_Opr flag_val = gen->new_register(T_INT);
75 __ load(mark_active_flag_addr, flag_val);
76 __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
77
78 LIR_PatchCode pre_val_patch_code = lir_patch_none;
79
80 CodeStub* slow;
81
82 if (do_load) {
83 assert(pre_val == LIR_OprFact::illegalOpr, "sanity");
84 assert(addr_opr != LIR_OprFact::illegalOpr, "sanity");
85
86 if (patch)
87 pre_val_patch_code = lir_patch_normal;
88
89 pre_val = gen->new_register(T_OBJECT);
90
91 if (!addr_opr->is_address()) {
92 assert(addr_opr->is_register(), "must be");
93 addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, T_OBJECT));
94 }
95 slow = new ShenandoahPreBarrierStub(addr_opr, pre_val, pre_val_patch_code, info ? new CodeEmitInfo(info) : NULL);
96 } else {
97 assert(addr_opr == LIR_OprFact::illegalOpr, "sanity");
98 assert(pre_val->is_register(), "must be");
99 assert(pre_val->type() == T_OBJECT, "must be an object");
100
101 slow = new ShenandoahPreBarrierStub(pre_val);
102 }
103
104 __ branch(lir_cond_notEqual, T_INT, slow);
105 __ branch_destination(slow->continuation());
106 }
107
108 LIR_Opr ShenandoahBarrierSetC1::read_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
109 if (UseShenandoahGC && ShenandoahReadBarrier) {
110 return read_barrier_impl(gen, obj, info, need_null_check);
111 } else {
112 return obj;
113 }
114 }
115
116 LIR_Opr ShenandoahBarrierSetC1::read_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
117 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "Should be enabled");
118 LabelObj* done = new LabelObj();
119 LIR_Opr result = gen->new_register(T_OBJECT);
120 __ move(obj, result);
121 if (need_null_check) {
122 __ cmp(lir_cond_equal, result, LIR_OprFact::oopConst(NULL));
123 __ branch(lir_cond_equal, T_LONG, done->label());
124 }
125 LIR_Address* brooks_ptr_address = gen->generate_address(result, ShenandoahBrooksPointer::byte_offset(), T_ADDRESS);
126 __ load(brooks_ptr_address, result, info ? new CodeEmitInfo(info) : NULL, lir_patch_none);
127
128 __ branch_destination(done->label());
129 return result;
130 }
131
132 LIR_Opr ShenandoahBarrierSetC1::write_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
133 if (UseShenandoahGC && ShenandoahWriteBarrier) {
134 return write_barrier_impl(gen, obj, info, need_null_check);
135 } else {
136 return obj;
137 }
138 }
139
140 LIR_Opr ShenandoahBarrierSetC1::write_barrier_impl(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, bool need_null_check) {
141 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
142
143 obj = ensure_in_register(gen, obj);
144 assert(obj->is_register(), "must be a register at this point");
145 LIR_Opr result = gen->new_register(T_OBJECT);
146 __ move(obj, result);
147
148 LIR_Opr thrd = gen->getThreadPointer();
149 LIR_Address* active_flag_addr =
150 new LIR_Address(thrd,
151 in_bytes(ShenandoahThreadLocalData::gc_state_offset()),
152 T_BYTE);
153 // Read and check the gc-state-flag.
154 LIR_Opr flag_val = gen->new_register(T_INT);
155 __ load(active_flag_addr, flag_val);
156 LIR_Opr mask = LIR_OprFact::intConst(ShenandoahHeap::HAS_FORWARDED |
157 ShenandoahHeap::EVACUATION |
158 ShenandoahHeap::TRAVERSAL);
159 LIR_Opr mask_reg = gen->new_register(T_INT);
160 __ move(mask, mask_reg);
161
162 if (TwoOperandLIRForm) {
163 __ logical_and(flag_val, mask_reg, flag_val);
164 } else {
165 LIR_Opr masked_flag = gen->new_register(T_INT);
166 __ logical_and(flag_val, mask_reg, masked_flag);
167 flag_val = masked_flag;
168 }
169 __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0));
170
171 CodeStub* slow = new ShenandoahWriteBarrierStub(obj, result, info ? new CodeEmitInfo(info) : NULL, need_null_check);
172 __ branch(lir_cond_notEqual, T_INT, slow);
173 __ branch_destination(slow->continuation());
174
175 return result;
176 }
177
178 LIR_Opr ShenandoahBarrierSetC1::ensure_in_register(LIRGenerator* gen, LIR_Opr obj) {
179 if (!obj->is_register()) {
180 LIR_Opr obj_reg = gen->new_register(T_OBJECT);
181 if (obj->is_constant()) {
182 __ move(obj, obj_reg);
183 } else {
184 __ leal(obj, obj_reg);
185 }
186 obj = obj_reg;
187 }
188 return obj;
189 }
190
191 LIR_Opr ShenandoahBarrierSetC1::storeval_barrier(LIRGenerator* gen, LIR_Opr obj, CodeEmitInfo* info, DecoratorSet decorators) {
192 bool need_null_check = (decorators & IS_NOT_NULL) == 0;
193 if (ShenandoahStoreValEnqueueBarrier) {
194 obj = write_barrier_impl(gen, obj, info, need_null_check);
195 pre_barrier(gen, info, decorators, LIR_OprFact::illegalOpr, obj);
196 }
197 if (ShenandoahStoreValReadBarrier) {
198 obj = read_barrier_impl(gen, obj, info, true /*need_null_check*/);
199 }
200 return obj;
201 }
202
203 LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
204 DecoratorSet decorators = access.decorators();
205 bool is_array = (decorators & IS_ARRAY) != 0;
206 bool needs_patching = (decorators & C1_NEEDS_PATCHING) != 0;
207
208 bool is_write = (decorators & ACCESS_WRITE) != 0;
209 bool needs_null_check = (decorators & IS_NOT_NULL) == 0;
210
211 LIR_Opr base = access.base().item().result();
212 LIR_Opr offset = access.offset().opr();
213 LIRGenerator* gen = access.gen();
214
215 if (is_write) {
216 base = write_barrier(gen, base, access.access_emit_info(), needs_null_check);
217 } else {
218 base = read_barrier(gen, base, access.access_emit_info(), needs_null_check);
219 }
220
221 LIR_Opr addr_opr;
222 if (is_array) {
223 addr_opr = LIR_OprFact::address(gen->emit_array_address(base, offset, access.type()));
224 } else if (needs_patching) {
225 // we need to patch the offset in the instruction so don't allow
226 // generate_address to try to be smart about emitting the -1.
227 // Otherwise the patching code won't know how to find the
228 // instruction to patch.
229 addr_opr = LIR_OprFact::address(new LIR_Address(base, PATCHED_ADDR, access.type()));
230 } else {
231 addr_opr = LIR_OprFact::address(gen->generate_address(base, offset, 0, 0, access.type()));
232 }
233
234 if (resolve_in_register) {
235 LIR_Opr resolved_addr = gen->new_pointer_register();
236 __ leal(addr_opr, resolved_addr);
237 resolved_addr = LIR_OprFact::address(new LIR_Address(resolved_addr, access.type()));
238 return resolved_addr;
239 } else {
240 return addr_opr;
241 }
242 }
243
244 void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
245 if (access.is_oop()) {
246 if (ShenandoahSATBBarrier) {
247 pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), access.resolved_addr(), LIR_OprFact::illegalOpr /* pre_val */);
248 }
249 value = storeval_barrier(access.gen(), value, access.access_emit_info(), access.decorators());
250 }
251 BarrierSetC1::store_at_resolved(access, value);
252 }
253
254 void ShenandoahBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
255 BarrierSetC1::load_at_resolved(access, result);
256
257 if (ShenandoahKeepAliveBarrier) {
258 DecoratorSet decorators = access.decorators();
259 bool is_weak = (decorators & ON_WEAK_OOP_REF) != 0;
260 bool is_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
261 bool is_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
262 LIRGenerator *gen = access.gen();
263 if (access.is_oop() && (is_weak || is_phantom || is_anonymous)) {
264 // Register the value in the referent field with the pre-barrier
265 LabelObj *Lcont_anonymous;
266 if (is_anonymous) {
267 Lcont_anonymous = new LabelObj();
268 generate_referent_check(access, Lcont_anonymous);
269 }
270 pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr /* addr_opr */,
271 result /* pre_val */);
272 if (is_anonymous) {
273 __ branch_destination(Lcont_anonymous->label());
274 }
275 }
276 }
277 }
278
279 LIR_Opr ShenandoahBarrierSetC1::atomic_add_at_resolved(LIRAccess& access, LIRItem& value) {
280 return BarrierSetC1::atomic_add_at_resolved(access, value);
281 }
282
283 LIR_Opr ShenandoahBarrierSetC1::resolve(LIRGenerator* gen, DecoratorSet decorators, LIR_Opr obj) {
284 bool is_write = decorators & ACCESS_WRITE;
285 if (is_write) {
286 return write_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0);
287 } else {
288 return read_barrier(gen, obj, NULL, (decorators & IS_NOT_NULL) == 0);
289 }
290 }
291
292 class C1ShenandoahPreBarrierCodeGenClosure : public StubAssemblerCodeGenClosure {
293 virtual OopMapSet* generate_code(StubAssembler* sasm) {
294 ShenandoahBarrierSetAssembler* bs = (ShenandoahBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
295 bs->generate_c1_pre_barrier_runtime_stub(sasm);
296 return NULL;
297 }
298 };
299
300 void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) {
301 C1ShenandoahPreBarrierCodeGenClosure pre_code_gen_cl;
302 _pre_barrier_c1_runtime_code_blob = Runtime1::generate_blob(buffer_blob, -1,
303 "shenandoah_pre_barrier_slow",
304 false, &pre_code_gen_cl);
305 }