1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/brooksPointer.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35
36 #define __ masm->
37
38 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
39 Register addr, Register count, RegSet saved_regs) {
40 if (is_oop) {
41 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
42 if (!dest_uninitialized) {
43 __ push(saved_regs, sp);
44 if (count == c_rarg0) {
45 if (addr == c_rarg1) {
46 // exactly backwards!!
47 __ mov(rscratch1, c_rarg0);
48 __ mov(c_rarg0, c_rarg1);
49 __ mov(c_rarg1, rscratch1);
50 } else {
51 __ mov(c_rarg1, count);
52 __ mov(c_rarg0, addr);
53 }
54 } else {
55 __ mov(c_rarg0, addr);
56 __ mov(c_rarg1, count);
57 }
58 if (UseCompressedOops) {
59 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_narrow_oop_entry), 2);
60 } else {
61 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_oop_entry), 2);
62 }
63 __ pop(saved_regs, sp);
64 }
65 }
66 }
67
68 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
69 Register start, Register end, Register scratch, RegSet saved_regs) {
70 if (is_oop) {
71 __ push(saved_regs, sp);
72 // must compute element count unless barrier set interface is changed (other platforms supply count)
73 assert_different_registers(start, end, scratch);
74 __ lea(scratch, Address(end, BytesPerHeapOop));
75 __ sub(scratch, scratch, start); // subtract start to get #bytes
76 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
77 __ mov(c_rarg0, start);
78 __ mov(c_rarg1, scratch);
79 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
80 __ pop(saved_regs, sp);
81 }
82 }
83
84 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
85 Register obj,
86 Register pre_val,
87 Register thread,
88 Register tmp,
89 bool tosca_live,
90 bool expand_call) {
91 if (ShenandoahSATBBarrier) {
92 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
93 }
94 }
95
96 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
97 Register obj,
98 Register pre_val,
99 Register thread,
100 Register tmp,
101 bool tosca_live,
102 bool expand_call) {
103 // If expand_call is true then we expand the call_VM_leaf macro
104 // directly to skip generating the check by
105 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
106
107 assert(thread == rthread, "must be");
108
109 Label done;
110 Label runtime;
111
112 assert_different_registers(obj, pre_val, tmp, rscratch1);
113 assert(pre_val != noreg && tmp != noreg, "expecting a register");
114
115 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
116 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
117 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
118
119 // Is marking active?
120 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
121 __ ldrw(tmp, in_progress);
122 } else {
123 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
124 __ ldrb(tmp, in_progress);
125 }
126 __ cbzw(tmp, done);
127
128 // Do we need to load the previous value?
129 if (obj != noreg) {
130 __ load_heap_oop(pre_val, Address(obj, 0));
131 }
132
133 // Is the previous value null?
134 __ cbz(pre_val, done);
135
136 // Can we store original value in the thread's buffer?
137 // Is index == 0?
138 // (The index field is typed as size_t.)
139
140 __ ldr(tmp, index); // tmp := *index_adr
141 __ cbz(tmp, runtime); // tmp == 0?
142 // If yes, goto runtime
143
144 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
145 __ str(tmp, index); // *index_adr := tmp
146 __ ldr(rscratch1, buffer);
147 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
148
149 // Record the previous value
150 __ str(pre_val, Address(tmp, 0));
151 __ b(done);
152
153 __ bind(runtime);
154 // save the live input values
155 RegSet saved = RegSet::of(pre_val);
156 if (tosca_live) saved += RegSet::of(r0);
157 if (obj != noreg) saved += RegSet::of(obj);
158
159 __ push(saved, sp);
160
161 // Calling the runtime using the regular call_VM_leaf mechanism generates
162 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
163 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
164 //
165 // If we care generating the pre-barrier without a frame (e.g. in the
166 // intrinsified Reference.get() routine) then ebp might be pointing to
167 // the caller frame and so this check will most likely fail at runtime.
168 //
169 // Expanding the call directly bypasses the generation of the check.
170 // So when we do not have have a full interpreter frame on the stack
171 // expand_call should be passed true.
172
173 if (expand_call) {
174 assert(pre_val != c_rarg1, "smashed arg");
175 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
176 } else {
177 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
178 }
179
180 __ pop(saved, sp);
181
182 __ bind(done);
183 }
184
185 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_post(MacroAssembler* masm,
186 Register store_addr,
187 Register new_val,
188 Register thread,
189 Register tmp,
190 Register tmp2) {
191 assert(thread == rthread, "must be");
192 assert(UseShenandoahGC, "expect Shenandoah GC");
193
194 if (! UseShenandoahMatrix) {
195 // No need for that barrier if not using matrix.
196 return;
197 }
198
199 assert_different_registers(store_addr, new_val, thread, tmp, tmp2, rscratch1);
200
201 Label done;
202 __ cbz(new_val, done);
203
204 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
205
206 // Compute to-region index
207 __ lsr(tmp, new_val, ShenandoahHeapRegion::region_size_bytes_shift_jint());
208
209 // Compute from-region index
210 __ lsr(tmp2, store_addr, ShenandoahHeapRegion::region_size_bytes_shift_jint());
211
212 // Compute matrix index
213 __ mov(rscratch1, matrix->stride_jint());
214 // Address is _matrix[to * stride + from]
215 __ madd(tmp, tmp, rscratch1, tmp2);
216 __ mov(rscratch1, matrix->magic_offset());
217 Address loc(tmp, rscratch1);
218
219 __ ldrb(tmp2, loc);
220 __ cbnz(tmp2, done);
221 __ mov(tmp2, 1);
222 __ strb(tmp2, loc);
223 __ bind(done);
224 }
225
226 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
227 if (ShenandoahReadBarrier) {
228 read_barrier_impl(masm, dst);
229 }
230 }
231
232 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
233 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
234 Label is_null;
235 __ cbz(dst, is_null);
236 read_barrier_not_null_impl(masm, dst);
237 __ bind(is_null);
238 }
239
240 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
241 if (ShenandoahReadBarrier) {
242 read_barrier_not_null_impl(masm, dst);
243 }
244 }
245
246
247 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
248 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
249 __ ldr(dst, Address(dst, BrooksPointer::byte_offset()));
250 }
251
252 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
253 if (ShenandoahWriteBarrier) {
254 write_barrier_impl(masm, dst);
255 }
256 }
257
258 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
259 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
260 assert(dst != rscratch1, "different regs");
261 assert(dst != rscratch2, "Need rscratch2");
262
263 Label done;
264
265 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
266 __ ldrb(rscratch1, gc_state);
267 __ membar(Assembler::LoadLoad);
268
269 // Now check if evacuation is in progress.
270 read_barrier_not_null(masm, dst);
271
272 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
273 __ tst(rscratch1, rscratch2);
274 __ br(Assembler::EQ, done);
275
276 __ lsr(rscratch1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
277 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
278 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
279 __ tst(rscratch2, 0x1);
280 __ br(Assembler::EQ, done);
281
282 // Save possibly live regs.
283 RegSet live_regs = RegSet::range(r0, r4) - dst;
284 __ push(live_regs, sp);
285 __ strd(v0, __ pre(sp, 2 * -wordSize));
286
287 // Call into runtime
288 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_IRT), dst);
289
290 // Move result into dst reg.
291 __ mov(dst, r0);
292
293 // Restore possibly live regs.
294 __ ldrd(v0, __ post(sp, 2 * wordSize));
295 __ pop(live_regs, sp);
296
297 __ bind(done);
298 }
299
300 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
301 if (ShenandoahStoreValEnqueueBarrier) {
302 Label is_null;
303 __ cbz(dst, is_null);
304 write_barrier_impl(masm, dst);
305 __ bind(is_null);
306 // Save possibly live regs.
307 RegSet live_regs = RegSet::range(r0, r4) - dst;
308 __ push(live_regs, sp);
309 __ strd(v0, __ pre(sp, 2 * -wordSize));
310
311 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
312
313 // Restore possibly live regs.
314 __ ldrd(v0, __ post(sp, 2 * wordSize));
315 __ pop(live_regs, sp);
316 }
317 if (ShenandoahStoreValReadBarrier) {
318 read_barrier_impl(masm, dst);
319 }
320 }
321
322 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
323 Register dst, Address src, Register tmp1, Register tmp_thread) {
324 bool on_oop = type == T_OBJECT || type == T_ARRAY;
325 bool in_heap = (decorators & IN_HEAP) != 0;
326 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
327 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
328 bool on_reference = on_weak || on_phantom;
329 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
330 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
331 satb_write_barrier_pre(masm /* masm */,
332 noreg /* obj */,
333 dst /* pre_val */,
334 rthread /* thread */,
335 tmp1 /* tmp */,
336 true /* tosca_live */,
337 true /* expand_call */);
338 }
339 }
340
341 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
342 Address dst, Register val, Register tmp1, Register tmp2) {
343 bool on_oop = type == T_OBJECT || type == T_ARRAY;
344 if (!on_oop) {
345 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
346 return;
347 }
348
349 // flatten object address if needed
350 if (dst.index() == noreg && dst.offset() == 0) {
351 if (dst.base() != r3) {
352 __ mov(r3, dst.base());
353 }
354 } else {
355 __ lea(r3, dst);
356 }
357
358 shenandoah_write_barrier_pre(masm,
359 r3 /* obj */,
360 tmp2 /* pre_val */,
361 rthread /* thread */,
362 tmp1 /* tmp */,
363 val != noreg /* tosca_live */,
364 false /* expand_call */);
365
366 if (val == noreg) {
367 __ store_heap_oop_null(Address(r3, 0));
368 } else {
369 storeval_barrier(masm, val, tmp1);
370 // G1 barrier needs uncompressed oop for region cross check.
371 Register new_val = val;
372 if (UseCompressedOops) {
373 new_val = rscratch2;
374 __ mov(new_val, val);
375 }
376 __ store_heap_oop(Address(r3, 0), val);
377 shenandoah_write_barrier_post(masm,
378 r3 /* store_adr */,
379 new_val /* new_val */,
380 rthread /* thread */,
381 tmp1 /* tmp */,
382 tmp2 /* tmp2 */);
383 }
384
385 }
386
387 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, DecoratorSet decorators, Register op1, Register op2) {
388 __ cmp(op1, op2);
389 if (ShenandoahAcmpBarrier) {
390 Label done;
391 __ br(Assembler::EQ, done);
392 // The object may have been evacuated, but we won't see it without a
393 // membar here.
394 __ membar(Assembler::LoadStore| Assembler::LoadLoad);
395 read_barrier(masm, op1);
396 read_barrier(masm, op2);
397 __ cmp(op1, op2);
398 __ bind(done);
399 }
400 }
401
402 void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
403 bool oop_not_null = (decorators & OOP_NOT_NULL) != 0;
404 if (oop_not_null) {
405 read_barrier_not_null(masm, obj);
406 } else {
407 read_barrier(masm, obj);
408 }
409 }
410
411 void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
412 write_barrier(masm, obj);
413 }