1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/brooksPointer.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
27 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
28 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35
36 #define __ masm->
37
38 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
39 Register addr, Register count, RegSet saved_regs) {
40 if (is_oop) {
41 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
42 if (!dest_uninitialized) {
43 __ push(saved_regs, sp);
44 if (count == c_rarg0) {
45 if (addr == c_rarg1) {
46 // exactly backwards!!
47 __ mov(rscratch1, c_rarg0);
48 __ mov(c_rarg0, c_rarg1);
49 __ mov(c_rarg1, rscratch1);
50 } else {
51 __ mov(c_rarg1, count);
52 __ mov(c_rarg0, addr);
53 }
54 } else {
55 __ mov(c_rarg0, addr);
56 __ mov(c_rarg1, count);
57 }
58 if (UseCompressedOops) {
59 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_narrow_oop_entry), 2);
60 } else {
61 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_pre_oop_entry), 2);
62 }
63 __ pop(saved_regs, sp);
64 }
65 }
66 }
67
68 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
69 Register start, Register end, Register scratch, RegSet saved_regs) {
70 if (is_oop) {
71 __ push(saved_regs, sp);
72 // must compute element count unless barrier set interface is changed (other platforms supply count)
73 assert_different_registers(start, end, scratch);
74 __ lea(scratch, Address(end, BytesPerHeapOop));
75 __ sub(scratch, scratch, start); // subtract start to get #bytes
76 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
77 __ mov(c_rarg0, start);
78 __ mov(c_rarg1, scratch);
79 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
80 __ pop(saved_regs, sp);
81 }
82 }
83
84 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
85 Register obj,
86 Register pre_val,
87 Register thread,
88 Register tmp,
89 bool tosca_live,
90 bool expand_call) {
91 if (ShenandoahConditionalSATBBarrier) {
92 Label done;
93 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
94 __ ldrb(tmp, gc_state);
95 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
96 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
97 __ bind(done);
98 }
99 if (ShenandoahSATBBarrier) {
100 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
105 Register obj,
106 Register pre_val,
107 Register thread,
108 Register tmp,
109 bool tosca_live,
110 bool expand_call) {
111 // If expand_call is true then we expand the call_VM_leaf macro
112 // directly to skip generating the check by
113 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
114
115 assert(thread == rthread, "must be");
116
117 Label done;
118 Label runtime;
119
120 assert_different_registers(obj, pre_val, tmp, rscratch1);
121 assert(pre_val != noreg && tmp != noreg, "expecting a register");
122
123 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
124 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
125 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
126
127 // Is marking active?
128 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
129 __ ldrw(tmp, in_progress);
130 } else {
131 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
132 __ ldrb(tmp, in_progress);
133 }
134 __ cbzw(tmp, done);
135
136 // Do we need to load the previous value?
137 if (obj != noreg) {
138 __ load_heap_oop(pre_val, Address(obj, 0));
139 }
140
141 // Is the previous value null?
142 __ cbz(pre_val, done);
143
144 // Can we store original value in the thread's buffer?
145 // Is index == 0?
146 // (The index field is typed as size_t.)
147
148 __ ldr(tmp, index); // tmp := *index_adr
149 __ cbz(tmp, runtime); // tmp == 0?
150 // If yes, goto runtime
151
152 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
153 __ str(tmp, index); // *index_adr := tmp
154 __ ldr(rscratch1, buffer);
155 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
156
157 // Record the previous value
158 __ str(pre_val, Address(tmp, 0));
159 __ b(done);
160
161 __ bind(runtime);
162 // save the live input values
163 RegSet saved = RegSet::of(pre_val);
164 if (tosca_live) saved += RegSet::of(r0);
165 if (obj != noreg) saved += RegSet::of(obj);
166
167 __ push(saved, sp);
168
169 // Calling the runtime using the regular call_VM_leaf mechanism generates
170 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
171 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
172 //
173 // If we care generating the pre-barrier without a frame (e.g. in the
174 // intrinsified Reference.get() routine) then ebp might be pointing to
175 // the caller frame and so this check will most likely fail at runtime.
176 //
177 // Expanding the call directly bypasses the generation of the check.
178 // So when we do not have have a full interpreter frame on the stack
179 // expand_call should be passed true.
180
181 if (expand_call) {
182 assert(pre_val != c_rarg1, "smashed arg");
183 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
184 } else {
185 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
186 }
187
188 __ pop(saved, sp);
189
190 __ bind(done);
191 }
192
193 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_post(MacroAssembler* masm,
194 Register store_addr,
195 Register new_val,
196 Register thread,
197 Register tmp,
198 Register tmp2) {
199 assert(thread == rthread, "must be");
200 assert(UseShenandoahGC, "expect Shenandoah GC");
201
202 if (! UseShenandoahMatrix) {
203 // No need for that barrier if not using matrix.
204 return;
205 }
206
207 assert_different_registers(store_addr, new_val, thread, tmp, tmp2, rscratch1);
208
209 Label done;
210 __ cbz(new_val, done);
211
212 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
213
214 // Compute to-region index
215 __ lsr(tmp, new_val, ShenandoahHeapRegion::region_size_bytes_shift_jint());
216
217 // Compute from-region index
218 __ lsr(tmp2, store_addr, ShenandoahHeapRegion::region_size_bytes_shift_jint());
219
220 // Compute matrix index
221 __ mov(rscratch1, matrix->stride_jint());
222 // Address is _matrix[to * stride + from]
223 __ madd(tmp, tmp, rscratch1, tmp2);
224 __ mov(rscratch1, matrix->magic_offset());
225 Address loc(tmp, rscratch1);
226
227 __ ldrb(tmp2, loc);
228 __ cbnz(tmp2, done);
229 __ mov(tmp2, 1);
230 __ strb(tmp2, loc);
231 __ bind(done);
232 }
233
234 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
235 if (ShenandoahReadBarrier) {
236 read_barrier_impl(masm, dst);
237 }
238 }
239
240 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
241 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
242 Label is_null;
243 __ cbz(dst, is_null);
244 read_barrier_not_null_impl(masm, dst);
245 __ bind(is_null);
246 }
247
248 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
249 if (ShenandoahReadBarrier) {
250 read_barrier_not_null_impl(masm, dst);
251 }
252 }
253
254
255 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
256 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
257 __ ldr(dst, Address(dst, BrooksPointer::byte_offset()));
258 }
259
260 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
261 if (ShenandoahWriteBarrier) {
262 write_barrier_impl(masm, dst);
263 }
264 }
265
266 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
267 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
268 assert(dst != rscratch1, "different regs");
269 assert(dst != rscratch2, "Need rscratch2");
270
271 Label done;
272
273 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
274 __ ldrb(rscratch1, gc_state);
275 __ membar(Assembler::LoadLoad);
276
277 // Now check if evacuation is in progress.
278 read_barrier_not_null(masm, dst);
279
280 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL);
281 __ tst(rscratch1, rscratch2);
282 __ br(Assembler::EQ, done);
283
284 __ lsr(rscratch1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
285 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
286 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
287 __ tst(rscratch2, 0x1);
288 __ br(Assembler::EQ, done);
289
290 // Save possibly live regs.
291 RegSet live_regs = RegSet::range(r0, r4) - dst;
292 __ push(live_regs, sp);
293 __ strd(v0, __ pre(sp, 2 * -wordSize));
294
295 // Call into runtime
296 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_IRT), dst);
297
298 // Move result into dst reg.
299 __ mov(dst, r0);
300
301 // Restore possibly live regs.
302 __ ldrd(v0, __ post(sp, 2 * wordSize));
303 __ pop(live_regs, sp);
304
305 __ bind(done);
306 }
307
308 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
309 if (ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
310 Label is_null;
311 __ cbz(dst, is_null);
312 write_barrier_impl(masm, dst);
313 __ bind(is_null);
314 }
315 if (ShenandoahStoreValEnqueueBarrier) {
316 // Save possibly live regs.
317 RegSet live_regs = RegSet::range(r0, r4) - dst;
318 __ push(live_regs, sp);
319 __ strd(v0, __ pre(sp, 2 * -wordSize));
320
321 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
322
323 // Restore possibly live regs.
324 __ ldrd(v0, __ post(sp, 2 * wordSize));
325 __ pop(live_regs, sp);
326 }
327 if (ShenandoahStoreValReadBarrier) {
328 read_barrier_impl(masm, dst);
329 }
330 }
331
332 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
333 Register dst, Address src, Register tmp1, Register tmp_thread) {
334 bool on_oop = type == T_OBJECT || type == T_ARRAY;
335 bool in_heap = (decorators & IN_HEAP) != 0;
336 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
337 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
338 bool on_reference = on_weak || on_phantom;
339 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
340 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
341 satb_write_barrier_pre(masm /* masm */,
342 noreg /* obj */,
343 dst /* pre_val */,
344 rthread /* thread */,
345 tmp1 /* tmp */,
346 true /* tosca_live */,
347 true /* expand_call */);
348 }
349 }
350
351 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
352 Address dst, Register val, Register tmp1, Register tmp2) {
353 bool on_oop = type == T_OBJECT || type == T_ARRAY;
354 if (!on_oop) {
355 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
356 return;
357 }
358
359 // flatten object address if needed
360 if (dst.index() == noreg && dst.offset() == 0) {
361 if (dst.base() != r3) {
362 __ mov(r3, dst.base());
363 }
364 } else {
365 __ lea(r3, dst);
366 }
367
368 shenandoah_write_barrier_pre(masm,
369 r3 /* obj */,
370 tmp2 /* pre_val */,
371 rthread /* thread */,
372 tmp1 /* tmp */,
373 val != noreg /* tosca_live */,
374 false /* expand_call */);
375
376 if (val == noreg) {
377 __ store_heap_oop_null(Address(r3, 0));
378 } else {
379 storeval_barrier(masm, val, tmp1);
380 // G1 barrier needs uncompressed oop for region cross check.
381 Register new_val = val;
382 if (UseCompressedOops) {
383 new_val = rscratch2;
384 __ mov(new_val, val);
385 }
386 __ store_heap_oop(Address(r3, 0), val);
387 shenandoah_write_barrier_post(masm,
388 r3 /* store_adr */,
389 new_val /* new_val */,
390 rthread /* thread */,
391 tmp1 /* tmp */,
392 tmp2 /* tmp2 */);
393 }
394
395 }
396
397 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, DecoratorSet decorators, Register op1, Register op2) {
398 __ cmp(op1, op2);
399 if (ShenandoahAcmpBarrier) {
400 Label done;
401 __ br(Assembler::EQ, done);
402 // The object may have been evacuated, but we won't see it without a
403 // membar here.
404 __ membar(Assembler::LoadStore| Assembler::LoadLoad);
405 read_barrier(masm, op1);
406 read_barrier(masm, op2);
407 __ cmp(op1, op2);
408 __ bind(done);
409 }
410 }
411
412 void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
413 bool oop_not_null = (decorators & OOP_NOT_NULL) != 0;
414 if (oop_not_null) {
415 read_barrier_not_null(masm, obj);
416 } else {
417 read_barrier(masm, obj);
418 }
419 }
420
421 void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
422 write_barrier(masm, obj);
423 }