71 __ push(saved_regs, sp);
72 // must compute element count unless barrier set interface is changed (other platforms supply count)
73 assert_different_registers(start, end, scratch);
74 __ lea(scratch, Address(end, BytesPerHeapOop));
75 __ sub(scratch, scratch, start); // subtract start to get #bytes
76 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
77 __ mov(c_rarg0, start);
78 __ mov(c_rarg1, scratch);
79 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
80 __ pop(saved_regs, sp);
81 }
82 }
83
84 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
85 Register obj,
86 Register pre_val,
87 Register thread,
88 Register tmp,
89 bool tosca_live,
90 bool expand_call) {
91 if (ShenandoahConditionalSATBBarrier) {
92 Label done;
93 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
94 __ ldrb(tmp, gc_state);
95 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
96 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
97 __ bind(done);
98 }
99 if (ShenandoahSATBBarrier) {
100 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
105 Register obj,
106 Register pre_val,
107 Register thread,
108 Register tmp,
109 bool tosca_live,
110 bool expand_call) {
111 // If expand_call is true then we expand the call_VM_leaf macro
112 // directly to skip generating the check by
113 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
114
115 assert(thread == rthread, "must be");
116
117 Label done;
118 Label runtime;
247
248 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
249 if (ShenandoahReadBarrier) {
250 read_barrier_not_null_impl(masm, dst);
251 }
252 }
253
254
255 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
256 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
257 __ ldr(dst, Address(dst, BrooksPointer::byte_offset()));
258 }
259
260 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
261 if (ShenandoahWriteBarrier) {
262 write_barrier_impl(masm, dst);
263 }
264 }
265
266 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
267 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
268 assert(dst != rscratch1, "different regs");
269 assert(dst != rscratch2, "Need rscratch2");
270
271 Label done;
272
273 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
274 __ ldrb(rscratch1, gc_state);
275 __ membar(Assembler::LoadLoad);
276
277 // Now check if evacuation is in progress.
278 read_barrier_not_null(masm, dst);
279
280 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::PARTIAL | ShenandoahHeap::TRAVERSAL);
281 __ tst(rscratch1, rscratch2);
282 __ br(Assembler::EQ, done);
283
284 __ lsr(rscratch1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
285 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
286 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
287 __ tst(rscratch2, 0x1);
288 __ br(Assembler::EQ, done);
289
290 // Save possibly live regs.
291 RegSet live_regs = RegSet::range(r0, r4) - dst;
292 __ push(live_regs, sp);
293 __ strd(v0, __ pre(sp, 2 * -wordSize));
294
295 // Call into runtime
296 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_IRT), dst);
297
298 // Move result into dst reg.
299 __ mov(dst, r0);
300
301 // Restore possibly live regs.
302 __ ldrd(v0, __ post(sp, 2 * wordSize));
303 __ pop(live_regs, sp);
304
305 __ bind(done);
306 }
307
308 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
309 if (ShenandoahStoreValWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
310 Label is_null;
311 __ cbz(dst, is_null);
312 write_barrier_impl(masm, dst);
313 __ bind(is_null);
314 }
315 if (ShenandoahStoreValEnqueueBarrier) {
316 // Save possibly live regs.
317 RegSet live_regs = RegSet::range(r0, r4) - dst;
318 __ push(live_regs, sp);
319 __ strd(v0, __ pre(sp, 2 * -wordSize));
320
321 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
322
323 // Restore possibly live regs.
324 __ ldrd(v0, __ post(sp, 2 * wordSize));
325 __ pop(live_regs, sp);
326 }
327 if (ShenandoahStoreValReadBarrier) {
328 read_barrier_impl(masm, dst);
329 }
330 }
331
332 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
333 Register dst, Address src, Register tmp1, Register tmp_thread) {
334 bool on_oop = type == T_OBJECT || type == T_ARRAY;
335 bool in_heap = (decorators & IN_HEAP) != 0;
|
71 __ push(saved_regs, sp);
72 // must compute element count unless barrier set interface is changed (other platforms supply count)
73 assert_different_registers(start, end, scratch);
74 __ lea(scratch, Address(end, BytesPerHeapOop));
75 __ sub(scratch, scratch, start); // subtract start to get #bytes
76 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
77 __ mov(c_rarg0, start);
78 __ mov(c_rarg1, scratch);
79 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_ref_array_post_entry), 2);
80 __ pop(saved_regs, sp);
81 }
82 }
83
84 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
85 Register obj,
86 Register pre_val,
87 Register thread,
88 Register tmp,
89 bool tosca_live,
90 bool expand_call) {
91 if (ShenandoahSATBBarrier) {
92 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
93 }
94 }
95
96 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
97 Register obj,
98 Register pre_val,
99 Register thread,
100 Register tmp,
101 bool tosca_live,
102 bool expand_call) {
103 // If expand_call is true then we expand the call_VM_leaf macro
104 // directly to skip generating the check by
105 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
106
107 assert(thread == rthread, "must be");
108
109 Label done;
110 Label runtime;
239
240 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
241 if (ShenandoahReadBarrier) {
242 read_barrier_not_null_impl(masm, dst);
243 }
244 }
245
246
247 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
248 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
249 __ ldr(dst, Address(dst, BrooksPointer::byte_offset()));
250 }
251
252 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
253 if (ShenandoahWriteBarrier) {
254 write_barrier_impl(masm, dst);
255 }
256 }
257
258 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
259 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
260 assert(dst != rscratch1, "different regs");
261 assert(dst != rscratch2, "Need rscratch2");
262
263 Label done;
264
265 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
266 __ ldrb(rscratch1, gc_state);
267 __ membar(Assembler::LoadLoad);
268
269 // Now check if evacuation is in progress.
270 read_barrier_not_null(masm, dst);
271
272 __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
273 __ tst(rscratch1, rscratch2);
274 __ br(Assembler::EQ, done);
275
276 __ lsr(rscratch1, dst, ShenandoahHeapRegion::region_size_bytes_shift_jint());
277 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
278 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
279 __ tst(rscratch2, 0x1);
280 __ br(Assembler::EQ, done);
281
282 // Save possibly live regs.
283 RegSet live_regs = RegSet::range(r0, r4) - dst;
284 __ push(live_regs, sp);
285 __ strd(v0, __ pre(sp, 2 * -wordSize));
286
287 // Call into runtime
288 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahBarrierSet::write_barrier_IRT), dst);
289
290 // Move result into dst reg.
291 __ mov(dst, r0);
292
293 // Restore possibly live regs.
294 __ ldrd(v0, __ post(sp, 2 * wordSize));
295 __ pop(live_regs, sp);
296
297 __ bind(done);
298 }
299
300 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
301 if (ShenandoahStoreValEnqueueBarrier) {
302 Label is_null;
303 __ cbz(dst, is_null);
304 write_barrier_impl(masm, dst);
305 __ bind(is_null);
306 // Save possibly live regs.
307 RegSet live_regs = RegSet::range(r0, r4) - dst;
308 __ push(live_regs, sp);
309 __ strd(v0, __ pre(sp, 2 * -wordSize));
310
311 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
312
313 // Restore possibly live regs.
314 __ ldrd(v0, __ post(sp, 2 * wordSize));
315 __ pop(live_regs, sp);
316 }
317 if (ShenandoahStoreValReadBarrier) {
318 read_barrier_impl(masm, dst);
319 }
320 }
321
322 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
323 Register dst, Address src, Register tmp1, Register tmp_thread) {
324 bool on_oop = type == T_OBJECT || type == T_ARRAY;
325 bool in_heap = (decorators & IN_HEAP) != 0;
|