194 // If we care generating the pre-barrier without a frame (e.g. in the
195 // intrinsified Reference.get() routine) then ebp might be pointing to
196 // the caller frame and so this check will most likely fail at runtime.
197 //
198 // Expanding the call directly bypasses the generation of the check.
199 // So when we do not have have a full interpreter frame on the stack
200 // expand_call should be passed true.
201
202 if (expand_call) {
203 assert(pre_val != c_rarg1, "smashed arg");
204 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
205 } else {
206 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
207 }
208
209 __ pop(saved, sp);
210
211 __ bind(done);
212 }
213
214 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst) {
215 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
216 Label is_null;
217 __ cbz(dst, is_null);
218 resolve_forward_pointer_not_null(masm, dst);
219 __ bind(is_null);
220 }
221
222 // IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2.
223 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst) {
224 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
225 __ ldr(dst, Address(dst, ShenandoahForwarding::byte_offset()));
226 }
227
228 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
229 assert(ShenandoahLoadRefBarrier, "Should be enabled");
230 assert(dst != rscratch2, "need rscratch2");
231
232 Label done;
233 __ enter();
234 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
235 __ ldrb(rscratch2, gc_state);
236
237 // Check for heap stability
238 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
239
240 RegSet to_save = RegSet::of(r0);
241 if (dst != r0) {
242 __ push(to_save, sp);
243 __ mov(r0, dst);
244 }
245
326 rthread /* thread */,
327 tmp1 /* tmp */,
328 val != noreg /* tosca_live */,
329 false /* expand_call */);
330
331 if (val == noreg) {
332 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
333 } else {
334 storeval_barrier(masm, val, tmp1);
335 // G1 barrier needs uncompressed oop for region cross check.
336 Register new_val = val;
337 if (UseCompressedOops) {
338 new_val = rscratch2;
339 __ mov(new_val, val);
340 }
341 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
342 }
343
344 }
345
346 void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
347 Register var_size_in_bytes,
348 int con_size_in_bytes,
349 Register t1,
350 Register t2,
351 Label& slow_case) {
352
353 assert_different_registers(obj, t2);
354 assert_different_registers(obj, var_size_in_bytes);
355 Register end = t2;
356
357 __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
358 if (var_size_in_bytes == noreg) {
359 __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahForwarding::byte_size())));
360 } else {
361 __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahForwarding::byte_size());
362 __ lea(end, Address(obj, var_size_in_bytes));
363 }
364 __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
365 __ cmp(end, rscratch1);
366 __ br(Assembler::HI, slow_case);
367
368 // update the tlab top pointer
369 __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
370
371 __ add(obj, obj, ShenandoahForwarding::byte_size());
372 __ str(obj, Address(obj, ShenandoahForwarding::byte_offset()));
373
374 // recover var_size_in_bytes if necessary
375 if (var_size_in_bytes == end) {
376 __ sub(var_size_in_bytes, var_size_in_bytes, obj);
377 }
378 }
379
380 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
381 bool acquire, bool release, bool weak, bool is_cae,
382 Register result) {
383 Register tmp1 = rscratch1;
384 Register tmp2 = rscratch2;
385 bool is_narrow = UseCompressedOops;
386 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
387
388 assert_different_registers(addr, expected, new_val, tmp1, tmp2);
389
390 Label retry, done, fail;
391
392 // CAS, using LL/SC pair.
393 __ bind(retry);
394 __ load_exclusive(tmp1, addr, size, acquire);
395 if (is_narrow) {
396 __ cmpw(tmp1, expected);
397 } else {
398 __ cmp(tmp1, expected);
399 }
400 __ br(Assembler::NE, fail);
401 __ store_exclusive(tmp2, new_val, addr, size, release);
402 if (weak) {
403 __ cmpw(tmp2, 0u); // If the store fails, return NE to our caller
404 } else {
405 __ cbnzw(tmp2, retry);
406 }
407 __ b(done);
408
409 __ bind(fail);
410 // Check if rb(expected)==rb(tmp1)
411 // Shuffle registers so that we have memory value ready for next expected.
412 __ mov(tmp2, expected);
413 __ mov(expected, tmp1);
414 if (is_narrow) {
415 __ decode_heap_oop(tmp1, tmp1);
416 __ decode_heap_oop(tmp2, tmp2);
417 }
418 resolve_forward_pointer(masm, tmp1);
419 resolve_forward_pointer(masm, tmp2);
420 __ cmp(tmp1, tmp2);
421 // Retry with expected now being the value we just loaded from addr.
422 __ br(Assembler::EQ, retry);
423 if (is_cae && is_narrow) {
424 // For cmp-and-exchange and narrow oops, we need to restore
425 // the compressed old-value. We moved it to 'expected' a few lines up.
426 __ mov(tmp1, expected);
427 }
428 __ bind(done);
429
430 if (is_cae) {
431 __ mov(result, tmp1);
432 } else {
433 __ cset(result, Assembler::EQ);
434 }
435 }
436
437 #ifdef COMPILER1
438
439 #undef __
552 //
553 // Output:
554 // r0: Pointer to evacuated OOP.
555 //
556 // Trash rscratch1, rscratch2. Preserve everything else.
557 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
558
559 __ align(6);
560 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
561 address start = __ pc();
562
563 Label work, done;
564 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
565 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
566 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
567 __ tbnz(rscratch2, 0, work);
568 __ ret(lr);
569 __ bind(work);
570
571 __ mov(rscratch2, r0);
572 resolve_forward_pointer_not_null(cgen->assembler(), r0);
573 __ cmp(rscratch2, r0);
574 __ br(Assembler::NE, done);
575
576 __ enter(); // required for proper stackwalking of RuntimeStub frame
577
578 __ push_call_clobbered_registers();
579
580 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
581 __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
582 __ mov(rscratch1, r0);
583 __ pop_call_clobbered_registers();
584 __ mov(r0, rscratch1);
585
586 __ leave(); // required for proper stackwalking of RuntimeStub frame
587 __ bind(done);
588 __ ret(lr);
589
590 return start;
591 }
592
|
194 // If we care generating the pre-barrier without a frame (e.g. in the
195 // intrinsified Reference.get() routine) then ebp might be pointing to
196 // the caller frame and so this check will most likely fail at runtime.
197 //
198 // Expanding the call directly bypasses the generation of the check.
199 // So when we do not have have a full interpreter frame on the stack
200 // expand_call should be passed true.
201
202 if (expand_call) {
203 assert(pre_val != c_rarg1, "smashed arg");
204 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
205 } else {
206 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
207 }
208
209 __ pop(saved, sp);
210
211 __ bind(done);
212 }
213
214 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
215 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
216 Label is_null;
217 __ cbz(dst, is_null);
218 resolve_forward_pointer_not_null(masm, dst, tmp);
219 __ bind(is_null);
220 }
221
222 // IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitely
223 // passed in.
224 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
225 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
226 // The below loads the mark word, checks if the lowest two bits are
227 // set, and if so, clear the lowest two bits and copy the result
228 // to dst. Otherwise it leaves dst alone.
229 // Implementing this is surprisingly awkward. I do it here by:
230 // - Inverting the mark word
231 // - Test lowest two bits == 0
232 // - If so, set the lowest two bits
233 // - Invert the result back, and copy to dst
234 Label done;
235 __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
236 __ eon(tmp, tmp, zr);
237 __ ands(zr, tmp, markOopDesc::lock_mask_in_place);
238 __ br(Assembler::NE, done);
239 __ orr(tmp, tmp, markOopDesc::marked_value);
240 __ eon(dst, tmp, zr);
241 __ bind(done);
242 }
243
244 void ShenandoahBarrierSetAssembler::load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Register tmp) {
245 assert(ShenandoahLoadRefBarrier, "Should be enabled");
246 assert(dst != rscratch2, "need rscratch2");
247
248 Label done;
249 __ enter();
250 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
251 __ ldrb(rscratch2, gc_state);
252
253 // Check for heap stability
254 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
255
256 RegSet to_save = RegSet::of(r0);
257 if (dst != r0) {
258 __ push(to_save, sp);
259 __ mov(r0, dst);
260 }
261
342 rthread /* thread */,
343 tmp1 /* tmp */,
344 val != noreg /* tosca_live */,
345 false /* expand_call */);
346
347 if (val == noreg) {
348 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
349 } else {
350 storeval_barrier(masm, val, tmp1);
351 // G1 barrier needs uncompressed oop for region cross check.
352 Register new_val = val;
353 if (UseCompressedOops) {
354 new_val = rscratch2;
355 __ mov(new_val, val);
356 }
357 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
358 }
359
360 }
361
362 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
363 bool acquire, bool release, bool weak, bool is_cae,
364 Register tmp, Register result) {
365 Register tmp1 = rscratch1;
366 Register tmp2 = rscratch2;
367 bool is_narrow = UseCompressedOops;
368 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
369
370 assert_different_registers(addr, expected, new_val, tmp1, tmp2);
371
372 Label retry, done, fail;
373
374 // CAS, using LL/SC pair.
375 __ bind(retry);
376 __ load_exclusive(tmp1, addr, size, acquire);
377 if (is_narrow) {
378 __ cmpw(tmp1, expected);
379 } else {
380 __ cmp(tmp1, expected);
381 }
382 __ br(Assembler::NE, fail);
383 __ store_exclusive(tmp2, new_val, addr, size, release);
384 if (weak) {
385 __ cmpw(tmp2, 0u); // If the store fails, return NE to our caller
386 } else {
387 __ cbnzw(tmp2, retry);
388 }
389 __ b(done);
390
391 __ bind(fail);
392 // Check if rb(expected)==rb(tmp1)
393 // Shuffle registers so that we have memory value ready for next expected.
394 __ mov(tmp2, expected);
395 __ mov(expected, tmp1);
396 if (is_narrow) {
397 __ decode_heap_oop(tmp1, tmp1);
398 __ decode_heap_oop(tmp2, tmp2);
399 }
400 resolve_forward_pointer(masm, tmp1, tmp);
401 resolve_forward_pointer(masm, tmp2, tmp);
402 __ cmp(tmp1, tmp2);
403 // Retry with expected now being the value we just loaded from addr.
404 __ br(Assembler::EQ, retry);
405 if (is_cae && is_narrow) {
406 // For cmp-and-exchange and narrow oops, we need to restore
407 // the compressed old-value. We moved it to 'expected' a few lines up.
408 __ mov(tmp1, expected);
409 }
410 __ bind(done);
411
412 if (is_cae) {
413 __ mov(result, tmp1);
414 } else {
415 __ cset(result, Assembler::EQ);
416 }
417 }
418
419 #ifdef COMPILER1
420
421 #undef __
534 //
535 // Output:
536 // r0: Pointer to evacuated OOP.
537 //
538 // Trash rscratch1, rscratch2. Preserve everything else.
539 address ShenandoahBarrierSetAssembler::generate_shenandoah_lrb(StubCodeGenerator* cgen) {
540
541 __ align(6);
542 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_lrb");
543 address start = __ pc();
544
545 Label work, done;
546 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
547 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
548 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
549 __ tbnz(rscratch2, 0, work);
550 __ ret(lr);
551 __ bind(work);
552
553 __ mov(rscratch2, r0);
554 resolve_forward_pointer_not_null(cgen->assembler(), r0, rscratch1);
555 __ cmp(rscratch2, r0);
556 __ br(Assembler::NE, done);
557
558 __ enter(); // required for proper stackwalking of RuntimeStub frame
559
560 __ push_call_clobbered_registers();
561
562 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_JRT));
563 __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
564 __ mov(rscratch1, r0);
565 __ pop_call_clobbered_registers();
566 __ mov(r0, rscratch1);
567
568 __ leave(); // required for proper stackwalking of RuntimeStub frame
569 __ bind(done);
570 __ ret(lr);
571
572 return start;
573 }
574
|