329 __ push(live_regs, sp);
330 __ strd(v0, __ pre(sp, 2 * -wordSize));
331
332 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
333
334 // Restore possibly live regs.
335 __ ldrd(v0, __ post(sp, 2 * wordSize));
336 __ pop(live_regs, sp);
337 }
338 }
339
340 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr) {
341 if (ShenandoahLoadRefBarrier) {
342 Label is_null;
343 __ cbz(dst, is_null);
344 load_reference_barrier_not_null(masm, dst, load_addr);
345 __ bind(is_null);
346 }
347 }
348
349 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
350 Register dst, Address src, Register tmp1, Register tmp_thread) {
351 bool on_oop = is_reference_type(type);
352 bool not_in_heap = (decorators & IN_NATIVE) != 0;
353 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
354 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
355 bool on_reference = on_weak || on_phantom;
356 bool is_traversal_mode = ShenandoahHeap::heap()->is_traversal_mode();
357 bool keep_alive = (decorators & AS_NO_KEEPALIVE) == 0 || is_traversal_mode;
358
359 Register result_dst = dst;
360
361 if (on_oop) {
362 // We want to preserve src
363 if (dst == src.base() || dst == src.index()) {
364 dst = rscratch1;
365 }
366 assert_different_registers(dst, src.base(), src.index());
367 }
368
369 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
370 if (on_oop) {
371 if (not_in_heap && !is_traversal_mode) {
372 load_reference_barrier_native(masm, dst, src);
373 } else {
374 load_reference_barrier(masm, dst, src);
375 }
376
377 if (dst != result_dst) {
378 __ mov(result_dst, dst);
379 dst = result_dst;
380 }
381
382 if (ShenandoahKeepAliveBarrier && on_reference && keep_alive) {
383 __ enter();
384 satb_write_barrier_pre(masm /* masm */,
385 noreg /* obj */,
386 dst /* pre_val */,
387 rthread /* thread */,
388 tmp1 /* tmp */,
389 true /* tosca_live */,
390 true /* expand_call */);
391 __ leave();
392 }
393 }
394 }
395
396 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
397 Address dst, Register val, Register tmp1, Register tmp2) {
398 bool on_oop = is_reference_type(type);
399 if (!on_oop) {
400 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
401 return;
402 }
403
404 // flatten object address if needed
405 if (dst.index() == noreg && dst.offset() == 0) {
406 if (dst.base() != r3) {
407 __ mov(r3, dst.base());
408 }
409 } else {
410 __ lea(r3, dst);
411 }
412
|
329 __ push(live_regs, sp);
330 __ strd(v0, __ pre(sp, 2 * -wordSize));
331
332 satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
333
334 // Restore possibly live regs.
335 __ ldrd(v0, __ post(sp, 2 * wordSize));
336 __ pop(live_regs, sp);
337 }
338 }
339
340 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr) {
341 if (ShenandoahLoadRefBarrier) {
342 Label is_null;
343 __ cbz(dst, is_null);
344 load_reference_barrier_not_null(masm, dst, load_addr);
345 __ bind(is_null);
346 }
347 }
348
349
350 //
351 // Arguments:
352 //
353 // Inputs:
354 // src: oop location to load from, might be clobbered
355 // tmp1: unused
356 // tmp_thread: unused
357 //
358 // Output:
359 // dst: oop loaded from src location
360 //
361 // Kill:
362 // rscratch1 (scratch reg)
363 //
364 // Alias:
365 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
366 //
367 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
368 Register dst, Address src, Register tmp1, Register tmp_thread) {
369
370 Register result_dst = dst;
371 bool need_load_reference_barrier = ShenandoahBarrierSet::need_load_reference_barrier(decorators, type);
372
373 // Only preserve src address if we need load reference barrier
374 if (need_load_reference_barrier) {
375 // Use rscratch1 as temporary output register to avoid clobbering src
376 if (dst == src.base() || dst == src.index()) {
377 dst = rscratch1;
378 }
379 assert_different_registers(dst, src.base(), src.index());
380 }
381
382 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
383 if (!need_load_reference_barrier) {
384 return;
385 }
386
387 if (ShenandoahBarrierSet::use_native_load_reference_barrier(decorators, type)) {
388 load_reference_barrier_native(masm, dst, src);
389 } else {
390 load_reference_barrier(masm, dst, src);
391 }
392
393 // Move loaded oop to final destination
394 if (dst != result_dst) {
395 __ mov(result_dst, dst);
396 dst = result_dst;
397 }
398
399 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
400 __ enter();
401 satb_write_barrier_pre(masm /* masm */,
402 noreg /* obj */,
403 dst /* pre_val */,
404 rthread /* thread */,
405 tmp1 /* tmp */,
406 true /* tosca_live */,
407 true /* expand_call */);
408 __ leave();
409 }
410 }
411
412 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
413 Address dst, Register val, Register tmp1, Register tmp2) {
414 bool on_oop = is_reference_type(type);
415 if (!on_oop) {
416 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
417 return;
418 }
419
420 // flatten object address if needed
421 if (dst.index() == noreg && dst.offset() == 0) {
422 if (dst.base() != r3) {
423 __ mov(r3, dst.base());
424 }
425 } else {
426 __ lea(r3, dst);
427 }
428
|