31 #include "gc/shared/collectedHeap.inline.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/klass.inline.hpp"
36 #include "prims/methodHandles.hpp"
37 #include "runtime/biasedLocking.hpp"
38 #include "runtime/interfaceSupport.inline.hpp"
39 #include "runtime/jniHandles.inline.hpp"
40 #include "runtime/objectMonitor.hpp"
41 #include "runtime/os.inline.hpp"
42 #include "runtime/safepoint.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/align.hpp"
47 #include "utilities/macros.hpp"
48 #if INCLUDE_ALL_GCS
49 #include "gc/g1/g1BarrierSet.hpp"
50 #include "gc/g1/g1CardTable.hpp"
51 #include "gc/g1/heapRegion.hpp"
52 #endif // INCLUDE_ALL_GCS
53 #ifdef COMPILER2
54 #include "opto/intrinsicnode.hpp"
55 #endif
56
57 #ifdef PRODUCT
58 #define BLOCK_COMMENT(str) /* nothing */
59 #define STOP(error) stop(error)
60 #else
61 #define BLOCK_COMMENT(str) block_comment(str)
62 #define STOP(error) block_comment(error); stop(error)
63 #endif
64
65 // Convert the raw encoding form into the form expected by the
66 // constructor for Address.
67 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
68 assert(scale == 0, "not supported");
69 RelocationHolder rspec;
70 if (disp_reloc != relocInfo::none) {
3399 static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions?
3400
3401 static void generate_satb_log_enqueue(bool with_frame) {
3402 BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize);
3403 CodeBuffer buf(bb);
3404 MacroAssembler masm(&buf);
3405
3406 #define __ masm.
3407
3408 address start = __ pc();
3409 Register pre_val;
3410
3411 Label refill, restart;
3412 if (with_frame) {
3413 __ save_frame(0);
3414 pre_val = I0; // Was O0 before the save.
3415 } else {
3416 pre_val = O0;
3417 }
3418
3419 int satb_q_index_byte_offset =
3420 in_bytes(JavaThread::satb_mark_queue_offset() +
3421 SATBMarkQueue::byte_offset_of_index());
3422
3423 int satb_q_buf_byte_offset =
3424 in_bytes(JavaThread::satb_mark_queue_offset() +
3425 SATBMarkQueue::byte_offset_of_buf());
3426
3427 assert(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t) &&
3428 in_bytes(SATBMarkQueue::byte_width_of_buf()) == sizeof(intptr_t),
3429 "check sizes in assembly below");
3430
3431 __ bind(restart);
3432
3433 // Load the index into the SATB buffer. SATBMarkQueue::_index is a size_t
3434 // so ld_ptr is appropriate.
3435 __ ld_ptr(G2_thread, satb_q_index_byte_offset, L0);
3436
3437 // index == 0?
3438 __ cmp_and_brx_short(L0, G0, Assembler::equal, Assembler::pn, refill);
3439
3440 __ ld_ptr(G2_thread, satb_q_buf_byte_offset, L1);
3441 __ sub(L0, oopSize, L0);
3442
3443 __ st_ptr(pre_val, L1, L0); // [_buf + index] := I0
3444 if (!with_frame) {
3445 // Use return-from-leaf
3492 int offset,
3493 Register pre_val,
3494 Register tmp,
3495 bool preserve_o_regs) {
3496 Label filtered;
3497
3498 if (obj == noreg) {
3499 // We are not loading the previous value so make
3500 // sure that we don't trash the value in pre_val
3501 // with the code below.
3502 assert_different_registers(pre_val, tmp);
3503 } else {
3504 // We will be loading the previous value
3505 // in this code so...
3506 assert(offset == 0 || index == noreg, "choose one");
3507 assert(pre_val == noreg, "check this code");
3508 }
3509
3510 // Is marking active?
3511 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
3512 ld(G2,
3513 in_bytes(JavaThread::satb_mark_queue_offset() +
3514 SATBMarkQueue::byte_offset_of_active()),
3515 tmp);
3516 } else {
3517 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1,
3518 "Assumption");
3519 ldsb(G2,
3520 in_bytes(JavaThread::satb_mark_queue_offset() +
3521 SATBMarkQueue::byte_offset_of_active()),
3522 tmp);
3523 }
3524
3525 // Is marking active?
3526 cmp_and_br_short(tmp, G0, Assembler::equal, Assembler::pt, filtered);
3527
3528 // Do we need to load the previous value?
3529 if (obj != noreg) {
3530 // Load the previous value...
3531 if (index == noreg) {
3532 if (Assembler::is_simm13(offset)) {
3533 load_heap_oop(obj, offset, tmp);
3534 } else {
3535 set(offset, tmp);
3536 load_heap_oop(obj, tmp, tmp);
3537 }
3538 } else {
3539 load_heap_oop(obj, index, tmp);
3540 }
3541 // Previous value has been loaded into tmp
3542 pre_val = tmp;
3592 __ ldub(O0, O1, O2); // O2 := [O0 + O1]
3593
3594 assert(CardTable::dirty_card_val() == 0, "otherwise check this code");
3595 __ cmp_and_br_short(O2, G0, Assembler::notEqual, Assembler::pt, not_already_dirty);
3596
3597 __ bind(young_card);
3598 // We didn't take the branch, so we're already dirty: return.
3599 // Use return-from-leaf
3600 __ retl();
3601 __ delayed()->nop();
3602
3603 // Not dirty.
3604 __ bind(not_already_dirty);
3605
3606 // Get O0 + O1 into a reg by itself
3607 __ add(O0, O1, O3);
3608
3609 // First, dirty it.
3610 __ stb(G0, O3, G0); // [cardPtr] := 0 (i.e., dirty).
3611
3612 int dirty_card_q_index_byte_offset =
3613 in_bytes(JavaThread::dirty_card_queue_offset() +
3614 DirtyCardQueue::byte_offset_of_index());
3615 int dirty_card_q_buf_byte_offset =
3616 in_bytes(JavaThread::dirty_card_queue_offset() +
3617 DirtyCardQueue::byte_offset_of_buf());
3618 __ bind(restart);
3619
3620 // Load the index into the update buffer. DirtyCardQueue::_index is
3621 // a size_t so ld_ptr is appropriate here.
3622 __ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, L0);
3623
3624 // index == 0?
3625 __ cmp_and_brx_short(L0, G0, Assembler::equal, Assembler::pn, refill);
3626
3627 __ ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, L1);
3628 __ sub(L0, oopSize, L0);
3629
3630 __ st_ptr(O3, L1, L0); // [_buf + index] := I0
3631 // Use return-from-leaf
3632 __ retl();
3633 __ delayed()->st_ptr(L0, G2_thread, dirty_card_q_index_byte_offset);
3634
3635 __ bind(refill);
3636 address handle_zero =
3637 CAST_FROM_FN_PTR(address,
|
31 #include "gc/shared/collectedHeap.inline.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "memory/universe.hpp"
35 #include "oops/klass.inline.hpp"
36 #include "prims/methodHandles.hpp"
37 #include "runtime/biasedLocking.hpp"
38 #include "runtime/interfaceSupport.inline.hpp"
39 #include "runtime/jniHandles.inline.hpp"
40 #include "runtime/objectMonitor.hpp"
41 #include "runtime/os.inline.hpp"
42 #include "runtime/safepoint.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/align.hpp"
47 #include "utilities/macros.hpp"
48 #if INCLUDE_ALL_GCS
49 #include "gc/g1/g1BarrierSet.hpp"
50 #include "gc/g1/g1CardTable.hpp"
51 #include "gc/g1/g1ThreadLocalData.hpp"
52 #include "gc/g1/heapRegion.hpp"
53 #endif // INCLUDE_ALL_GCS
54 #ifdef COMPILER2
55 #include "opto/intrinsicnode.hpp"
56 #endif
57
58 #ifdef PRODUCT
59 #define BLOCK_COMMENT(str) /* nothing */
60 #define STOP(error) stop(error)
61 #else
62 #define BLOCK_COMMENT(str) block_comment(str)
63 #define STOP(error) block_comment(error); stop(error)
64 #endif
65
66 // Convert the raw encoding form into the form expected by the
67 // constructor for Address.
68 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
69 assert(scale == 0, "not supported");
70 RelocationHolder rspec;
71 if (disp_reloc != relocInfo::none) {
3400 static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions?
3401
3402 static void generate_satb_log_enqueue(bool with_frame) {
3403 BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize);
3404 CodeBuffer buf(bb);
3405 MacroAssembler masm(&buf);
3406
3407 #define __ masm.
3408
3409 address start = __ pc();
3410 Register pre_val;
3411
3412 Label refill, restart;
3413 if (with_frame) {
3414 __ save_frame(0);
3415 pre_val = I0; // Was O0 before the save.
3416 } else {
3417 pre_val = O0;
3418 }
3419
3420 int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
3421 int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
3422
3423 assert(in_bytes(SATBMarkQueue::byte_width_of_index()) == sizeof(intptr_t) &&
3424 in_bytes(SATBMarkQueue::byte_width_of_buf()) == sizeof(intptr_t),
3425 "check sizes in assembly below");
3426
3427 __ bind(restart);
3428
3429 // Load the index into the SATB buffer. SATBMarkQueue::_index is a size_t
3430 // so ld_ptr is appropriate.
3431 __ ld_ptr(G2_thread, satb_q_index_byte_offset, L0);
3432
3433 // index == 0?
3434 __ cmp_and_brx_short(L0, G0, Assembler::equal, Assembler::pn, refill);
3435
3436 __ ld_ptr(G2_thread, satb_q_buf_byte_offset, L1);
3437 __ sub(L0, oopSize, L0);
3438
3439 __ st_ptr(pre_val, L1, L0); // [_buf + index] := I0
3440 if (!with_frame) {
3441 // Use return-from-leaf
3488 int offset,
3489 Register pre_val,
3490 Register tmp,
3491 bool preserve_o_regs) {
3492 Label filtered;
3493
3494 if (obj == noreg) {
3495 // We are not loading the previous value so make
3496 // sure that we don't trash the value in pre_val
3497 // with the code below.
3498 assert_different_registers(pre_val, tmp);
3499 } else {
3500 // We will be loading the previous value
3501 // in this code so...
3502 assert(offset == 0 || index == noreg, "choose one");
3503 assert(pre_val == noreg, "check this code");
3504 }
3505
3506 // Is marking active?
3507 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
3508 ld(G2, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), tmp);
3509 } else {
3510 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
3511 ldsb(G2, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), tmp);
3512 }
3513
3514 // Is marking active?
3515 cmp_and_br_short(tmp, G0, Assembler::equal, Assembler::pt, filtered);
3516
3517 // Do we need to load the previous value?
3518 if (obj != noreg) {
3519 // Load the previous value...
3520 if (index == noreg) {
3521 if (Assembler::is_simm13(offset)) {
3522 load_heap_oop(obj, offset, tmp);
3523 } else {
3524 set(offset, tmp);
3525 load_heap_oop(obj, tmp, tmp);
3526 }
3527 } else {
3528 load_heap_oop(obj, index, tmp);
3529 }
3530 // Previous value has been loaded into tmp
3531 pre_val = tmp;
3581 __ ldub(O0, O1, O2); // O2 := [O0 + O1]
3582
3583 assert(CardTable::dirty_card_val() == 0, "otherwise check this code");
3584 __ cmp_and_br_short(O2, G0, Assembler::notEqual, Assembler::pt, not_already_dirty);
3585
3586 __ bind(young_card);
3587 // We didn't take the branch, so we're already dirty: return.
3588 // Use return-from-leaf
3589 __ retl();
3590 __ delayed()->nop();
3591
3592 // Not dirty.
3593 __ bind(not_already_dirty);
3594
3595 // Get O0 + O1 into a reg by itself
3596 __ add(O0, O1, O3);
3597
3598 // First, dirty it.
3599 __ stb(G0, O3, G0); // [cardPtr] := 0 (i.e., dirty).
3600
3601 int dirty_card_q_index_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
3602 int dirty_card_q_buf_byte_offset = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
3603 __ bind(restart);
3604
3605 // Load the index into the update buffer. DirtyCardQueue::_index is
3606 // a size_t so ld_ptr is appropriate here.
3607 __ ld_ptr(G2_thread, dirty_card_q_index_byte_offset, L0);
3608
3609 // index == 0?
3610 __ cmp_and_brx_short(L0, G0, Assembler::equal, Assembler::pn, refill);
3611
3612 __ ld_ptr(G2_thread, dirty_card_q_buf_byte_offset, L1);
3613 __ sub(L0, oopSize, L0);
3614
3615 __ st_ptr(O3, L1, L0); // [_buf + index] := I0
3616 // Use return-from-leaf
3617 __ retl();
3618 __ delayed()->st_ptr(L0, G2_thread, dirty_card_q_index_byte_offset);
3619
3620 __ bind(refill);
3621 address handle_zero =
3622 CAST_FROM_FN_PTR(address,
|