3337 3338 bind(done); 3339 } 3340 3341 void MacroAssembler::g1_write_barrier_post(Register store_addr, 3342 Register new_val, 3343 Register thread, 3344 Register tmp, 3345 Register tmp2) { 3346 #ifdef _LP64 3347 assert(thread == r15_thread, "must be"); 3348 #endif // _LP64 3349 3350 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + 3351 PtrQueue::byte_offset_of_index())); 3352 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + 3353 PtrQueue::byte_offset_of_buf())); 3354 3355 BarrierSet* bs = Universe::heap()->barrier_set(); 3356 CardTableModRefBS* ct = (CardTableModRefBS*)bs; 3357 Label done; 3358 Label runtime; 3359 3360 // Does store cross heap regions? 3361 3362 movptr(tmp, store_addr); 3363 xorptr(tmp, new_val); 3364 shrptr(tmp, HeapRegion::LogOfHRGrainBytes); 3365 jcc(Assembler::equal, done); 3366 3367 // crosses regions, storing NULL? 3368 3369 cmpptr(new_val, (int32_t) NULL_WORD); 3370 jcc(Assembler::equal, done); 3371 3372 // storing region crossing non-NULL, is card already dirty? 3373 3374 ExternalAddress cardtable((address) ct->byte_map_base); 3375 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 3376 #ifdef _LP64 3377 const Register card_addr = tmp; 3378 3379 movq(card_addr, store_addr); 3380 shrq(card_addr, CardTableModRefBS::card_shift); 3381 3382 lea(tmp2, cardtable); 3383 3384 // get the address of the card 3385 addq(card_addr, tmp2); 3386 #else 3387 const Register card_index = tmp; 3388 3389 movl(card_index, store_addr); 3390 shrl(card_index, CardTableModRefBS::card_shift); 3391 3392 Address index(noreg, card_index, Address::times_1); 3393 const Register card_addr = tmp; 3394 lea(card_addr, as_Address(ArrayAddress(cardtable, index))); 3395 #endif 3396 cmpb(Address(card_addr, 0), (int)G1SATBCardTableModRefBS::g1_young_card_val()); 3397 jcc(Assembler::equal, done); 3398 3399 membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); 3400 cmpb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val()); 3401 jcc(Assembler::equal, done); 3402 3403 3404 // storing a region crossing, non-NULL oop, card is clean. 3405 // dirty card and log. 3406 3407 movb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val()); 3408 3409 cmpl(queue_index, 0); 3410 jcc(Assembler::equal, runtime); 3411 subl(queue_index, wordSize); 3412 movptr(tmp2, buffer); 3413 #ifdef _LP64 3414 movslq(rscratch1, queue_index); 3415 addq(tmp2, rscratch1); 3416 movq(Address(tmp2, 0), card_addr); 3417 #else 3418 addl(tmp2, queue_index); 3419 movl(Address(tmp2, 0), card_index); 3420 #endif 3421 jmp(done); 3422 3423 bind(runtime); 3424 // save the live input values 3425 push(store_addr); 3426 push(new_val); 3427 #ifdef _LP64 3428 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread); 3429 #else 3430 push(thread); 3431 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); 3432 pop(thread); 3433 #endif 3434 pop(new_val); 3435 pop(store_addr); 3436 3437 bind(done); 3438 } 3439 3451 void MacroAssembler::store_check(Register obj, Address dst) { 3452 store_check(obj); 3453 } 3454 3455 3456 // split the store check operation so that other instructions can be scheduled inbetween 3457 void MacroAssembler::store_check_part_1(Register obj) { 3458 BarrierSet* bs = Universe::heap()->barrier_set(); 3459 assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); 3460 shrptr(obj, CardTableModRefBS::card_shift); 3461 } 3462 3463 void MacroAssembler::store_check_part_2(Register obj) { 3464 BarrierSet* bs = Universe::heap()->barrier_set(); 3465 assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); 3466 CardTableModRefBS* ct = (CardTableModRefBS*)bs; 3467 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 3468 3469 // The calculation for byte_map_base is as follows: 3470 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); 3471 // So this essentially converts an address to a displacement and 3472 // it will never need to be relocated. On 64bit however the value may be too 3473 // large for a 32bit displacement 3474 3475 intptr_t disp = (intptr_t) ct->byte_map_base; 3476 if (is_simm32(disp)) { 3477 Address cardtable(noreg, obj, Address::times_1, disp); 3478 movb(cardtable, 0); 3479 } else { 3480 // By doing it as an ExternalAddress disp could be converted to a rip-relative 3481 // displacement and done in a single instruction given favorable mapping and 3482 // a smarter version of as_Address. Worst case it is two instructions which 3483 // is no worse off then loading disp into a register and doing as a simple 3484 // Address() as above. 3485 // We can't do as ExternalAddress as the only style since if disp == 0 we'll 3486 // assert since NULL isn't acceptable in a reloci (see 6644928). In any case 3487 // in some cases we'll get a single instruction version. 3488 3489 ExternalAddress cardtable((address)disp); 3490 Address index(noreg, obj, Address::times_1); 3491 movb(as_Address(ArrayAddress(cardtable, index)), 0); 3492 } 3493 } 3494 3495 void MacroAssembler::subptr(Register dst, int32_t imm32) { 3496 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 3497 } 3498 3499 // Force generation of a 4 byte immediate value even if it fits into 8bit 3500 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 3501 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 3502 } 3503 3504 void MacroAssembler::subptr(Register dst, Register src) { 3505 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 3506 } 3507 3508 // C++ bool manipulation 3509 void MacroAssembler::testbool(Register dst) { | 3337 3338 bind(done); 3339 } 3340 3341 void MacroAssembler::g1_write_barrier_post(Register store_addr, 3342 Register new_val, 3343 Register thread, 3344 Register tmp, 3345 Register tmp2) { 3346 #ifdef _LP64 3347 assert(thread == r15_thread, "must be"); 3348 #endif // _LP64 3349 3350 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + 3351 PtrQueue::byte_offset_of_index())); 3352 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + 3353 PtrQueue::byte_offset_of_buf())); 3354 3355 BarrierSet* bs = Universe::heap()->barrier_set(); 3356 CardTableModRefBS* ct = (CardTableModRefBS*)bs; 3357 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 3358 3359 Label done; 3360 Label runtime; 3361 3362 // Does store cross heap regions? 3363 3364 movptr(tmp, store_addr); 3365 xorptr(tmp, new_val); 3366 shrptr(tmp, HeapRegion::LogOfHRGrainBytes); 3367 jcc(Assembler::equal, done); 3368 3369 // crosses regions, storing NULL? 3370 3371 cmpptr(new_val, (int32_t) NULL_WORD); 3372 jcc(Assembler::equal, done); 3373 3374 // storing region crossing non-NULL, is card already dirty? 3375 3376 const Register card_addr = tmp; 3377 const Register cardtable = tmp2; 3378 3379 movptr(card_addr, store_addr); 3380 shrptr(card_addr, CardTableModRefBS::card_shift); 3381 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT 3382 // a valid address and therefore is not properly handled by the relocation code. 3383 // See 8028109. 3384 movptr(cardtable, (intptr_t)ct->byte_map_base); 3385 addptr(card_addr, cardtable); 3386 3387 cmpb(Address(card_addr, 0), (int)G1SATBCardTableModRefBS::g1_young_card_val()); 3388 jcc(Assembler::equal, done); 3389 3390 membar(Assembler::Membar_mask_bits(Assembler::StoreLoad)); 3391 cmpb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val()); 3392 jcc(Assembler::equal, done); 3393 3394 3395 // storing a region crossing, non-NULL oop, card is clean. 3396 // dirty card and log. 3397 3398 movb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val()); 3399 3400 cmpl(queue_index, 0); 3401 jcc(Assembler::equal, runtime); 3402 subl(queue_index, wordSize); 3403 movptr(tmp2, buffer); 3404 #ifdef _LP64 3405 movslq(rscratch1, queue_index); 3406 addq(tmp2, rscratch1); 3407 movq(Address(tmp2, 0), card_addr); 3408 #else 3409 addl(tmp2, queue_index); 3410 movl(Address(tmp2, 0), card_addr); 3411 #endif 3412 jmp(done); 3413 3414 bind(runtime); 3415 // save the live input values 3416 push(store_addr); 3417 push(new_val); 3418 #ifdef _LP64 3419 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread); 3420 #else 3421 push(thread); 3422 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); 3423 pop(thread); 3424 #endif 3425 pop(new_val); 3426 pop(store_addr); 3427 3428 bind(done); 3429 } 3430 3442 void MacroAssembler::store_check(Register obj, Address dst) { 3443 store_check(obj); 3444 } 3445 3446 3447 // split the store check operation so that other instructions can be scheduled inbetween 3448 void MacroAssembler::store_check_part_1(Register obj) { 3449 BarrierSet* bs = Universe::heap()->barrier_set(); 3450 assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); 3451 shrptr(obj, CardTableModRefBS::card_shift); 3452 } 3453 3454 void MacroAssembler::store_check_part_2(Register obj) { 3455 BarrierSet* bs = Universe::heap()->barrier_set(); 3456 assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); 3457 CardTableModRefBS* ct = (CardTableModRefBS*)bs; 3458 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 3459 3460 // The calculation for byte_map_base is as follows: 3461 // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); 3462 // So this essentially converts an address to a displacement and it will 3463 // never need to be relocated. On 64bit however the value may be too 3464 // large for a 32bit displacement. 3465 intptr_t disp = (intptr_t) ct->byte_map_base; 3466 if (is_simm32(disp)) { 3467 Address cardtable(noreg, obj, Address::times_1, disp); 3468 movb(cardtable, 0); 3469 } else { 3470 // By doing it as an ExternalAddress 'disp' could be converted to a rip-relative 3471 // displacement and done in a single instruction given favorable mapping and a 3472 // smarter version of as_Address. However, 'ExternalAddress' generates a relocation 3473 // entry and that entry is not properly handled by the relocation code (8028109). 3474 AddressLiteral cardtable((address)ct->byte_map_base, relocInfo::none); 3475 Address index(noreg, obj, Address::times_1); 3476 movb(as_Address(ArrayAddress(cardtable, index)), 0); 3477 } 3478 } 3479 3480 void MacroAssembler::subptr(Register dst, int32_t imm32) { 3481 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); 3482 } 3483 3484 // Force generation of a 4 byte immediate value even if it fits into 8bit 3485 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { 3486 LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); 3487 } 3488 3489 void MacroAssembler::subptr(Register dst, Register src) { 3490 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); 3491 } 3492 3493 // C++ bool manipulation 3494 void MacroAssembler::testbool(Register dst) { |