3712 } else {
3713 lea(rscratch1, src);
3714 Assembler::mulsd(dst, Address(rscratch1, 0));
3715 }
3716 }
3717
3718 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3719 if (reachable(src)) {
3720 Assembler::mulss(dst, as_Address(src));
3721 } else {
3722 lea(rscratch1, src);
3723 Assembler::mulss(dst, Address(rscratch1, 0));
3724 }
3725 }
3726
3727 void MacroAssembler::null_check(Register reg, int offset) {
3728 if (needs_explicit_null_check(offset)) {
3729 // provoke OS NULL exception if reg = NULL by
3730 // accessing M[reg] w/o changing any (non-CC) registers
3731 // NOTE: cmpl is plenty here to provoke a segv
3732 cmpptr(rax, Address(reg, 0));
3733 // Note: should probably use testl(rax, Address(reg, 0));
3734 // may be shorter code (however, this version of
3735 // testl needs to be implemented first)
3736 } else {
3737 // nothing to do, (later) access of M[reg + offset]
3738 // will provoke OS NULL exception if reg = NULL
3739 }
3740 }
3741
3742 void MacroAssembler::os_breakpoint() {
3743 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3744 // (e.g., MSVC can't call ps() otherwise)
3745 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3746 }
3747
3748 void MacroAssembler::pop_CPU_state() {
3749 pop_FPU_state();
3750 pop_IU_state();
3751 }
4210 if (pre_val != rax)
4211 pop(pre_val);
4212
4213 if (obj != noreg && obj != rax)
4214 pop(obj);
4215
4216 if(tosca_live) pop(rax);
4217
4218 bind(done);
4219 }
4220
4221 void MacroAssembler::g1_write_barrier_post(Register store_addr,
4222 Register new_val,
4223 Register thread,
4224 Register tmp,
4225 Register tmp2) {
4226 #ifdef _LP64
4227 assert(thread == r15_thread, "must be");
4228 #endif // _LP64
4229
4230 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
4231 PtrQueue::byte_offset_of_index()));
4232 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
4233 PtrQueue::byte_offset_of_buf()));
4234
4235 CardTableModRefBS* ct =
4236 barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
4237 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
4238
4239 Label done;
4240 Label runtime;
4241
4242 // Does store cross heap regions?
4243
4244 movptr(tmp, store_addr);
4245 xorptr(tmp, new_val);
4246 shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
4247 jcc(Assembler::equal, done);
4248
4249 // crosses regions, storing NULL?
4394 void MacroAssembler::testptr(Register dst, Register src) {
4395 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
4396 }
4397
4398 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4399 void MacroAssembler::tlab_allocate(Register obj,
4400 Register var_size_in_bytes,
4401 int con_size_in_bytes,
4402 Register t1,
4403 Register t2,
4404 Label& slow_case) {
4405 assert_different_registers(obj, t1, t2);
4406 assert_different_registers(obj, var_size_in_bytes, t1);
4407 Register end = t2;
4408 Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
4409
4410 verify_tlab();
4411
4412 NOT_LP64(get_thread(thread));
4413
4414 movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
4415 if (var_size_in_bytes == noreg) {
4416 lea(end, Address(obj, con_size_in_bytes));
4417 } else {
4418 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
4419 }
4420 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
4421 jcc(Assembler::above, slow_case);
4422
4423 // update the tlab top pointer
4424 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
4425
4426 // recover var_size_in_bytes if necessary
4427 if (var_size_in_bytes == end) {
4428 subptr(var_size_in_bytes, obj);
4429 }
4430 verify_tlab();
4431 }
4432
4433 // Preserves rbx, and rdx.
4434 Register MacroAssembler::tlab_refill(Label& retry,
4435 Label& try_eden,
4436 Label& slow_case) {
4437 Register top = rax;
4438 Register t1 = rcx;
4439 Register t2 = rsi;
4440 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
4441 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
4442 Label do_refill, discard_tlab;
4443
4444 if (!Universe::heap()->supports_inline_contig_alloc()) {
4445 // No allocation in the shared eden.
5673 } else if (CheckJNICalls) {
5674 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5675 }
5676 }
5677 if (VM_Version::supports_avx()) {
5678 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5679 vzeroupper();
5680 }
5681
5682 #ifndef _LP64
5683 // Either restore the x87 floating pointer control word after returning
5684 // from the JNI call or verify that it wasn't changed.
5685 if (CheckJNICalls) {
5686 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
5687 }
5688 #endif // _LP64
5689 }
5690
5691
5692 void MacroAssembler::load_klass(Register dst, Register src) {
5693 #ifdef _LP64
5694 if (UseCompressedClassPointers) {
5695 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5696 decode_klass_not_null(dst);
5697 } else
5698 #endif
5699 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5700 }
5701
5702 void MacroAssembler::load_prototype_header(Register dst, Register src) {
5703 load_klass(dst, src);
5704 movptr(dst, Address(dst, Klass::prototype_header_offset()));
5705 }
5706
5707 void MacroAssembler::store_klass(Register dst, Register src) {
5708 #ifdef _LP64
5709 if (UseCompressedClassPointers) {
5710 encode_klass_not_null(src);
5711 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5712 } else
|
3712 } else {
3713 lea(rscratch1, src);
3714 Assembler::mulsd(dst, Address(rscratch1, 0));
3715 }
3716 }
3717
3718 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3719 if (reachable(src)) {
3720 Assembler::mulss(dst, as_Address(src));
3721 } else {
3722 lea(rscratch1, src);
3723 Assembler::mulss(dst, Address(rscratch1, 0));
3724 }
3725 }
3726
3727 void MacroAssembler::null_check(Register reg, int offset) {
3728 if (needs_explicit_null_check(offset)) {
3729 // provoke OS NULL exception if reg = NULL by
3730 // accessing M[reg] w/o changing any (non-CC) registers
3731 // NOTE: cmpl is plenty here to provoke a segv
3732
3733 if (ShenandoahVerifyReadsToFromSpace) {
3734 oopDesc::bs()->interpreter_read_barrier(this, reg);
3735 }
3736
3737 cmpptr(rax, Address(reg, 0));
3738 // Note: should probably use testl(rax, Address(reg, 0));
3739 // may be shorter code (however, this version of
3740 // testl needs to be implemented first)
3741 } else {
3742 // nothing to do, (later) access of M[reg + offset]
3743 // will provoke OS NULL exception if reg = NULL
3744 }
3745 }
3746
3747 void MacroAssembler::os_breakpoint() {
3748 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3749 // (e.g., MSVC can't call ps() otherwise)
3750 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3751 }
3752
3753 void MacroAssembler::pop_CPU_state() {
3754 pop_FPU_state();
3755 pop_IU_state();
3756 }
4215 if (pre_val != rax)
4216 pop(pre_val);
4217
4218 if (obj != noreg && obj != rax)
4219 pop(obj);
4220
4221 if(tosca_live) pop(rax);
4222
4223 bind(done);
4224 }
4225
4226 void MacroAssembler::g1_write_barrier_post(Register store_addr,
4227 Register new_val,
4228 Register thread,
4229 Register tmp,
4230 Register tmp2) {
4231 #ifdef _LP64
4232 assert(thread == r15_thread, "must be");
4233 #endif // _LP64
4234
4235 if (UseShenandoahGC) {
4236 // No need for this in Shenandoah.
4237 return;
4238 }
4239
4240 assert(UseG1GC, "expect G1 GC");
4241
4242 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
4243 PtrQueue::byte_offset_of_index()));
4244 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
4245 PtrQueue::byte_offset_of_buf()));
4246
4247 CardTableModRefBS* ct =
4248 barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
4249 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
4250
4251 Label done;
4252 Label runtime;
4253
4254 // Does store cross heap regions?
4255
4256 movptr(tmp, store_addr);
4257 xorptr(tmp, new_val);
4258 shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
4259 jcc(Assembler::equal, done);
4260
4261 // crosses regions, storing NULL?
4406 void MacroAssembler::testptr(Register dst, Register src) {
4407 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
4408 }
4409
4410 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4411 void MacroAssembler::tlab_allocate(Register obj,
4412 Register var_size_in_bytes,
4413 int con_size_in_bytes,
4414 Register t1,
4415 Register t2,
4416 Label& slow_case) {
4417 assert_different_registers(obj, t1, t2);
4418 assert_different_registers(obj, var_size_in_bytes, t1);
4419 Register end = t2;
4420 Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
4421
4422 verify_tlab();
4423
4424 NOT_LP64(get_thread(thread));
4425
4426 uint oop_extra_words = Universe::heap()->oop_extra_words();
4427
4428 movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
4429 if (var_size_in_bytes == noreg) {
4430 lea(end, Address(obj, con_size_in_bytes + oop_extra_words * HeapWordSize));
4431 } else {
4432 if (oop_extra_words > 0) {
4433 addq(var_size_in_bytes, oop_extra_words * HeapWordSize);
4434 }
4435 lea(end, Address(obj, var_size_in_bytes, Address::times_1));
4436 }
4437 cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
4438 jcc(Assembler::above, slow_case);
4439
4440 // update the tlab top pointer
4441 movptr(Address(thread, JavaThread::tlab_top_offset()), end);
4442
4443 Universe::heap()->compile_prepare_oop(this, obj);
4444
4445 // recover var_size_in_bytes if necessary
4446 if (var_size_in_bytes == end) {
4447 subptr(var_size_in_bytes, obj);
4448 }
4449 verify_tlab();
4450 }
4451
4452 // Preserves rbx, and rdx.
4453 Register MacroAssembler::tlab_refill(Label& retry,
4454 Label& try_eden,
4455 Label& slow_case) {
4456 Register top = rax;
4457 Register t1 = rcx;
4458 Register t2 = rsi;
4459 Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
4460 assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
4461 Label do_refill, discard_tlab;
4462
4463 if (!Universe::heap()->supports_inline_contig_alloc()) {
4464 // No allocation in the shared eden.
5692 } else if (CheckJNICalls) {
5693 call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5694 }
5695 }
5696 if (VM_Version::supports_avx()) {
5697 // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5698 vzeroupper();
5699 }
5700
5701 #ifndef _LP64
5702 // Either restore the x87 floating pointer control word after returning
5703 // from the JNI call or verify that it wasn't changed.
5704 if (CheckJNICalls) {
5705 call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
5706 }
5707 #endif // _LP64
5708 }
5709
5710
5711 void MacroAssembler::load_klass(Register dst, Register src) {
5712 if (ShenandoahVerifyReadsToFromSpace) {
5713 oopDesc::bs()->interpreter_read_barrier(this, src);
5714 }
5715 #ifdef _LP64
5716 if (UseCompressedClassPointers) {
5717 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5718 decode_klass_not_null(dst);
5719 } else
5720 #endif
5721 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5722 }
5723
5724 void MacroAssembler::load_prototype_header(Register dst, Register src) {
5725 load_klass(dst, src);
5726 movptr(dst, Address(dst, Klass::prototype_header_offset()));
5727 }
5728
5729 void MacroAssembler::store_klass(Register dst, Register src) {
5730 #ifdef _LP64
5731 if (UseCompressedClassPointers) {
5732 encode_klass_not_null(src);
5733 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5734 } else
|