986
987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
988 if (reachable(src)) {
989 Assembler::addpd(dst, as_Address(src));
990 } else {
991 lea(rscratch1, src);
992 Assembler::addpd(dst, Address(rscratch1, 0));
993 }
994 }
995
996 void MacroAssembler::align(int modulus) {
997 align(modulus, offset());
998 }
999
1000 void MacroAssembler::align(int modulus, int target) {
1001 if (target % modulus != 0) {
1002 nop(modulus - (target % modulus));
1003 }
1004 }
1005
1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1007 // Used in sign-masking with aligned address.
1008 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1009 if (reachable(src)) {
1010 Assembler::andpd(dst, as_Address(src));
1011 } else {
1012 lea(rscratch1, src);
1013 Assembler::andpd(dst, Address(rscratch1, 0));
1014 }
1015 }
1016
1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1018 // Used in sign-masking with aligned address.
1019 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1020 if (reachable(src)) {
1021 Assembler::andps(dst, as_Address(src));
1022 } else {
1023 lea(rscratch1, src);
1024 Assembler::andps(dst, Address(rscratch1, 0));
1025 }
1026 }
1027
1028 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1029 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1030 }
1031
1032 void MacroAssembler::atomic_incl(Address counter_addr) {
1033 lock();
1034 incrementl(counter_addr);
1035 }
1036
1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1038 if (reachable(counter_addr)) {
1039 atomic_incl(as_Address(counter_addr));
1040 } else {
1041 lea(scr, counter_addr);
1042 atomic_incl(Address(scr, 0));
1043 }
1044 }
3323 lea(scratchReg, src);
3324 movdqu(dst, Address(scratchReg, 0));
3325 }
3326 }
3327
3328 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3329 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3330 Assembler::vmovdqu(dst, src);
3331 }
3332
3333 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3334 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3335 Assembler::vmovdqu(dst, src);
3336 }
3337
3338 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3339 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3340 Assembler::vmovdqu(dst, src);
3341 }
3342
3343 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3344 if (reachable(src)) {
3345 vmovdqu(dst, as_Address(src));
3346 }
3347 else {
3348 lea(rscratch1, src);
3349 vmovdqu(dst, Address(rscratch1, 0));
3350 }
3351 }
3352
3353 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
3354 if (reachable(src)) {
3355 Assembler::evmovdquq(dst, as_Address(src), vector_len);
3356 } else {
3357 lea(rscratch, src);
3358 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
3359 }
3360 }
3361
3362 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3363 if (reachable(src)) {
3364 Assembler::movdqa(dst, as_Address(src));
3365 } else {
3366 lea(rscratch1, src);
3367 Assembler::movdqa(dst, Address(rscratch1, 0));
3368 }
3369 }
3681 }
3682
3683 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
3684 if (reachable(src)) {
3685 Assembler::ucomisd(dst, as_Address(src));
3686 } else {
3687 lea(rscratch1, src);
3688 Assembler::ucomisd(dst, Address(rscratch1, 0));
3689 }
3690 }
3691
3692 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
3693 if (reachable(src)) {
3694 Assembler::ucomiss(dst, as_Address(src));
3695 } else {
3696 lea(rscratch1, src);
3697 Assembler::ucomiss(dst, Address(rscratch1, 0));
3698 }
3699 }
3700
3701 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
3702 // Used in sign-bit flipping with aligned address.
3703 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3704 if (reachable(src)) {
3705 Assembler::xorpd(dst, as_Address(src));
3706 } else {
3707 lea(rscratch1, src);
3708 Assembler::xorpd(dst, Address(rscratch1, 0));
3709 }
3710 }
3711
3712 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
3713 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3714 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3715 }
3716 else {
3717 Assembler::xorpd(dst, src);
3718 }
3719 }
3720
3721 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
3722 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3723 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3724 } else {
3725 Assembler::xorps(dst, src);
3726 }
3727 }
3728
3729 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
3730 // Used in sign-bit flipping with aligned address.
3731 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3732 if (reachable(src)) {
3733 Assembler::xorps(dst, as_Address(src));
3734 } else {
3735 lea(rscratch1, src);
3736 Assembler::xorps(dst, Address(rscratch1, 0));
3737 }
3738 }
3739
3740 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
3741 // Used in sign-bit flipping with aligned address.
3742 bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
3743 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
3744 if (reachable(src)) {
3745 Assembler::pshufb(dst, as_Address(src));
3746 } else {
3747 lea(rscratch1, src);
3748 Assembler::pshufb(dst, Address(rscratch1, 0));
3749 }
3750 }
3751
3752 // AVX 3-operands instructions
3753
3754 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3755 if (reachable(src)) {
3756 vaddsd(dst, nds, as_Address(src));
3782 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3783 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3784 Assembler::vpaddb(dst, nds, src, vector_len);
3785 }
3786
3787 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3788 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3789 Assembler::vpaddb(dst, nds, src, vector_len);
3790 }
3791
3792 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3793 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3794 Assembler::vpaddw(dst, nds, src, vector_len);
3795 }
3796
3797 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3798 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3799 Assembler::vpaddw(dst, nds, src, vector_len);
3800 }
3801
3802 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
3803 if (reachable(src)) {
3804 Assembler::vpand(dst, nds, as_Address(src), vector_len);
3805 } else {
3806 lea(rscratch1, src);
3807 Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
3808 }
3809 }
3810
3811 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
3812 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3813 Assembler::vpbroadcastw(dst, src, vector_len);
3814 }
3815
3816 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3817 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3818 Assembler::vpcmpeqb(dst, nds, src, vector_len);
3819 }
3820
3821 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3822 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3823 Assembler::vpcmpeqw(dst, nds, src, vector_len);
3824 }
3825
3826 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3827 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3856 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3857 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3858 Assembler::vpsubw(dst, nds, src, vector_len);
3859 }
3860
3861 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3862 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3863 Assembler::vpsubw(dst, nds, src, vector_len);
3864 }
3865
3866 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3867 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3868 Assembler::vpsraw(dst, nds, shift, vector_len);
3869 }
3870
3871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3872 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3873 Assembler::vpsraw(dst, nds, shift, vector_len);
3874 }
3875
3876 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3877 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3878 Assembler::vpsrlw(dst, nds, shift, vector_len);
3879 }
3880
3881 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3882 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3883 Assembler::vpsrlw(dst, nds, shift, vector_len);
3884 }
3885
3886 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3887 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3888 Assembler::vpsllw(dst, nds, shift, vector_len);
3889 }
3890
3891 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3892 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3893 Assembler::vpsllw(dst, nds, shift, vector_len);
3894 }
3895
3896 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
3897 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
3898 Assembler::vptest(dst, src);
3899 }
3900
3901 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3902 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3903 Assembler::punpcklbw(dst, src);
3904 }
3905
3906 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
3907 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3908 Assembler::pshufd(dst, src, mode);
3909 }
3910
3911 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3912 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3913 Assembler::pshuflw(dst, src, mode);
3914 }
3915
3916 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
3917 if (reachable(src)) {
3918 vandpd(dst, nds, as_Address(src), vector_len);
3919 } else {
3920 lea(rscratch1, src);
3921 vandpd(dst, nds, Address(rscratch1, 0), vector_len);
3922 }
3923 }
3924
3925 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
3926 if (reachable(src)) {
3927 vandps(dst, nds, as_Address(src), vector_len);
3928 } else {
3929 lea(rscratch1, src);
3930 vandps(dst, nds, Address(rscratch1, 0), vector_len);
3931 }
3932 }
3933
3934 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3935 if (reachable(src)) {
3936 vdivsd(dst, nds, as_Address(src));
3937 } else {
3938 lea(rscratch1, src);
3939 vdivsd(dst, nds, Address(rscratch1, 0));
3940 }
3941 }
3942
3943 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3944 if (reachable(src)) {
3945 vdivss(dst, nds, as_Address(src));
3946 } else {
3947 lea(rscratch1, src);
3948 vdivss(dst, nds, Address(rscratch1, 0));
3949 }
3950 }
3978
3979 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3980 if (reachable(src)) {
3981 vsubss(dst, nds, as_Address(src));
3982 } else {
3983 lea(rscratch1, src);
3984 vsubss(dst, nds, Address(rscratch1, 0));
3985 }
3986 }
3987
3988 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3989 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3990 vxorps(dst, nds, src, Assembler::AVX_128bit);
3991 }
3992
3993 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3994 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
3995 vxorpd(dst, nds, src, Assembler::AVX_128bit);
3996 }
3997
3998 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
3999 if (reachable(src)) {
4000 vxorpd(dst, nds, as_Address(src), vector_len);
4001 } else {
4002 lea(rscratch1, src);
4003 vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
4004 }
4005 }
4006
4007 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4008 if (reachable(src)) {
4009 vxorps(dst, nds, as_Address(src), vector_len);
4010 } else {
4011 lea(rscratch1, src);
4012 vxorps(dst, nds, Address(rscratch1, 0), vector_len);
4013 }
4014 }
4015
4016 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
4017 const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
4018 STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
4019 // The inverted mask is sign-extended
4020 andptr(possibly_jweak, inverted_jweak_mask);
4021 }
4022
4023 void MacroAssembler::resolve_jobject(Register value,
4024 Register thread,
4025 Register tmp) {
4026 assert_different_registers(value, thread, tmp);
4027 Label done, not_weak;
4028 testptr(value, value);
4029 jcc(Assembler::zero, done); // Use NULL as-is.
4030 testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
4031 jcc(Assembler::zero, not_weak);
4032 // Resolve jweak.
4033 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4034 value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
|
986
987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
988 if (reachable(src)) {
989 Assembler::addpd(dst, as_Address(src));
990 } else {
991 lea(rscratch1, src);
992 Assembler::addpd(dst, Address(rscratch1, 0));
993 }
994 }
995
996 void MacroAssembler::align(int modulus) {
997 align(modulus, offset());
998 }
999
1000 void MacroAssembler::align(int modulus, int target) {
1001 if (target % modulus != 0) {
1002 nop(modulus - (target % modulus));
1003 }
1004 }
1005
1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
1007 // Used in sign-masking with aligned address.
1008 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1009 if (reachable(src)) {
1010 Assembler::andpd(dst, as_Address(src));
1011 } else {
1012 lea(scratch_reg, src);
1013 Assembler::andpd(dst, Address(scratch_reg, 0));
1014 }
1015 }
1016
1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
1018 // Used in sign-masking with aligned address.
1019 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1020 if (reachable(src)) {
1021 Assembler::andps(dst, as_Address(src));
1022 } else {
1023 lea(scratch_reg, src);
1024 Assembler::andps(dst, Address(scratch_reg, 0));
1025 }
1026 }
1027
1028 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1029 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1030 }
1031
1032 void MacroAssembler::atomic_incl(Address counter_addr) {
1033 lock();
1034 incrementl(counter_addr);
1035 }
1036
1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1038 if (reachable(counter_addr)) {
1039 atomic_incl(as_Address(counter_addr));
1040 } else {
1041 lea(scr, counter_addr);
1042 atomic_incl(Address(scr, 0));
1043 }
1044 }
3323 lea(scratchReg, src);
3324 movdqu(dst, Address(scratchReg, 0));
3325 }
3326 }
3327
3328 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3329 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3330 Assembler::vmovdqu(dst, src);
3331 }
3332
3333 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3334 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3335 Assembler::vmovdqu(dst, src);
3336 }
3337
3338 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3339 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3340 Assembler::vmovdqu(dst, src);
3341 }
3342
3343 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
3344 if (reachable(src)) {
3345 vmovdqu(dst, as_Address(src));
3346 }
3347 else {
3348 lea(scratch_reg, src);
3349 vmovdqu(dst, Address(scratch_reg, 0));
3350 }
3351 }
3352
3353 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
3354 if (reachable(src)) {
3355 Assembler::evmovdquq(dst, as_Address(src), vector_len);
3356 } else {
3357 lea(rscratch, src);
3358 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
3359 }
3360 }
3361
3362 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3363 if (reachable(src)) {
3364 Assembler::movdqa(dst, as_Address(src));
3365 } else {
3366 lea(rscratch1, src);
3367 Assembler::movdqa(dst, Address(rscratch1, 0));
3368 }
3369 }
3681 }
3682
3683 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
3684 if (reachable(src)) {
3685 Assembler::ucomisd(dst, as_Address(src));
3686 } else {
3687 lea(rscratch1, src);
3688 Assembler::ucomisd(dst, Address(rscratch1, 0));
3689 }
3690 }
3691
3692 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
3693 if (reachable(src)) {
3694 Assembler::ucomiss(dst, as_Address(src));
3695 } else {
3696 lea(rscratch1, src);
3697 Assembler::ucomiss(dst, Address(rscratch1, 0));
3698 }
3699 }
3700
3701 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
3702 // Used in sign-bit flipping with aligned address.
3703 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3704 if (reachable(src)) {
3705 Assembler::xorpd(dst, as_Address(src));
3706 } else {
3707 lea(scratch_reg, src);
3708 Assembler::xorpd(dst, Address(scratch_reg, 0));
3709 }
3710 }
3711
3712 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
3713 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3714 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3715 }
3716 else {
3717 Assembler::xorpd(dst, src);
3718 }
3719 }
3720
3721 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
3722 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3723 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3724 } else {
3725 Assembler::xorps(dst, src);
3726 }
3727 }
3728
3729 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register scratch_reg) {
3730 // Used in sign-bit flipping with aligned address.
3731 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3732 if (reachable(src)) {
3733 Assembler::xorps(dst, as_Address(src));
3734 } else {
3735 lea(scratch_reg, src);
3736 Assembler::xorps(dst, Address(scratch_reg, 0));
3737 }
3738 }
3739
3740 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
3741 // Used in sign-bit flipping with aligned address.
3742 bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
3743 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
3744 if (reachable(src)) {
3745 Assembler::pshufb(dst, as_Address(src));
3746 } else {
3747 lea(rscratch1, src);
3748 Assembler::pshufb(dst, Address(rscratch1, 0));
3749 }
3750 }
3751
3752 // AVX 3-operands instructions
3753
3754 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3755 if (reachable(src)) {
3756 vaddsd(dst, nds, as_Address(src));
3782 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3783 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3784 Assembler::vpaddb(dst, nds, src, vector_len);
3785 }
3786
3787 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3788 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3789 Assembler::vpaddb(dst, nds, src, vector_len);
3790 }
3791
3792 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3793 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3794 Assembler::vpaddw(dst, nds, src, vector_len);
3795 }
3796
3797 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3798 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3799 Assembler::vpaddw(dst, nds, src, vector_len);
3800 }
3801
3802 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3803 if (reachable(src)) {
3804 Assembler::vpand(dst, nds, as_Address(src), vector_len);
3805 } else {
3806 lea(scratch_reg, src);
3807 Assembler::vpand(dst, nds, Address(scratch_reg, 0), vector_len);
3808 }
3809 }
3810
3811 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
3812 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3813 Assembler::vpbroadcastw(dst, src, vector_len);
3814 }
3815
3816 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3817 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3818 Assembler::vpcmpeqb(dst, nds, src, vector_len);
3819 }
3820
3821 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3822 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3823 Assembler::vpcmpeqw(dst, nds, src, vector_len);
3824 }
3825
3826 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3827 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3856 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3857 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3858 Assembler::vpsubw(dst, nds, src, vector_len);
3859 }
3860
3861 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
3862 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3863 Assembler::vpsubw(dst, nds, src, vector_len);
3864 }
3865
3866 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3867 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3868 Assembler::vpsraw(dst, nds, shift, vector_len);
3869 }
3870
3871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3872 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3873 Assembler::vpsraw(dst, nds, shift, vector_len);
3874 }
3875
3876 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3877 assert(UseAVX > 2,"");
3878 if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3879 vector_len = 2;
3880 }
3881 Assembler::evpsraq(dst, nds, shift, vector_len);
3882 }
3883
3884 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3885 assert(UseAVX > 2,"");
3886 if (!VM_Version::supports_avx512vl() && vector_len < 2) {
3887 vector_len = 2;
3888 }
3889 Assembler::evpsraq(dst, nds, shift, vector_len);
3890 }
3891
3892 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3893 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3894 Assembler::vpsrlw(dst, nds, shift, vector_len);
3895 }
3896
3897 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3898 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3899 Assembler::vpsrlw(dst, nds, shift, vector_len);
3900 }
3901
3902 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
3903 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3904 Assembler::vpsllw(dst, nds, shift, vector_len);
3905 }
3906
3907 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
3908 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3909 Assembler::vpsllw(dst, nds, shift, vector_len);
3910 }
3911
3912 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
3913 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15");
3914 Assembler::vptest(dst, src);
3915 }
3916
3917 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3918 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3919 Assembler::punpcklbw(dst, src);
3920 }
3921
3922 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
3923 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15");
3924 Assembler::pshufd(dst, src, mode);
3925 }
3926
3927 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3928 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15");
3929 Assembler::pshuflw(dst, src, mode);
3930 }
3931
3932 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3933 if (reachable(src)) {
3934 vandpd(dst, nds, as_Address(src), vector_len);
3935 } else {
3936 lea(scratch_reg, src);
3937 vandpd(dst, nds, Address(scratch_reg, 0), vector_len);
3938 }
3939 }
3940
3941 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
3942 if (reachable(src)) {
3943 vandps(dst, nds, as_Address(src), vector_len);
3944 } else {
3945 lea(scratch_reg, src);
3946 vandps(dst, nds, Address(scratch_reg, 0), vector_len);
3947 }
3948 }
3949
3950 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3951 if (reachable(src)) {
3952 vdivsd(dst, nds, as_Address(src));
3953 } else {
3954 lea(rscratch1, src);
3955 vdivsd(dst, nds, Address(rscratch1, 0));
3956 }
3957 }
3958
3959 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3960 if (reachable(src)) {
3961 vdivss(dst, nds, as_Address(src));
3962 } else {
3963 lea(rscratch1, src);
3964 vdivss(dst, nds, Address(rscratch1, 0));
3965 }
3966 }
3994
3995 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3996 if (reachable(src)) {
3997 vsubss(dst, nds, as_Address(src));
3998 } else {
3999 lea(rscratch1, src);
4000 vsubss(dst, nds, Address(rscratch1, 0));
4001 }
4002 }
4003
4004 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4005 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
4006 vxorps(dst, nds, src, Assembler::AVX_128bit);
4007 }
4008
4009 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4010 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15");
4011 vxorpd(dst, nds, src, Assembler::AVX_128bit);
4012 }
4013
4014 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
4015 if (reachable(src)) {
4016 vxorpd(dst, nds, as_Address(src), vector_len);
4017 } else {
4018 lea(scratch_reg, src);
4019 vxorpd(dst, nds, Address(scratch_reg, 0), vector_len);
4020 }
4021 }
4022
4023 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
4024 if (reachable(src)) {
4025 vxorps(dst, nds, as_Address(src), vector_len);
4026 } else {
4027 lea(scratch_reg, src);
4028 vxorps(dst, nds, Address(scratch_reg, 0), vector_len);
4029 }
4030 }
4031
4032 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) {
4033 if (UseAVX > 1 || (vector_len < 1)) {
4034 if (reachable(src)) {
4035 Assembler::vpxor(dst, nds, as_Address(src), vector_len);
4036 } else {
4037 lea(scratch_reg, src);
4038 Assembler::vpxor(dst, nds, Address(scratch_reg, 0), vector_len);
4039 }
4040 }
4041 else {
4042 MacroAssembler::vxorpd(dst, nds, src, vector_len, scratch_reg);
4043 }
4044 }
4045
4046 //-------------------------------------------------------------------------------------------
4047 #ifdef COMPILER2
4048 // Generic instructions support for use in .ad files C2 code generation
4049
4050 void MacroAssembler::vabsnegd(int opcode, XMMRegister dst, Register scr) {
4051 if (opcode == Op_AbsVD) {
4052 andpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), scr);
4053 } else {
4054 assert((opcode == Op_NegVD),"opcode should be Op_NegD");
4055 xorpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), scr);
4056 }
4057 }
4058
4059 void MacroAssembler::vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr) {
4060 if (opcode == Op_AbsVD) {
4061 vandpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), vector_len, scr);
4062 } else {
4063 assert((opcode == Op_NegVD),"opcode should be Op_NegD");
4064 vxorpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), vector_len, scr);
4065 }
4066 }
4067
4068 void MacroAssembler::vabsnegf(int opcode, XMMRegister dst, Register scr) {
4069 if (opcode == Op_AbsVF) {
4070 andps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), scr);
4071 } else {
4072 assert((opcode == Op_NegVF),"opcode should be Op_NegF");
4073 xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), scr);
4074 }
4075 }
4076
4077 void MacroAssembler::vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr) {
4078 if (opcode == Op_AbsVF) {
4079 vandps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), vector_len, scr);
4080 } else {
4081 assert((opcode == Op_NegVF),"opcode should be Op_NegF");
4082 vxorps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vector_len, scr);
4083 }
4084 }
4085
4086 void MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src) {
4087 if (sign) {
4088 pmovsxbw(dst, src);
4089 } else {
4090 pmovzxbw(dst, src);
4091 }
4092 }
4093
4094 void MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len) {
4095 if (sign) {
4096 vpmovsxbw(dst, src, vector_len);
4097 } else {
4098 vpmovzxbw(dst, src, vector_len);
4099 }
4100 }
4101
4102 void MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister src) {
4103 if (opcode == Op_RShiftVI) {
4104 psrad(dst, src);
4105 } else if (opcode == Op_LShiftVI) {
4106 pslld(dst, src);
4107 } else {
4108 assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI");
4109 psrld(dst, src);
4110 }
4111 }
4112
4113 void MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4114 if (opcode == Op_RShiftVI) {
4115 vpsrad(dst, nds, src, vector_len);
4116 } else if (opcode == Op_LShiftVI) {
4117 vpslld(dst, nds, src, vector_len);
4118 } else {
4119 assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI");
4120 vpsrld(dst, nds, src, vector_len);
4121 }
4122 }
4123
4124 void MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister src) {
4125 if ((opcode == Op_RShiftVS) || (opcode == Op_RShiftVB)) {
4126 psraw(dst, src);
4127 } else if ((opcode == Op_LShiftVS) || (opcode == Op_LShiftVB)) {
4128 psllw(dst, src);
4129 } else {
4130 assert(((opcode == Op_URShiftVS) || (opcode == Op_URShiftVB)),"opcode should be one of Op_URShiftVS or Op_URShiftVB");
4131 psrlw(dst, src);
4132 }
4133 }
4134
4135 void MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4136 if ((opcode == Op_RShiftVS) || (opcode == Op_RShiftVB)) {
4137 vpsraw(dst, nds, src, vector_len);
4138 } else if ((opcode == Op_LShiftVS) || (opcode == Op_LShiftVB)) {
4139 vpsllw(dst, nds, src, vector_len);
4140 } else {
4141 assert(((opcode == Op_URShiftVS) || (opcode == Op_URShiftVB)),"opcode should be one of Op_URShiftVS or Op_URShiftVB");
4142 vpsrlw(dst, nds, src, vector_len);
4143 }
4144 }
4145
4146 void MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister src) {
4147 if (opcode == Op_RShiftVL) {
4148 psrlq(dst, src); // using srl to implement sra on pre-avs512 systems
4149 } else if (opcode == Op_LShiftVL) {
4150 psllq(dst, src);
4151 } else {
4152 assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL");
4153 psrlq(dst, src);
4154 }
4155 }
4156
4157 void MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4158 if (opcode == Op_RShiftVL) {
4159 evpsraq(dst, nds, src, vector_len);
4160 } else if (opcode == Op_LShiftVL) {
4161 vpsllq(dst, nds, src, vector_len);
4162 } else {
4163 assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL");
4164 vpsrlq(dst, nds, src, vector_len);
4165 }
4166 }
4167 #endif
4168 //-------------------------------------------------------------------------------------------
4169
4170 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
4171 const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
4172 STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
4173 // The inverted mask is sign-extended
4174 andptr(possibly_jweak, inverted_jweak_mask);
4175 }
4176
4177 void MacroAssembler::resolve_jobject(Register value,
4178 Register thread,
4179 Register tmp) {
4180 assert_different_registers(value, thread, tmp);
4181 Label done, not_weak;
4182 testptr(value, value);
4183 jcc(Assembler::zero, done); // Use NULL as-is.
4184 testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
4185 jcc(Assembler::zero, not_weak);
4186 // Resolve jweak.
4187 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4188 value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
|