986 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) { 988 if (reachable(src)) { 989 Assembler::addpd(dst, as_Address(src)); 990 } else { 991 lea(rscratch1, src); 992 Assembler::addpd(dst, Address(rscratch1, 0)); 993 } 994 } 995 996 void MacroAssembler::align(int modulus) { 997 align(modulus, offset()); 998 } 999 1000 void MacroAssembler::align(int modulus, int target) { 1001 if (target % modulus != 0) { 1002 nop(modulus - (target % modulus)); 1003 } 1004 } 1005 1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) { 1007 // Used in sign-masking with aligned address. 1008 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1009 if (reachable(src)) { 1010 Assembler::andpd(dst, as_Address(src)); 1011 } else { 1012 lea(rscratch1, src); 1013 Assembler::andpd(dst, Address(rscratch1, 0)); 1014 } 1015 } 1016 1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) { 1018 // Used in sign-masking with aligned address. 1019 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1020 if (reachable(src)) { 1021 Assembler::andps(dst, as_Address(src)); 1022 } else { 1023 lea(rscratch1, src); 1024 Assembler::andps(dst, Address(rscratch1, 0)); 1025 } 1026 } 1027 1028 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1029 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1030 } 1031 1032 void MacroAssembler::atomic_incl(Address counter_addr) { 1033 lock(); 1034 incrementl(counter_addr); 1035 } 1036 1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { 1038 if (reachable(counter_addr)) { 1039 atomic_incl(as_Address(counter_addr)); 1040 } else { 1041 lea(scr, counter_addr); 1042 atomic_incl(Address(scr, 0)); 1043 } 1044 } 3323 lea(scratchReg, src); 3324 movdqu(dst, Address(scratchReg, 0)); 3325 } 3326 } 3327 3328 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 3329 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3330 Assembler::vmovdqu(dst, src); 3331 } 3332 3333 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 3334 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3335 Assembler::vmovdqu(dst, src); 3336 } 3337 3338 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 3339 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3340 Assembler::vmovdqu(dst, src); 3341 } 3342 3343 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) { 3344 if (reachable(src)) { 3345 vmovdqu(dst, as_Address(src)); 3346 } 3347 else { 3348 lea(rscratch1, src); 3349 vmovdqu(dst, Address(rscratch1, 0)); 3350 } 3351 } 3352 3353 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3354 if (reachable(src)) { 3355 Assembler::evmovdquq(dst, as_Address(src), vector_len); 3356 } else { 3357 lea(rscratch, src); 3358 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 3359 } 3360 } 3361 3362 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) { 3363 if (reachable(src)) { 3364 Assembler::movdqa(dst, as_Address(src)); 3365 } else { 3366 lea(rscratch1, src); 3367 Assembler::movdqa(dst, Address(rscratch1, 0)); 3368 } 3369 } 3681 } 3682 3683 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { 3684 if (reachable(src)) { 3685 Assembler::ucomisd(dst, as_Address(src)); 3686 } else { 3687 lea(rscratch1, src); 3688 Assembler::ucomisd(dst, Address(rscratch1, 0)); 3689 } 3690 } 3691 3692 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { 3693 if (reachable(src)) { 3694 Assembler::ucomiss(dst, as_Address(src)); 3695 } else { 3696 lea(rscratch1, src); 3697 Assembler::ucomiss(dst, Address(rscratch1, 0)); 3698 } 3699 } 3700 3701 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) { 3702 // Used in sign-bit flipping with aligned address. 3703 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3704 if (reachable(src)) { 3705 Assembler::xorpd(dst, as_Address(src)); 3706 } else { 3707 lea(rscratch1, src); 3708 Assembler::xorpd(dst, Address(rscratch1, 0)); 3709 } 3710 } 3711 3712 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3713 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3714 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3715 } 3716 else { 3717 Assembler::xorpd(dst, src); 3718 } 3719 } 3720 3721 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3722 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3723 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3724 } else { 3725 Assembler::xorps(dst, src); 3726 } 3727 } 3728 3729 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) { 3730 // Used in sign-bit flipping with aligned address. 3731 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3732 if (reachable(src)) { 3733 Assembler::xorps(dst, as_Address(src)); 3734 } else { 3735 lea(rscratch1, src); 3736 Assembler::xorps(dst, Address(rscratch1, 0)); 3737 } 3738 } 3739 3740 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { 3741 // Used in sign-bit flipping with aligned address. 3742 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3743 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3744 if (reachable(src)) { 3745 Assembler::pshufb(dst, as_Address(src)); 3746 } else { 3747 lea(rscratch1, src); 3748 Assembler::pshufb(dst, Address(rscratch1, 0)); 3749 } 3750 } 3751 3752 // AVX 3-operands instructions 3753 3754 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3755 if (reachable(src)) { 3756 vaddsd(dst, nds, as_Address(src)); 3782 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3783 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3784 Assembler::vpaddb(dst, nds, src, vector_len); 3785 } 3786 3787 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3788 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3789 Assembler::vpaddb(dst, nds, src, vector_len); 3790 } 3791 3792 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3793 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3794 Assembler::vpaddw(dst, nds, src, vector_len); 3795 } 3796 3797 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3798 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3799 Assembler::vpaddw(dst, nds, src, vector_len); 3800 } 3801 3802 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3803 if (reachable(src)) { 3804 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3805 } else { 3806 lea(rscratch1, src); 3807 Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len); 3808 } 3809 } 3810 3811 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { 3812 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3813 Assembler::vpbroadcastw(dst, src, vector_len); 3814 } 3815 3816 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3817 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3818 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3819 } 3820 3821 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3822 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3823 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3824 } 3825 3826 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3827 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3856 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3857 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3858 Assembler::vpsubw(dst, nds, src, vector_len); 3859 } 3860 3861 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3862 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3863 Assembler::vpsubw(dst, nds, src, vector_len); 3864 } 3865 3866 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3867 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3868 Assembler::vpsraw(dst, nds, shift, vector_len); 3869 } 3870 3871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3872 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3873 Assembler::vpsraw(dst, nds, shift, vector_len); 3874 } 3875 3876 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3877 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3878 Assembler::vpsrlw(dst, nds, shift, vector_len); 3879 } 3880 3881 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3882 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3883 Assembler::vpsrlw(dst, nds, shift, vector_len); 3884 } 3885 3886 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3887 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3888 Assembler::vpsllw(dst, nds, shift, vector_len); 3889 } 3890 3891 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3892 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3893 Assembler::vpsllw(dst, nds, shift, vector_len); 3894 } 3895 3896 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3897 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3898 Assembler::vptest(dst, src); 3899 } 3900 3901 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3902 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3903 Assembler::punpcklbw(dst, src); 3904 } 3905 3906 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3907 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3908 Assembler::pshufd(dst, src, mode); 3909 } 3910 3911 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3912 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3913 Assembler::pshuflw(dst, src, mode); 3914 } 3915 3916 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3917 if (reachable(src)) { 3918 vandpd(dst, nds, as_Address(src), vector_len); 3919 } else { 3920 lea(rscratch1, src); 3921 vandpd(dst, nds, Address(rscratch1, 0), vector_len); 3922 } 3923 } 3924 3925 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3926 if (reachable(src)) { 3927 vandps(dst, nds, as_Address(src), vector_len); 3928 } else { 3929 lea(rscratch1, src); 3930 vandps(dst, nds, Address(rscratch1, 0), vector_len); 3931 } 3932 } 3933 3934 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3935 if (reachable(src)) { 3936 vdivsd(dst, nds, as_Address(src)); 3937 } else { 3938 lea(rscratch1, src); 3939 vdivsd(dst, nds, Address(rscratch1, 0)); 3940 } 3941 } 3942 3943 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3944 if (reachable(src)) { 3945 vdivss(dst, nds, as_Address(src)); 3946 } else { 3947 lea(rscratch1, src); 3948 vdivss(dst, nds, Address(rscratch1, 0)); 3949 } 3950 } 3978 3979 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3980 if (reachable(src)) { 3981 vsubss(dst, nds, as_Address(src)); 3982 } else { 3983 lea(rscratch1, src); 3984 vsubss(dst, nds, Address(rscratch1, 0)); 3985 } 3986 } 3987 3988 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3989 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3990 vxorps(dst, nds, src, Assembler::AVX_128bit); 3991 } 3992 3993 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3994 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3995 vxorpd(dst, nds, src, Assembler::AVX_128bit); 3996 } 3997 3998 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3999 if (reachable(src)) { 4000 vxorpd(dst, nds, as_Address(src), vector_len); 4001 } else { 4002 lea(rscratch1, src); 4003 vxorpd(dst, nds, Address(rscratch1, 0), vector_len); 4004 } 4005 } 4006 4007 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 4008 if (reachable(src)) { 4009 vxorps(dst, nds, as_Address(src), vector_len); 4010 } else { 4011 lea(rscratch1, src); 4012 vxorps(dst, nds, Address(rscratch1, 0), vector_len); 4013 } 4014 } 4015 4016 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) { 4017 const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask); 4018 STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code 4019 // The inverted mask is sign-extended 4020 andptr(possibly_jweak, inverted_jweak_mask); 4021 } 4022 4023 void MacroAssembler::resolve_jobject(Register value, 4024 Register thread, 4025 Register tmp) { 4026 assert_different_registers(value, thread, tmp); 4027 Label done, not_weak; 4028 testptr(value, value); 4029 jcc(Assembler::zero, done); // Use NULL as-is. 4030 testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag. 4031 jcc(Assembler::zero, not_weak); 4032 // Resolve jweak. | 986 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) { 988 if (reachable(src)) { 989 Assembler::addpd(dst, as_Address(src)); 990 } else { 991 lea(rscratch1, src); 992 Assembler::addpd(dst, Address(rscratch1, 0)); 993 } 994 } 995 996 void MacroAssembler::align(int modulus) { 997 align(modulus, offset()); 998 } 999 1000 void MacroAssembler::align(int modulus, int target) { 1001 if (target % modulus != 0) { 1002 nop(modulus - (target % modulus)); 1003 } 1004 } 1005 1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 1007 // Used in sign-masking with aligned address. 1008 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1009 if (reachable(src)) { 1010 Assembler::andpd(dst, as_Address(src)); 1011 } else { 1012 lea(scratch_reg, src); 1013 Assembler::andpd(dst, Address(scratch_reg, 0)); 1014 } 1015 } 1016 1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 1018 // Used in sign-masking with aligned address. 1019 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1020 if (reachable(src)) { 1021 Assembler::andps(dst, as_Address(src)); 1022 } else { 1023 lea(scratch_reg, src); 1024 Assembler::andps(dst, Address(scratch_reg, 0)); 1025 } 1026 } 1027 1028 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1029 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1030 } 1031 1032 void MacroAssembler::atomic_incl(Address counter_addr) { 1033 lock(); 1034 incrementl(counter_addr); 1035 } 1036 1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { 1038 if (reachable(counter_addr)) { 1039 atomic_incl(as_Address(counter_addr)); 1040 } else { 1041 lea(scr, counter_addr); 1042 atomic_incl(Address(scr, 0)); 1043 } 1044 } 3323 lea(scratchReg, src); 3324 movdqu(dst, Address(scratchReg, 0)); 3325 } 3326 } 3327 3328 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 3329 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3330 Assembler::vmovdqu(dst, src); 3331 } 3332 3333 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 3334 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3335 Assembler::vmovdqu(dst, src); 3336 } 3337 3338 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 3339 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3340 Assembler::vmovdqu(dst, src); 3341 } 3342 3343 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 3344 if (reachable(src)) { 3345 vmovdqu(dst, as_Address(src)); 3346 } 3347 else { 3348 lea(scratch_reg, src); 3349 vmovdqu(dst, Address(scratch_reg, 0)); 3350 } 3351 } 3352 3353 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3354 if (reachable(src)) { 3355 Assembler::evmovdquq(dst, as_Address(src), vector_len); 3356 } else { 3357 lea(rscratch, src); 3358 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 3359 } 3360 } 3361 3362 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) { 3363 if (reachable(src)) { 3364 Assembler::movdqa(dst, as_Address(src)); 3365 } else { 3366 lea(rscratch1, src); 3367 Assembler::movdqa(dst, Address(rscratch1, 0)); 3368 } 3369 } 3681 } 3682 3683 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { 3684 if (reachable(src)) { 3685 Assembler::ucomisd(dst, as_Address(src)); 3686 } else { 3687 lea(rscratch1, src); 3688 Assembler::ucomisd(dst, Address(rscratch1, 0)); 3689 } 3690 } 3691 3692 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { 3693 if (reachable(src)) { 3694 Assembler::ucomiss(dst, as_Address(src)); 3695 } else { 3696 lea(rscratch1, src); 3697 Assembler::ucomiss(dst, Address(rscratch1, 0)); 3698 } 3699 } 3700 3701 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 3702 // Used in sign-bit flipping with aligned address. 3703 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3704 if (reachable(src)) { 3705 Assembler::xorpd(dst, as_Address(src)); 3706 } else { 3707 lea(scratch_reg, src); 3708 Assembler::xorpd(dst, Address(scratch_reg, 0)); 3709 } 3710 } 3711 3712 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3713 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3714 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3715 } 3716 else { 3717 Assembler::xorpd(dst, src); 3718 } 3719 } 3720 3721 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3722 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3723 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3724 } else { 3725 Assembler::xorps(dst, src); 3726 } 3727 } 3728 3729 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 3730 // Used in sign-bit flipping with aligned address. 3731 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3732 if (reachable(src)) { 3733 Assembler::xorps(dst, as_Address(src)); 3734 } else { 3735 lea(scratch_reg, src); 3736 Assembler::xorps(dst, Address(scratch_reg, 0)); 3737 } 3738 } 3739 3740 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { 3741 // Used in sign-bit flipping with aligned address. 3742 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3743 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3744 if (reachable(src)) { 3745 Assembler::pshufb(dst, as_Address(src)); 3746 } else { 3747 lea(rscratch1, src); 3748 Assembler::pshufb(dst, Address(rscratch1, 0)); 3749 } 3750 } 3751 3752 // AVX 3-operands instructions 3753 3754 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3755 if (reachable(src)) { 3756 vaddsd(dst, nds, as_Address(src)); 3782 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3783 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3784 Assembler::vpaddb(dst, nds, src, vector_len); 3785 } 3786 3787 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3788 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3789 Assembler::vpaddb(dst, nds, src, vector_len); 3790 } 3791 3792 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3793 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3794 Assembler::vpaddw(dst, nds, src, vector_len); 3795 } 3796 3797 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3798 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3799 Assembler::vpaddw(dst, nds, src, vector_len); 3800 } 3801 3802 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 3803 if (reachable(src)) { 3804 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3805 } else { 3806 lea(scratch_reg, src); 3807 Assembler::vpand(dst, nds, Address(scratch_reg, 0), vector_len); 3808 } 3809 } 3810 3811 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { 3812 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3813 Assembler::vpbroadcastw(dst, src, vector_len); 3814 } 3815 3816 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3817 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3818 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3819 } 3820 3821 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3822 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3823 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3824 } 3825 3826 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3827 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3856 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3857 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3858 Assembler::vpsubw(dst, nds, src, vector_len); 3859 } 3860 3861 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3862 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3863 Assembler::vpsubw(dst, nds, src, vector_len); 3864 } 3865 3866 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3867 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3868 Assembler::vpsraw(dst, nds, shift, vector_len); 3869 } 3870 3871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3872 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3873 Assembler::vpsraw(dst, nds, shift, vector_len); 3874 } 3875 3876 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3877 assert(UseAVX > 2,""); 3878 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3879 vector_len = 2; 3880 } 3881 Assembler::evpsraq(dst, nds, shift, vector_len); 3882 } 3883 3884 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3885 assert(UseAVX > 2,""); 3886 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3887 vector_len = 2; 3888 } 3889 Assembler::evpsraq(dst, nds, shift, vector_len); 3890 } 3891 3892 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3893 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3894 Assembler::vpsrlw(dst, nds, shift, vector_len); 3895 } 3896 3897 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3898 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3899 Assembler::vpsrlw(dst, nds, shift, vector_len); 3900 } 3901 3902 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3903 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3904 Assembler::vpsllw(dst, nds, shift, vector_len); 3905 } 3906 3907 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3908 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3909 Assembler::vpsllw(dst, nds, shift, vector_len); 3910 } 3911 3912 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3913 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3914 Assembler::vptest(dst, src); 3915 } 3916 3917 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3918 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3919 Assembler::punpcklbw(dst, src); 3920 } 3921 3922 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3923 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3924 Assembler::pshufd(dst, src, mode); 3925 } 3926 3927 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3928 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3929 Assembler::pshuflw(dst, src, mode); 3930 } 3931 3932 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 3933 if (reachable(src)) { 3934 vandpd(dst, nds, as_Address(src), vector_len); 3935 } else { 3936 lea(scratch_reg, src); 3937 vandpd(dst, nds, Address(scratch_reg, 0), vector_len); 3938 } 3939 } 3940 3941 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 3942 if (reachable(src)) { 3943 vandps(dst, nds, as_Address(src), vector_len); 3944 } else { 3945 lea(scratch_reg, src); 3946 vandps(dst, nds, Address(scratch_reg, 0), vector_len); 3947 } 3948 } 3949 3950 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3951 if (reachable(src)) { 3952 vdivsd(dst, nds, as_Address(src)); 3953 } else { 3954 lea(rscratch1, src); 3955 vdivsd(dst, nds, Address(rscratch1, 0)); 3956 } 3957 } 3958 3959 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3960 if (reachable(src)) { 3961 vdivss(dst, nds, as_Address(src)); 3962 } else { 3963 lea(rscratch1, src); 3964 vdivss(dst, nds, Address(rscratch1, 0)); 3965 } 3966 } 3994 3995 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3996 if (reachable(src)) { 3997 vsubss(dst, nds, as_Address(src)); 3998 } else { 3999 lea(rscratch1, src); 4000 vsubss(dst, nds, Address(rscratch1, 0)); 4001 } 4002 } 4003 4004 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 4005 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 4006 vxorps(dst, nds, src, Assembler::AVX_128bit); 4007 } 4008 4009 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 4010 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 4011 vxorpd(dst, nds, src, Assembler::AVX_128bit); 4012 } 4013 4014 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 4015 if (reachable(src)) { 4016 vxorpd(dst, nds, as_Address(src), vector_len); 4017 } else { 4018 lea(scratch_reg, src); 4019 vxorpd(dst, nds, Address(scratch_reg, 0), vector_len); 4020 } 4021 } 4022 4023 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 4024 if (reachable(src)) { 4025 vxorps(dst, nds, as_Address(src), vector_len); 4026 } else { 4027 lea(scratch_reg, src); 4028 vxorps(dst, nds, Address(scratch_reg, 0), vector_len); 4029 } 4030 } 4031 4032 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 4033 if (UseAVX > 1 || (vector_len < 1)) { 4034 if (reachable(src)) { 4035 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 4036 } else { 4037 lea(scratch_reg, src); 4038 Assembler::vpxor(dst, nds, Address(scratch_reg, 0), vector_len); 4039 } 4040 } 4041 else { 4042 MacroAssembler::vxorpd(dst, nds, src, vector_len, scratch_reg); 4043 } 4044 } 4045 4046 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) { 4047 const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask); 4048 STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code 4049 // The inverted mask is sign-extended 4050 andptr(possibly_jweak, inverted_jweak_mask); 4051 } 4052 4053 void MacroAssembler::resolve_jobject(Register value, 4054 Register thread, 4055 Register tmp) { 4056 assert_different_registers(value, thread, tmp); 4057 Label done, not_weak; 4058 testptr(value, value); 4059 jcc(Assembler::zero, done); // Use NULL as-is. 4060 testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag. 4061 jcc(Assembler::zero, not_weak); 4062 // Resolve jweak. |