986 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) { 988 if (reachable(src)) { 989 Assembler::addpd(dst, as_Address(src)); 990 } else { 991 lea(rscratch1, src); 992 Assembler::addpd(dst, Address(rscratch1, 0)); 993 } 994 } 995 996 void MacroAssembler::align(int modulus) { 997 align(modulus, offset()); 998 } 999 1000 void MacroAssembler::align(int modulus, int target) { 1001 if (target % modulus != 0) { 1002 nop(modulus - (target % modulus)); 1003 } 1004 } 1005 1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) { 1007 // Used in sign-masking with aligned address. 1008 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1009 if (reachable(src)) { 1010 Assembler::andpd(dst, as_Address(src)); 1011 } else { 1012 lea(rscratch1, src); 1013 Assembler::andpd(dst, Address(rscratch1, 0)); 1014 } 1015 } 1016 1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) { 1018 // Used in sign-masking with aligned address. 1019 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1020 if (reachable(src)) { 1021 Assembler::andps(dst, as_Address(src)); 1022 } else { 1023 lea(rscratch1, src); 1024 Assembler::andps(dst, Address(rscratch1, 0)); 1025 } 1026 } 1027 1028 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1029 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1030 } 1031 1032 void MacroAssembler::atomic_incl(Address counter_addr) { 1033 lock(); 1034 incrementl(counter_addr); 1035 } 1036 1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { 1038 if (reachable(counter_addr)) { 1039 atomic_incl(as_Address(counter_addr)); 1040 } else { 1041 lea(scr, counter_addr); 1042 atomic_incl(Address(scr, 0)); 1043 } 1044 } 3323 lea(scratchReg, src); 3324 movdqu(dst, Address(scratchReg, 0)); 3325 } 3326 } 3327 3328 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 3329 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3330 Assembler::vmovdqu(dst, src); 3331 } 3332 3333 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 3334 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3335 Assembler::vmovdqu(dst, src); 3336 } 3337 3338 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 3339 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3340 Assembler::vmovdqu(dst, src); 3341 } 3342 3343 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) { 3344 if (reachable(src)) { 3345 vmovdqu(dst, as_Address(src)); 3346 } 3347 else { 3348 lea(rscratch1, src); 3349 vmovdqu(dst, Address(rscratch1, 0)); 3350 } 3351 } 3352 3353 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3354 if (reachable(src)) { 3355 Assembler::evmovdquq(dst, as_Address(src), vector_len); 3356 } else { 3357 lea(rscratch, src); 3358 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 3359 } 3360 } 3361 3362 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) { 3363 if (reachable(src)) { 3364 Assembler::movdqa(dst, as_Address(src)); 3365 } else { 3366 lea(rscratch1, src); 3367 Assembler::movdqa(dst, Address(rscratch1, 0)); 3368 } 3369 } 3681 } 3682 3683 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { 3684 if (reachable(src)) { 3685 Assembler::ucomisd(dst, as_Address(src)); 3686 } else { 3687 lea(rscratch1, src); 3688 Assembler::ucomisd(dst, Address(rscratch1, 0)); 3689 } 3690 } 3691 3692 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { 3693 if (reachable(src)) { 3694 Assembler::ucomiss(dst, as_Address(src)); 3695 } else { 3696 lea(rscratch1, src); 3697 Assembler::ucomiss(dst, Address(rscratch1, 0)); 3698 } 3699 } 3700 3701 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) { 3702 // Used in sign-bit flipping with aligned address. 3703 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3704 if (reachable(src)) { 3705 Assembler::xorpd(dst, as_Address(src)); 3706 } else { 3707 lea(rscratch1, src); 3708 Assembler::xorpd(dst, Address(rscratch1, 0)); 3709 } 3710 } 3711 3712 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3713 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3714 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3715 } 3716 else { 3717 Assembler::xorpd(dst, src); 3718 } 3719 } 3720 3721 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3722 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3723 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3724 } else { 3725 Assembler::xorps(dst, src); 3726 } 3727 } 3728 3729 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) { 3730 // Used in sign-bit flipping with aligned address. 3731 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3732 if (reachable(src)) { 3733 Assembler::xorps(dst, as_Address(src)); 3734 } else { 3735 lea(rscratch1, src); 3736 Assembler::xorps(dst, Address(rscratch1, 0)); 3737 } 3738 } 3739 3740 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { 3741 // Used in sign-bit flipping with aligned address. 3742 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3743 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3744 if (reachable(src)) { 3745 Assembler::pshufb(dst, as_Address(src)); 3746 } else { 3747 lea(rscratch1, src); 3748 Assembler::pshufb(dst, Address(rscratch1, 0)); 3749 } 3750 } 3751 3752 // AVX 3-operands instructions 3753 3754 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3755 if (reachable(src)) { 3756 vaddsd(dst, nds, as_Address(src)); 3782 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3783 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3784 Assembler::vpaddb(dst, nds, src, vector_len); 3785 } 3786 3787 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3788 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3789 Assembler::vpaddb(dst, nds, src, vector_len); 3790 } 3791 3792 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3793 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3794 Assembler::vpaddw(dst, nds, src, vector_len); 3795 } 3796 3797 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3798 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3799 Assembler::vpaddw(dst, nds, src, vector_len); 3800 } 3801 3802 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3803 if (reachable(src)) { 3804 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3805 } else { 3806 lea(rscratch1, src); 3807 Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len); 3808 } 3809 } 3810 3811 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { 3812 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3813 Assembler::vpbroadcastw(dst, src, vector_len); 3814 } 3815 3816 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3817 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3818 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3819 } 3820 3821 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3822 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3823 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3824 } 3825 3826 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3827 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3856 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3857 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3858 Assembler::vpsubw(dst, nds, src, vector_len); 3859 } 3860 3861 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3862 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3863 Assembler::vpsubw(dst, nds, src, vector_len); 3864 } 3865 3866 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3867 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3868 Assembler::vpsraw(dst, nds, shift, vector_len); 3869 } 3870 3871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3872 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3873 Assembler::vpsraw(dst, nds, shift, vector_len); 3874 } 3875 3876 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3877 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3878 Assembler::vpsrlw(dst, nds, shift, vector_len); 3879 } 3880 3881 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3882 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3883 Assembler::vpsrlw(dst, nds, shift, vector_len); 3884 } 3885 3886 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3887 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3888 Assembler::vpsllw(dst, nds, shift, vector_len); 3889 } 3890 3891 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3892 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3893 Assembler::vpsllw(dst, nds, shift, vector_len); 3894 } 3895 3896 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3897 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3898 Assembler::vptest(dst, src); 3899 } 3900 3901 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3902 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3903 Assembler::punpcklbw(dst, src); 3904 } 3905 3906 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3907 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3908 Assembler::pshufd(dst, src, mode); 3909 } 3910 3911 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3912 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3913 Assembler::pshuflw(dst, src, mode); 3914 } 3915 3916 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3917 if (reachable(src)) { 3918 vandpd(dst, nds, as_Address(src), vector_len); 3919 } else { 3920 lea(rscratch1, src); 3921 vandpd(dst, nds, Address(rscratch1, 0), vector_len); 3922 } 3923 } 3924 3925 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3926 if (reachable(src)) { 3927 vandps(dst, nds, as_Address(src), vector_len); 3928 } else { 3929 lea(rscratch1, src); 3930 vandps(dst, nds, Address(rscratch1, 0), vector_len); 3931 } 3932 } 3933 3934 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3935 if (reachable(src)) { 3936 vdivsd(dst, nds, as_Address(src)); 3937 } else { 3938 lea(rscratch1, src); 3939 vdivsd(dst, nds, Address(rscratch1, 0)); 3940 } 3941 } 3942 3943 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3944 if (reachable(src)) { 3945 vdivss(dst, nds, as_Address(src)); 3946 } else { 3947 lea(rscratch1, src); 3948 vdivss(dst, nds, Address(rscratch1, 0)); 3949 } 3950 } 3978 3979 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3980 if (reachable(src)) { 3981 vsubss(dst, nds, as_Address(src)); 3982 } else { 3983 lea(rscratch1, src); 3984 vsubss(dst, nds, Address(rscratch1, 0)); 3985 } 3986 } 3987 3988 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3989 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3990 vxorps(dst, nds, src, Assembler::AVX_128bit); 3991 } 3992 3993 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3994 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 3995 vxorpd(dst, nds, src, Assembler::AVX_128bit); 3996 } 3997 3998 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 3999 if (reachable(src)) { 4000 vxorpd(dst, nds, as_Address(src), vector_len); 4001 } else { 4002 lea(rscratch1, src); 4003 vxorpd(dst, nds, Address(rscratch1, 0), vector_len); 4004 } 4005 } 4006 4007 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) { 4008 if (reachable(src)) { 4009 vxorps(dst, nds, as_Address(src), vector_len); 4010 } else { 4011 lea(rscratch1, src); 4012 vxorps(dst, nds, Address(rscratch1, 0), vector_len); 4013 } 4014 } 4015 4016 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) { 4017 const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask); 4018 STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code 4019 // The inverted mask is sign-extended 4020 andptr(possibly_jweak, inverted_jweak_mask); 4021 } 4022 4023 void MacroAssembler::resolve_jobject(Register value, 4024 Register thread, 4025 Register tmp) { 4026 assert_different_registers(value, thread, tmp); 4027 Label done, not_weak; 4028 testptr(value, value); 4029 jcc(Assembler::zero, done); // Use NULL as-is. 4030 testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag. 4031 jcc(Assembler::zero, not_weak); 4032 // Resolve jweak. 4033 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 4034 value, Address(value, -JNIHandles::weak_tag_value), tmp, thread); | 986 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) { 988 if (reachable(src)) { 989 Assembler::addpd(dst, as_Address(src)); 990 } else { 991 lea(rscratch1, src); 992 Assembler::addpd(dst, Address(rscratch1, 0)); 993 } 994 } 995 996 void MacroAssembler::align(int modulus) { 997 align(modulus, offset()); 998 } 999 1000 void MacroAssembler::align(int modulus, int target) { 1001 if (target % modulus != 0) { 1002 nop(modulus - (target % modulus)); 1003 } 1004 } 1005 1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 1007 // Used in sign-masking with aligned address. 1008 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1009 if (reachable(src)) { 1010 Assembler::andpd(dst, as_Address(src)); 1011 } else { 1012 lea(scratch_reg, src); 1013 Assembler::andpd(dst, Address(scratch_reg, 0)); 1014 } 1015 } 1016 1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 1018 // Used in sign-masking with aligned address. 1019 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 1020 if (reachable(src)) { 1021 Assembler::andps(dst, as_Address(src)); 1022 } else { 1023 lea(scratch_reg, src); 1024 Assembler::andps(dst, Address(scratch_reg, 0)); 1025 } 1026 } 1027 1028 void MacroAssembler::andptr(Register dst, int32_t imm32) { 1029 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); 1030 } 1031 1032 void MacroAssembler::atomic_incl(Address counter_addr) { 1033 lock(); 1034 incrementl(counter_addr); 1035 } 1036 1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { 1038 if (reachable(counter_addr)) { 1039 atomic_incl(as_Address(counter_addr)); 1040 } else { 1041 lea(scr, counter_addr); 1042 atomic_incl(Address(scr, 0)); 1043 } 1044 } 3323 lea(scratchReg, src); 3324 movdqu(dst, Address(scratchReg, 0)); 3325 } 3326 } 3327 3328 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) { 3329 assert(((src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3330 Assembler::vmovdqu(dst, src); 3331 } 3332 3333 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) { 3334 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3335 Assembler::vmovdqu(dst, src); 3336 } 3337 3338 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) { 3339 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3340 Assembler::vmovdqu(dst, src); 3341 } 3342 3343 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 3344 if (reachable(src)) { 3345 vmovdqu(dst, as_Address(src)); 3346 } 3347 else { 3348 lea(scratch_reg, src); 3349 vmovdqu(dst, Address(scratch_reg, 0)); 3350 } 3351 } 3352 3353 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) { 3354 if (reachable(src)) { 3355 Assembler::evmovdquq(dst, as_Address(src), vector_len); 3356 } else { 3357 lea(rscratch, src); 3358 Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len); 3359 } 3360 } 3361 3362 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) { 3363 if (reachable(src)) { 3364 Assembler::movdqa(dst, as_Address(src)); 3365 } else { 3366 lea(rscratch1, src); 3367 Assembler::movdqa(dst, Address(rscratch1, 0)); 3368 } 3369 } 3681 } 3682 3683 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { 3684 if (reachable(src)) { 3685 Assembler::ucomisd(dst, as_Address(src)); 3686 } else { 3687 lea(rscratch1, src); 3688 Assembler::ucomisd(dst, Address(rscratch1, 0)); 3689 } 3690 } 3691 3692 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { 3693 if (reachable(src)) { 3694 Assembler::ucomiss(dst, as_Address(src)); 3695 } else { 3696 lea(rscratch1, src); 3697 Assembler::ucomiss(dst, Address(rscratch1, 0)); 3698 } 3699 } 3700 3701 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 3702 // Used in sign-bit flipping with aligned address. 3703 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3704 if (reachable(src)) { 3705 Assembler::xorpd(dst, as_Address(src)); 3706 } else { 3707 lea(scratch_reg, src); 3708 Assembler::xorpd(dst, Address(scratch_reg, 0)); 3709 } 3710 } 3711 3712 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) { 3713 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3714 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3715 } 3716 else { 3717 Assembler::xorpd(dst, src); 3718 } 3719 } 3720 3721 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) { 3722 if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) { 3723 Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit); 3724 } else { 3725 Assembler::xorps(dst, src); 3726 } 3727 } 3728 3729 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src, Register scratch_reg) { 3730 // Used in sign-bit flipping with aligned address. 3731 assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); 3732 if (reachable(src)) { 3733 Assembler::xorps(dst, as_Address(src)); 3734 } else { 3735 lea(scratch_reg, src); 3736 Assembler::xorps(dst, Address(scratch_reg, 0)); 3737 } 3738 } 3739 3740 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { 3741 // Used in sign-bit flipping with aligned address. 3742 bool aligned_adr = (((intptr_t)src.target() & 15) == 0); 3743 assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); 3744 if (reachable(src)) { 3745 Assembler::pshufb(dst, as_Address(src)); 3746 } else { 3747 lea(rscratch1, src); 3748 Assembler::pshufb(dst, Address(rscratch1, 0)); 3749 } 3750 } 3751 3752 // AVX 3-operands instructions 3753 3754 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3755 if (reachable(src)) { 3756 vaddsd(dst, nds, as_Address(src)); 3782 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3783 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3784 Assembler::vpaddb(dst, nds, src, vector_len); 3785 } 3786 3787 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3788 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3789 Assembler::vpaddb(dst, nds, src, vector_len); 3790 } 3791 3792 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3793 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3794 Assembler::vpaddw(dst, nds, src, vector_len); 3795 } 3796 3797 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3798 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3799 Assembler::vpaddw(dst, nds, src, vector_len); 3800 } 3801 3802 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 3803 if (reachable(src)) { 3804 Assembler::vpand(dst, nds, as_Address(src), vector_len); 3805 } else { 3806 lea(scratch_reg, src); 3807 Assembler::vpand(dst, nds, Address(scratch_reg, 0), vector_len); 3808 } 3809 } 3810 3811 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) { 3812 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3813 Assembler::vpbroadcastw(dst, src, vector_len); 3814 } 3815 3816 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3817 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3818 Assembler::vpcmpeqb(dst, nds, src, vector_len); 3819 } 3820 3821 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3822 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3823 Assembler::vpcmpeqw(dst, nds, src, vector_len); 3824 } 3825 3826 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) { 3827 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3856 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 3857 assert(((dst->encoding() < 16 && src->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3858 Assembler::vpsubw(dst, nds, src, vector_len); 3859 } 3860 3861 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { 3862 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3863 Assembler::vpsubw(dst, nds, src, vector_len); 3864 } 3865 3866 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3867 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3868 Assembler::vpsraw(dst, nds, shift, vector_len); 3869 } 3870 3871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3872 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3873 Assembler::vpsraw(dst, nds, shift, vector_len); 3874 } 3875 3876 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3877 assert(UseAVX > 2,""); 3878 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3879 vector_len = 2; 3880 } 3881 Assembler::evpsraq(dst, nds, shift, vector_len); 3882 } 3883 3884 void MacroAssembler::evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3885 assert(UseAVX > 2,""); 3886 if (!VM_Version::supports_avx512vl() && vector_len < 2) { 3887 vector_len = 2; 3888 } 3889 Assembler::evpsraq(dst, nds, shift, vector_len); 3890 } 3891 3892 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3893 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3894 Assembler::vpsrlw(dst, nds, shift, vector_len); 3895 } 3896 3897 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3898 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3899 Assembler::vpsrlw(dst, nds, shift, vector_len); 3900 } 3901 3902 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) { 3903 assert(((dst->encoding() < 16 && shift->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3904 Assembler::vpsllw(dst, nds, shift, vector_len); 3905 } 3906 3907 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) { 3908 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3909 Assembler::vpsllw(dst, nds, shift, vector_len); 3910 } 3911 3912 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) { 3913 assert((dst->encoding() < 16 && src->encoding() < 16),"XMM register should be 0-15"); 3914 Assembler::vptest(dst, src); 3915 } 3916 3917 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) { 3918 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3919 Assembler::punpcklbw(dst, src); 3920 } 3921 3922 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) { 3923 assert(((dst->encoding() < 16) || VM_Version::supports_avx512vl()),"XMM register should be 0-15"); 3924 Assembler::pshufd(dst, src, mode); 3925 } 3926 3927 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) { 3928 assert(((dst->encoding() < 16 && src->encoding() < 16) || VM_Version::supports_avx512vlbw()),"XMM register should be 0-15"); 3929 Assembler::pshuflw(dst, src, mode); 3930 } 3931 3932 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 3933 if (reachable(src)) { 3934 vandpd(dst, nds, as_Address(src), vector_len); 3935 } else { 3936 lea(scratch_reg, src); 3937 vandpd(dst, nds, Address(scratch_reg, 0), vector_len); 3938 } 3939 } 3940 3941 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 3942 if (reachable(src)) { 3943 vandps(dst, nds, as_Address(src), vector_len); 3944 } else { 3945 lea(scratch_reg, src); 3946 vandps(dst, nds, Address(scratch_reg, 0), vector_len); 3947 } 3948 } 3949 3950 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3951 if (reachable(src)) { 3952 vdivsd(dst, nds, as_Address(src)); 3953 } else { 3954 lea(rscratch1, src); 3955 vdivsd(dst, nds, Address(rscratch1, 0)); 3956 } 3957 } 3958 3959 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3960 if (reachable(src)) { 3961 vdivss(dst, nds, as_Address(src)); 3962 } else { 3963 lea(rscratch1, src); 3964 vdivss(dst, nds, Address(rscratch1, 0)); 3965 } 3966 } 3994 3995 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 3996 if (reachable(src)) { 3997 vsubss(dst, nds, as_Address(src)); 3998 } else { 3999 lea(rscratch1, src); 4000 vsubss(dst, nds, Address(rscratch1, 0)); 4001 } 4002 } 4003 4004 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 4005 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 4006 vxorps(dst, nds, src, Assembler::AVX_128bit); 4007 } 4008 4009 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { 4010 assert(((dst->encoding() < 16 && nds->encoding() < 16) || VM_Version::supports_avx512vldq()),"XMM register should be 0-15"); 4011 vxorpd(dst, nds, src, Assembler::AVX_128bit); 4012 } 4013 4014 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 4015 if (reachable(src)) { 4016 vxorpd(dst, nds, as_Address(src), vector_len); 4017 } else { 4018 lea(scratch_reg, src); 4019 vxorpd(dst, nds, Address(scratch_reg, 0), vector_len); 4020 } 4021 } 4022 4023 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 4024 if (reachable(src)) { 4025 vxorps(dst, nds, as_Address(src), vector_len); 4026 } else { 4027 lea(scratch_reg, src); 4028 vxorps(dst, nds, Address(scratch_reg, 0), vector_len); 4029 } 4030 } 4031 4032 void MacroAssembler::vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg) { 4033 if (UseAVX > 1 || (vector_len < 1)) { 4034 if (reachable(src)) { 4035 Assembler::vpxor(dst, nds, as_Address(src), vector_len); 4036 } else { 4037 lea(scratch_reg, src); 4038 Assembler::vpxor(dst, nds, Address(scratch_reg, 0), vector_len); 4039 } 4040 } 4041 else { 4042 MacroAssembler::vxorpd(dst, nds, src, vector_len, scratch_reg); 4043 } 4044 } 4045 4046 //------------------------------------------------------------------------------------------- 4047 #ifdef COMPILER2 4048 // Generic instructions support for use in .ad files C2 code generation 4049 4050 void MacroAssembler::vabsnegd(int opcode, XMMRegister dst, Register scr) { 4051 if (opcode == Op_AbsVD) { 4052 andpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), scr); 4053 } else { 4054 assert((opcode == Op_NegVD),"opcode should be Op_NegD"); 4055 xorpd(dst, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), scr); 4056 } 4057 } 4058 4059 void MacroAssembler::vabsnegd(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr) { 4060 if (opcode == Op_AbsVD) { 4061 vandpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_mask()), vector_len, scr); 4062 } else { 4063 assert((opcode == Op_NegVD),"opcode should be Op_NegD"); 4064 vxorpd(dst, src, ExternalAddress(StubRoutines::x86::vector_double_sign_flip()), vector_len, scr); 4065 } 4066 } 4067 4068 void MacroAssembler::vabsnegf(int opcode, XMMRegister dst, Register scr) { 4069 if (opcode == Op_AbsVF) { 4070 andps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), scr); 4071 } else { 4072 assert((opcode == Op_NegVF),"opcode should be Op_NegF"); 4073 xorps(dst, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), scr); 4074 } 4075 } 4076 4077 void MacroAssembler::vabsnegf(int opcode, XMMRegister dst, XMMRegister src, int vector_len, Register scr) { 4078 if (opcode == Op_AbsVF) { 4079 vandps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_mask()), vector_len, scr); 4080 } else { 4081 assert((opcode == Op_NegVF),"opcode should be Op_NegF"); 4082 vxorps(dst, src, ExternalAddress(StubRoutines::x86::vector_float_sign_flip()), vector_len, scr); 4083 } 4084 } 4085 4086 void MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src) { 4087 if (sign) { 4088 pmovsxbw(dst, src); 4089 } else { 4090 pmovzxbw(dst, src); 4091 } 4092 } 4093 4094 void MacroAssembler::vextendbw(bool sign, XMMRegister dst, XMMRegister src, int vector_len) { 4095 if (sign) { 4096 vpmovsxbw(dst, src, vector_len); 4097 } else { 4098 vpmovzxbw(dst, src, vector_len); 4099 } 4100 } 4101 4102 void MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister src) { 4103 if (opcode == Op_RShiftVI) { 4104 psrad(dst, src); 4105 } else if (opcode == Op_LShiftVI) { 4106 pslld(dst, src); 4107 } else { 4108 assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI"); 4109 psrld(dst, src); 4110 } 4111 } 4112 4113 void MacroAssembler::vshiftd(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4114 if (opcode == Op_RShiftVI) { 4115 vpsrad(dst, nds, src, vector_len); 4116 } else if (opcode == Op_LShiftVI) { 4117 vpslld(dst, nds, src, vector_len); 4118 } else { 4119 assert((opcode == Op_URShiftVI),"opcode should be Op_URShiftVI"); 4120 vpsrld(dst, nds, src, vector_len); 4121 } 4122 } 4123 4124 void MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister src) { 4125 if ((opcode == Op_RShiftVS) || (opcode == Op_RShiftVB)) { 4126 psraw(dst, src); 4127 } else if ((opcode == Op_LShiftVS) || (opcode == Op_LShiftVB)) { 4128 psllw(dst, src); 4129 } else { 4130 assert(((opcode == Op_URShiftVS) || (opcode == Op_URShiftVB)),"opcode should be one of Op_URShiftVS or Op_URShiftVB"); 4131 psrlw(dst, src); 4132 } 4133 } 4134 4135 void MacroAssembler::vshiftw(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4136 if ((opcode == Op_RShiftVS) || (opcode == Op_RShiftVB)) { 4137 vpsraw(dst, nds, src, vector_len); 4138 } else if ((opcode == Op_LShiftVS) || (opcode == Op_LShiftVB)) { 4139 vpsllw(dst, nds, src, vector_len); 4140 } else { 4141 assert(((opcode == Op_URShiftVS) || (opcode == Op_URShiftVB)),"opcode should be one of Op_URShiftVS or Op_URShiftVB"); 4142 vpsrlw(dst, nds, src, vector_len); 4143 } 4144 } 4145 4146 void MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister src) { 4147 if (opcode == Op_RShiftVL) { 4148 psrlq(dst, src); // using srl to implement sra on pre-avs512 systems 4149 } else if (opcode == Op_LShiftVL) { 4150 psllq(dst, src); 4151 } else { 4152 assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL"); 4153 psrlq(dst, src); 4154 } 4155 } 4156 4157 void MacroAssembler::vshiftq(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { 4158 if (opcode == Op_RShiftVL) { 4159 evpsraq(dst, nds, src, vector_len); 4160 } else if (opcode == Op_LShiftVL) { 4161 vpsllq(dst, nds, src, vector_len); 4162 } else { 4163 assert((opcode == Op_URShiftVL),"opcode should be Op_URShiftVL"); 4164 vpsrlq(dst, nds, src, vector_len); 4165 } 4166 } 4167 #endif 4168 //------------------------------------------------------------------------------------------- 4169 4170 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) { 4171 const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask); 4172 STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code 4173 // The inverted mask is sign-extended 4174 andptr(possibly_jweak, inverted_jweak_mask); 4175 } 4176 4177 void MacroAssembler::resolve_jobject(Register value, 4178 Register thread, 4179 Register tmp) { 4180 assert_different_registers(value, thread, tmp); 4181 Label done, not_weak; 4182 testptr(value, value); 4183 jcc(Assembler::zero, done); // Use NULL as-is. 4184 testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag. 4185 jcc(Assembler::zero, not_weak); 4186 // Resolve jweak. 4187 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 4188 value, Address(value, -JNIHandles::weak_tag_value), tmp, thread); |