120 }
121 }
122
123 // Implementation of Address
124
125 #ifdef _LP64
126
127 Address Address::make_array(ArrayAddress adr) {
128 // Not implementable on 64bit machines
129 // Should have been handled higher up the call chain.
130 ShouldNotReachHere();
131 return Address();
132 }
133
134 // exceedingly dangerous constructor
135 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
136 _base = noreg;
137 _index = noreg;
138 _scale = no_scale;
139 _disp = disp;
140 switch (rtype) {
141 case relocInfo::external_word_type:
142 _rspec = external_word_Relocation::spec(loc);
143 break;
144 case relocInfo::internal_word_type:
145 _rspec = internal_word_Relocation::spec(loc);
146 break;
147 case relocInfo::runtime_call_type:
148 // HMM
149 _rspec = runtime_call_Relocation::spec();
150 break;
151 case relocInfo::poll_type:
152 case relocInfo::poll_return_type:
153 _rspec = Relocation::spec_simple(rtype);
154 break;
155 case relocInfo::none:
156 break;
157 default:
158 ShouldNotReachHere();
159 }
160 }
161 #else // LP64
162
163 Address Address::make_array(ArrayAddress adr) {
164 AddressLiteral base = adr.base();
165 Address index = adr.index();
166 assert(index._disp == 0, "must not have disp"); // maybe it can?
167 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
168 array._rspec = base._rspec;
169 return array;
170 }
171
172 // exceedingly dangerous constructor
173 Address::Address(address loc, RelocationHolder spec) {
174 _base = noreg;
175 _index = noreg;
176 _scale = no_scale;
177 _disp = (intptr_t) loc;
178 _rspec = spec;
179 }
180
181 #endif // _LP64
182
183
184
185 // Convert the raw encoding form into the form expected by the constructor for
186 // Address. An index of 4 (rsp) corresponds to having no index, so convert
187 // that to noreg for the Address constructor.
188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
189 RelocationHolder rspec;
190 if (disp_reloc != relocInfo::none) {
191 rspec = Relocation::spec_simple(disp_reloc);
192 }
193 bool valid_index = index != rsp->encoding();
194 if (valid_index) {
195 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
196 madr._rspec = rspec;
197 return madr;
198 } else {
591 emit_int8(0x25);
592 emit_data(disp, rspec, disp32_operand);
593 }
594 }
595 }
596
597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
598 Address::ScaleFactor scale, int disp,
599 RelocationHolder const& rspec) {
600 if (UseAVX > 2) {
601 int xreg_enc = reg->encoding();
602 if (xreg_enc > 15) {
603 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
604 emit_operand((Register)new_reg, base, index, scale, disp, rspec);
605 return;
606 }
607 }
608 emit_operand((Register)reg, base, index, scale, disp, rspec);
609 }
610
611 // Secret local extension to Assembler::WhichOperand:
612 #define end_pc_operand (_WhichOperand_limit)
613
614 address Assembler::locate_operand(address inst, WhichOperand which) {
615 // Decode the given instruction, and return the address of
616 // an embedded 32-bit operand word.
617
618 // If "which" is disp32_operand, selects the displacement portion
619 // of an effective address specifier.
620 // If "which" is imm64_operand, selects the trailing immediate constant.
621 // If "which" is call32_operand, selects the displacement of a call or jump.
622 // Caller is responsible for ensuring that there is such an operand,
623 // and that it is 32/64 bits wide.
624
625 // If "which" is end_pc_operand, find the end of the instruction.
626
627 address ip = inst;
628 bool is_64bit = false;
629
630 debug_only(bool has_disp32 = false);
1091 }
1092 assert(opnd == pc(), "must put operand where relocs can find it");
1093 }
1094 #endif // ASSERT
1095
1096 void Assembler::emit_operand32(Register reg, Address adr) {
1097 assert(reg->encoding() < 8, "no extended registers");
1098 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1099 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1100 adr._rspec);
1101 }
1102
1103 void Assembler::emit_operand(Register reg, Address adr,
1104 int rip_relative_correction) {
1105 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1106 adr._rspec,
1107 rip_relative_correction);
1108 }
1109
1110 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1111 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1112 adr._rspec);
1113 }
1114
1115 // MMX operations
1116 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1117 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1118 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1119 }
1120
1121 // work around gcc (3.2.1-7a) bug
1122 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1123 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1124 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1125 }
1126
1127
1128 void Assembler::emit_farith(int b1, int b2, int i) {
1129 assert(isByte(b1) && isByte(b2), "wrong opcode");
1130 assert(0 <= i && i < 8, "illegal stack offset");
1131 emit_int8(b1);
1132 emit_int8(b2 + i);
3379 }
3380
3381 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3382 assert(VM_Version::supports_avx2(), "");
3383 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3384 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3385 emit_int8(0x46);
3386 emit_int8(0xC0 | encode);
3387 emit_int8(imm8);
3388 }
3389
3390 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3391 assert(VM_Version::supports_avx(), "");
3392 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3393 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3394 emit_int8(0x06);
3395 emit_int8(0xC0 | encode);
3396 emit_int8(imm8);
3397 }
3398
3399
3400 void Assembler::pause() {
3401 emit_int8((unsigned char)0xF3);
3402 emit_int8((unsigned char)0x90);
3403 }
3404
3405 void Assembler::ud2() {
3406 emit_int8(0x0F);
3407 emit_int8(0x0B);
3408 }
3409
3410 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3411 assert(VM_Version::supports_sse4_2(), "");
3412 InstructionMark im(this);
3413 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3414 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3415 emit_int8(0x61);
3416 emit_operand(dst, src);
3417 emit_int8(imm8);
3418 }
3830
3831 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3832 assert(VM_Version::supports_sse4_1(), "");
3833 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3834 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3835 emit_int8(0x30);
3836 emit_int8((unsigned char)(0xC0 | encode));
3837 }
3838
3839 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3840 assert(VM_Version::supports_avx(), "");
3841 InstructionMark im(this);
3842 assert(dst != xnoreg, "sanity");
3843 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3844 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3845 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3846 emit_int8(0x30);
3847 emit_operand(dst, src);
3848 }
3849
3850 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3851 assert(is_vector_masking(), "");
3852 assert(VM_Version::supports_avx512vlbw(), "");
3853 assert(dst != xnoreg, "sanity");
3854 InstructionMark im(this);
3855 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3856 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3857 attributes.set_embedded_opmask_register_specifier(mask);
3858 attributes.set_is_evex_instruction();
3859 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3860 emit_int8(0x30);
3861 emit_operand(dst, src);
3862 }
3863
3864 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3865 assert(VM_Version::supports_avx512vlbw(), "");
3866 assert(src != xnoreg, "sanity");
3867 InstructionMark im(this);
3868 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3869 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3870 attributes.set_is_evex_instruction();
3871 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3872 emit_int8(0x30);
3873 emit_operand(src, dst);
3874 }
3875
3876 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3877 assert(is_vector_masking(), "");
3878 assert(VM_Version::supports_avx512vlbw(), "");
3879 assert(src != xnoreg, "sanity");
3880 InstructionMark im(this);
3881 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3882 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3883 attributes.reset_is_clear_context();
3884 attributes.set_embedded_opmask_register_specifier(mask);
3885 attributes.set_is_evex_instruction();
3886 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3887 emit_int8(0x30);
3888 emit_operand(src, dst);
3889 }
3890
3891 // generic
3892 void Assembler::pop(Register dst) {
3893 int encode = prefix_and_encode(dst->encoding());
3894 emit_int8(0x58 | encode);
3895 }
3896
3897 void Assembler::popcntl(Register dst, Address src) {
3898 assert(VM_Version::supports_popcnt(), "must support");
3899 InstructionMark im(this);
3900 emit_int8((unsigned char)0xF3);
3901 prefix(src, dst);
3902 emit_int8(0x0F);
3903 emit_int8((unsigned char)0xB8);
3904 emit_operand(dst, src);
3905 }
3906
3907 void Assembler::popcntl(Register dst, Register src) {
3908 assert(VM_Version::supports_popcnt(), "must support");
3909 emit_int8((unsigned char)0xF3);
3910 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6031 emit_int8((unsigned char)(0xC0 | encode));
6032 }
6033
6034 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6035 assert(UseAVX > 0, "requires some form of AVX");
6036 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6037 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6038 emit_int8((unsigned char)0xD2);
6039 emit_int8((unsigned char)(0xC0 | encode));
6040 }
6041
6042 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6043 assert(UseAVX > 0, "requires some form of AVX");
6044 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6045 attributes.set_rex_vex_w_reverted();
6046 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6047 emit_int8((unsigned char)0xD3);
6048 emit_int8((unsigned char)(0xC0 | encode));
6049 }
6050
6051 // Shift packed integers arithmetically right by specified number of bits.
6052 void Assembler::psraw(XMMRegister dst, int shift) {
6053 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6054 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6055 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6056 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6057 emit_int8(0x71);
6058 emit_int8((unsigned char)(0xC0 | encode));
6059 emit_int8(shift & 0xFF);
6060 }
6061
6062 void Assembler::psrad(XMMRegister dst, int shift) {
6063 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6064 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6065 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6066 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6067 emit_int8(0x72);
6068 emit_int8((unsigned char)(0xC0 | encode));
6069 emit_int8(shift & 0xFF);
6070 }
6132 }
6133
6134 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6135 assert(UseAVX > 0, "requires some form of AVX");
6136 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6137 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6138 emit_int8((unsigned char)0xDB);
6139 emit_int8((unsigned char)(0xC0 | encode));
6140 }
6141
6142 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6143 assert(UseAVX > 0, "requires some form of AVX");
6144 InstructionMark im(this);
6145 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6146 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6147 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6148 emit_int8((unsigned char)0xDB);
6149 emit_operand(dst, src);
6150 }
6151
6152 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6153 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6154 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6155 attributes.set_rex_vex_w_reverted();
6156 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6157 emit_int8((unsigned char)0xDF);
6158 emit_int8((unsigned char)(0xC0 | encode));
6159 }
6160
6161 void Assembler::por(XMMRegister dst, XMMRegister src) {
6162 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6163 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6164 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6165 emit_int8((unsigned char)0xEB);
6166 emit_int8((unsigned char)(0xC0 | encode));
6167 }
6168
6169 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6170 assert(UseAVX > 0, "requires some form of AVX");
6171 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6172 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6173 emit_int8((unsigned char)0xEB);
6174 emit_int8((unsigned char)(0xC0 | encode));
6175 }
6176
6177 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6178 assert(UseAVX > 0, "requires some form of AVX");
6179 InstructionMark im(this);
6180 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6181 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6182 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6183 emit_int8((unsigned char)0xEB);
6184 emit_operand(dst, src);
6185 }
6186
6187 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6188 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6189 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6190 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6191 emit_int8((unsigned char)0xEF);
6192 emit_int8((unsigned char)(0xC0 | encode));
6193 }
6194
6195 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6196 assert(UseAVX > 0, "requires some form of AVX");
6197 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6198 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6199 emit_int8((unsigned char)0xEF);
6200 emit_int8((unsigned char)(0xC0 | encode));
6201 }
6202
6203 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6204 assert(UseAVX > 0, "requires some form of AVX");
6205 InstructionMark im(this);
6206 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6778 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6779 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6780 assert(VM_Version::supports_evex(), "");
6781 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6782 attributes.set_is_evex_instruction();
6783 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6784 emit_int8(0x7C);
6785 emit_int8((unsigned char)(0xC0 | encode));
6786 }
6787
6788 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6789 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6790 assert(VM_Version::supports_evex(), "");
6791 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6792 attributes.set_is_evex_instruction();
6793 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6794 emit_int8(0x7C);
6795 emit_int8((unsigned char)(0xC0 | encode));
6796 }
6797
6798
6799 // Carry-Less Multiplication Quadword
6800 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6801 assert(VM_Version::supports_clmul(), "");
6802 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6803 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6804 emit_int8(0x44);
6805 emit_int8((unsigned char)(0xC0 | encode));
6806 emit_int8((unsigned char)mask);
6807 }
6808
6809 // Carry-Less Multiplication Quadword
6810 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6811 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6812 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6813 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6814 emit_int8(0x44);
6815 emit_int8((unsigned char)(0xC0 | encode));
6816 emit_int8((unsigned char)mask);
6817 }
7403 // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7404 int byte4 = (_attributes->is_no_reg_mask()) ?
7405 0 :
7406 _attributes->get_embedded_opmask_register_specifier();
7407 // EVEX.v` for extending EVEX.vvvv or VIDX
7408 byte4 |= (evex_v ? 0: EVEX_V);
7409 // third EXEC.b for broadcast actions
7410 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7411 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7412 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7413 // last is EVEX.z for zero/merge actions
7414 if (_attributes->is_no_reg_mask() == false) {
7415 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7416 }
7417 emit_int8(byte4);
7418 }
7419
7420 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7421 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7422 bool vex_b = adr.base_needs_rex();
7423 bool vex_x = adr.index_needs_rex();
7424 set_attributes(attributes);
7425 attributes->set_current_assembler(this);
7426
7427 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7428 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7429 switch (attributes->get_vector_len()) {
7430 case AVX_128bit:
7431 case AVX_256bit:
7432 attributes->set_is_legacy_mode();
7433 break;
7434 }
7435 }
7436
7437 // For pure EVEX check and see if this instruction
7438 // is allowed in legacy mode and has resources which will
7439 // fit in it. Pure EVEX instructions will use set_is_evex_instruction in their definition,
7440 // else that field is set when we encode to EVEX
7441 if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7442 !_is_managed && !attributes->is_evex_instruction()) {
7443 if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7444 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7445 if (check_register_bank) {
7446 // check nds_enc and xreg_enc for upper bank usage
7447 if (nds_enc < 16 && xreg_enc < 16) {
7448 attributes->set_is_legacy_mode();
7449 }
7450 } else {
7451 attributes->set_is_legacy_mode();
7452 }
7453 }
7454 }
7455
7456 _is_managed = false;
7457 if (UseAVX > 2 && !attributes->is_legacy_mode())
7458 {
7459 bool evex_r = (xreg_enc >= 16);
7460 bool evex_v = (nds_enc >= 16);
7461 attributes->set_is_evex_instruction();
7462 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7463 } else {
7464 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7465 attributes->set_rex_vex_w(false);
7466 }
7467 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7468 }
7469 }
7470
7471 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7472 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7473 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7474 bool vex_x = false;
7475 set_attributes(attributes);
7476 attributes->set_current_assembler(this);
7477 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7478
7479 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7480 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
|
120 }
121 }
122
123 // Implementation of Address
124
125 #ifdef _LP64
126
127 Address Address::make_array(ArrayAddress adr) {
128 // Not implementable on 64bit machines
129 // Should have been handled higher up the call chain.
130 ShouldNotReachHere();
131 return Address();
132 }
133
134 // exceedingly dangerous constructor
135 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
136 _base = noreg;
137 _index = noreg;
138 _scale = no_scale;
139 _disp = disp;
140 _xmmindex = xnoreg;
141 _isxmmindex = false;
142 switch (rtype) {
143 case relocInfo::external_word_type:
144 _rspec = external_word_Relocation::spec(loc);
145 break;
146 case relocInfo::internal_word_type:
147 _rspec = internal_word_Relocation::spec(loc);
148 break;
149 case relocInfo::runtime_call_type:
150 // HMM
151 _rspec = runtime_call_Relocation::spec();
152 break;
153 case relocInfo::poll_type:
154 case relocInfo::poll_return_type:
155 _rspec = Relocation::spec_simple(rtype);
156 break;
157 case relocInfo::none:
158 break;
159 default:
160 ShouldNotReachHere();
161 }
162 }
163 #else // LP64
164
165 Address Address::make_array(ArrayAddress adr) {
166 AddressLiteral base = adr.base();
167 Address index = adr.index();
168 assert(index._disp == 0, "must not have disp"); // maybe it can?
169 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
170 array._rspec = base._rspec;
171 return array;
172 }
173
174 // exceedingly dangerous constructor
175 Address::Address(address loc, RelocationHolder spec) {
176 _base = noreg;
177 _index = noreg;
178 _scale = no_scale;
179 _disp = (intptr_t) loc;
180 _rspec = spec;
181 _xmmindex = xnoreg;
182 _isxmmindex = false;
183 }
184
185 #endif // _LP64
186
187
188
189 // Convert the raw encoding form into the form expected by the constructor for
190 // Address. An index of 4 (rsp) corresponds to having no index, so convert
191 // that to noreg for the Address constructor.
192 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
193 RelocationHolder rspec;
194 if (disp_reloc != relocInfo::none) {
195 rspec = Relocation::spec_simple(disp_reloc);
196 }
197 bool valid_index = index != rsp->encoding();
198 if (valid_index) {
199 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
200 madr._rspec = rspec;
201 return madr;
202 } else {
595 emit_int8(0x25);
596 emit_data(disp, rspec, disp32_operand);
597 }
598 }
599 }
600
601 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
602 Address::ScaleFactor scale, int disp,
603 RelocationHolder const& rspec) {
604 if (UseAVX > 2) {
605 int xreg_enc = reg->encoding();
606 if (xreg_enc > 15) {
607 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
608 emit_operand((Register)new_reg, base, index, scale, disp, rspec);
609 return;
610 }
611 }
612 emit_operand((Register)reg, base, index, scale, disp, rspec);
613 }
614
615 void Assembler::emit_operand(XMMRegister reg, Register base, XMMRegister index,
616 Address::ScaleFactor scale, int disp,
617 RelocationHolder const& rspec) {
618 if (UseAVX > 2) {
619 int xreg_enc = reg->encoding();
620 int xmmindex_enc = index->encoding();
621 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
622 XMMRegister new_index = as_XMMRegister(xmmindex_enc & 0xf);
623 emit_operand((Register)new_reg, base, (Register)new_index, scale, disp, rspec);
624 } else {
625 emit_operand((Register)reg, base, (Register)index, scale, disp, rspec);
626 }
627 }
628
629
630 // Secret local extension to Assembler::WhichOperand:
631 #define end_pc_operand (_WhichOperand_limit)
632
633 address Assembler::locate_operand(address inst, WhichOperand which) {
634 // Decode the given instruction, and return the address of
635 // an embedded 32-bit operand word.
636
637 // If "which" is disp32_operand, selects the displacement portion
638 // of an effective address specifier.
639 // If "which" is imm64_operand, selects the trailing immediate constant.
640 // If "which" is call32_operand, selects the displacement of a call or jump.
641 // Caller is responsible for ensuring that there is such an operand,
642 // and that it is 32/64 bits wide.
643
644 // If "which" is end_pc_operand, find the end of the instruction.
645
646 address ip = inst;
647 bool is_64bit = false;
648
649 debug_only(bool has_disp32 = false);
1110 }
1111 assert(opnd == pc(), "must put operand where relocs can find it");
1112 }
1113 #endif // ASSERT
1114
1115 void Assembler::emit_operand32(Register reg, Address adr) {
1116 assert(reg->encoding() < 8, "no extended registers");
1117 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1118 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1119 adr._rspec);
1120 }
1121
1122 void Assembler::emit_operand(Register reg, Address adr,
1123 int rip_relative_correction) {
1124 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1125 adr._rspec,
1126 rip_relative_correction);
1127 }
1128
1129 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1130 if (adr.isxmmindex()) {
1131 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
1132 } else {
1133 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1134 adr._rspec);
1135 }
1136 }
1137
1138 // MMX operations
1139 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1140 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1141 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1142 }
1143
1144 // work around gcc (3.2.1-7a) bug
1145 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1146 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1147 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1148 }
1149
1150
1151 void Assembler::emit_farith(int b1, int b2, int i) {
1152 assert(isByte(b1) && isByte(b2), "wrong opcode");
1153 assert(0 <= i && i < 8, "illegal stack offset");
1154 emit_int8(b1);
1155 emit_int8(b2 + i);
3402 }
3403
3404 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3405 assert(VM_Version::supports_avx2(), "");
3406 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3407 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3408 emit_int8(0x46);
3409 emit_int8(0xC0 | encode);
3410 emit_int8(imm8);
3411 }
3412
3413 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3414 assert(VM_Version::supports_avx(), "");
3415 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3416 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3417 emit_int8(0x06);
3418 emit_int8(0xC0 | encode);
3419 emit_int8(imm8);
3420 }
3421
3422 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3423 assert(VM_Version::supports_evex(), "");
3424 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3425 attributes.set_is_evex_instruction();
3426 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3427 emit_int8(0x76);
3428 emit_int8((unsigned char)(0xC0 | encode));
3429 }
3430
3431
3432 void Assembler::pause() {
3433 emit_int8((unsigned char)0xF3);
3434 emit_int8((unsigned char)0x90);
3435 }
3436
3437 void Assembler::ud2() {
3438 emit_int8(0x0F);
3439 emit_int8(0x0B);
3440 }
3441
3442 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3443 assert(VM_Version::supports_sse4_2(), "");
3444 InstructionMark im(this);
3445 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3446 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3447 emit_int8(0x61);
3448 emit_operand(dst, src);
3449 emit_int8(imm8);
3450 }
3862
3863 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3864 assert(VM_Version::supports_sse4_1(), "");
3865 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3866 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3867 emit_int8(0x30);
3868 emit_int8((unsigned char)(0xC0 | encode));
3869 }
3870
3871 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3872 assert(VM_Version::supports_avx(), "");
3873 InstructionMark im(this);
3874 assert(dst != xnoreg, "sanity");
3875 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3876 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3877 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3878 emit_int8(0x30);
3879 emit_operand(dst, src);
3880 }
3881
3882 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3883 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3884 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3885 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3886 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3887 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3888 emit_int8(0x30);
3889 emit_int8((unsigned char) (0xC0 | encode));
3890 }
3891
3892
3893 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3894 assert(is_vector_masking(), "");
3895 assert(VM_Version::supports_avx512vlbw(), "");
3896 assert(dst != xnoreg, "sanity");
3897 InstructionMark im(this);
3898 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3899 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3900 attributes.set_embedded_opmask_register_specifier(mask);
3901 attributes.set_is_evex_instruction();
3902 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3903 emit_int8(0x30);
3904 emit_operand(dst, src);
3905 }
3906 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3907 assert(VM_Version::supports_avx512vlbw(), "");
3908 assert(src != xnoreg, "sanity");
3909 InstructionMark im(this);
3910 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3911 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3912 attributes.set_is_evex_instruction();
3913 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3914 emit_int8(0x30);
3915 emit_operand(src, dst);
3916 }
3917
3918 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3919 assert(is_vector_masking(), "");
3920 assert(VM_Version::supports_avx512vlbw(), "");
3921 assert(src != xnoreg, "sanity");
3922 InstructionMark im(this);
3923 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3924 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3925 attributes.reset_is_clear_context();
3926 attributes.set_embedded_opmask_register_specifier(mask);
3927 attributes.set_is_evex_instruction();
3928 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3929 emit_int8(0x30);
3930 emit_operand(src, dst);
3931 }
3932
3933 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
3934 assert(VM_Version::supports_evex(), "");
3935 assert(src != xnoreg, "sanity");
3936 InstructionMark im(this);
3937 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3938 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
3939 attributes.set_is_evex_instruction();
3940 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3941 emit_int8(0x31);
3942 emit_operand(src, dst);
3943 }
3944
3945 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
3946 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3947 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3948 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
3949 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3950 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3951 emit_int8(0x33);
3952 emit_int8((unsigned char)(0xC0 | encode));
3953 }
3954
3955 // generic
3956 void Assembler::pop(Register dst) {
3957 int encode = prefix_and_encode(dst->encoding());
3958 emit_int8(0x58 | encode);
3959 }
3960
3961 void Assembler::popcntl(Register dst, Address src) {
3962 assert(VM_Version::supports_popcnt(), "must support");
3963 InstructionMark im(this);
3964 emit_int8((unsigned char)0xF3);
3965 prefix(src, dst);
3966 emit_int8(0x0F);
3967 emit_int8((unsigned char)0xB8);
3968 emit_operand(dst, src);
3969 }
3970
3971 void Assembler::popcntl(Register dst, Register src) {
3972 assert(VM_Version::supports_popcnt(), "must support");
3973 emit_int8((unsigned char)0xF3);
3974 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6095 emit_int8((unsigned char)(0xC0 | encode));
6096 }
6097
6098 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6099 assert(UseAVX > 0, "requires some form of AVX");
6100 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6101 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6102 emit_int8((unsigned char)0xD2);
6103 emit_int8((unsigned char)(0xC0 | encode));
6104 }
6105
6106 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6107 assert(UseAVX > 0, "requires some form of AVX");
6108 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6109 attributes.set_rex_vex_w_reverted();
6110 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6111 emit_int8((unsigned char)0xD3);
6112 emit_int8((unsigned char)(0xC0 | encode));
6113 }
6114
6115 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6116 assert(VM_Version::supports_avx512bw(), "");
6117 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6118 attributes.set_is_evex_instruction();
6119 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6120 emit_int8(0x10);
6121 emit_int8((unsigned char)(0xC0 | encode));
6122 }
6123
6124 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6125 assert(VM_Version::supports_avx512bw(), "");
6126 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6127 attributes.set_is_evex_instruction();
6128 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6129 emit_int8(0x12);
6130 emit_int8((unsigned char)(0xC0 | encode));
6131 }
6132
6133 // Shift packed integers arithmetically right by specified number of bits.
6134 void Assembler::psraw(XMMRegister dst, int shift) {
6135 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6136 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6137 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6138 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6139 emit_int8(0x71);
6140 emit_int8((unsigned char)(0xC0 | encode));
6141 emit_int8(shift & 0xFF);
6142 }
6143
6144 void Assembler::psrad(XMMRegister dst, int shift) {
6145 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6146 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6147 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6148 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6149 emit_int8(0x72);
6150 emit_int8((unsigned char)(0xC0 | encode));
6151 emit_int8(shift & 0xFF);
6152 }
6214 }
6215
6216 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6217 assert(UseAVX > 0, "requires some form of AVX");
6218 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6219 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6220 emit_int8((unsigned char)0xDB);
6221 emit_int8((unsigned char)(0xC0 | encode));
6222 }
6223
6224 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6225 assert(UseAVX > 0, "requires some form of AVX");
6226 InstructionMark im(this);
6227 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6228 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6229 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6230 emit_int8((unsigned char)0xDB);
6231 emit_operand(dst, src);
6232 }
6233
6234 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6235 assert(VM_Version::supports_evex(), "");
6236 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6237 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6238 emit_int8((unsigned char)0xDB);
6239 emit_int8((unsigned char)(0xC0 | encode));
6240 }
6241
6242
6243 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6244 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6245 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6246 attributes.set_rex_vex_w_reverted();
6247 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6248 emit_int8((unsigned char)0xDF);
6249 emit_int8((unsigned char)(0xC0 | encode));
6250 }
6251
6252 void Assembler::por(XMMRegister dst, XMMRegister src) {
6253 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6254 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6255 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6256 emit_int8((unsigned char)0xEB);
6257 emit_int8((unsigned char)(0xC0 | encode));
6258 }
6259
6260 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6261 assert(UseAVX > 0, "requires some form of AVX");
6262 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6263 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6264 emit_int8((unsigned char)0xEB);
6265 emit_int8((unsigned char)(0xC0 | encode));
6266 }
6267
6268 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6269 assert(UseAVX > 0, "requires some form of AVX");
6270 InstructionMark im(this);
6271 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6272 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6273 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6274 emit_int8((unsigned char)0xEB);
6275 emit_operand(dst, src);
6276 }
6277
6278 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6279 assert(VM_Version::supports_evex(), "");
6280 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6281 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6282 emit_int8((unsigned char)0xEB);
6283 emit_int8((unsigned char)(0xC0 | encode));
6284 }
6285
6286
6287 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6288 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6289 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6290 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6291 emit_int8((unsigned char)0xEF);
6292 emit_int8((unsigned char)(0xC0 | encode));
6293 }
6294
6295 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6296 assert(UseAVX > 0, "requires some form of AVX");
6297 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6298 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6299 emit_int8((unsigned char)0xEF);
6300 emit_int8((unsigned char)(0xC0 | encode));
6301 }
6302
6303 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6304 assert(UseAVX > 0, "requires some form of AVX");
6305 InstructionMark im(this);
6306 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6878 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6879 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6880 assert(VM_Version::supports_evex(), "");
6881 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6882 attributes.set_is_evex_instruction();
6883 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6884 emit_int8(0x7C);
6885 emit_int8((unsigned char)(0xC0 | encode));
6886 }
6887
6888 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6889 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6890 assert(VM_Version::supports_evex(), "");
6891 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6892 attributes.set_is_evex_instruction();
6893 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6894 emit_int8(0x7C);
6895 emit_int8((unsigned char)(0xC0 | encode));
6896 }
6897
6898 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
6899 assert(VM_Version::supports_evex(), "");
6900 assert(dst != xnoreg, "sanity");
6901 InstructionMark im(this);
6902 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6903 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6904 attributes.reset_is_clear_context();
6905 attributes.set_embedded_opmask_register_specifier(mask);
6906 attributes.set_is_evex_instruction();
6907 // swap src<->dst for encoding
6908 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6909 emit_int8((unsigned char)0x90);
6910 emit_operand(dst, src);
6911 }
6912
6913 // Carry-Less Multiplication Quadword
6914 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6915 assert(VM_Version::supports_clmul(), "");
6916 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6917 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6918 emit_int8(0x44);
6919 emit_int8((unsigned char)(0xC0 | encode));
6920 emit_int8((unsigned char)mask);
6921 }
6922
6923 // Carry-Less Multiplication Quadword
6924 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6925 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6926 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6927 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6928 emit_int8(0x44);
6929 emit_int8((unsigned char)(0xC0 | encode));
6930 emit_int8((unsigned char)mask);
6931 }
7517 // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7518 int byte4 = (_attributes->is_no_reg_mask()) ?
7519 0 :
7520 _attributes->get_embedded_opmask_register_specifier();
7521 // EVEX.v` for extending EVEX.vvvv or VIDX
7522 byte4 |= (evex_v ? 0: EVEX_V);
7523 // third EXEC.b for broadcast actions
7524 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7525 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7526 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7527 // last is EVEX.z for zero/merge actions
7528 if (_attributes->is_no_reg_mask() == false) {
7529 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7530 }
7531 emit_int8(byte4);
7532 }
7533
7534 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7535 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7536 bool vex_b = adr.base_needs_rex();
7537 bool vex_x;
7538 if (adr.isxmmindex()) {
7539 vex_x = adr.xmmindex_needs_rex();
7540 } else {
7541 vex_x = adr.index_needs_rex();
7542 }
7543 set_attributes(attributes);
7544 attributes->set_current_assembler(this);
7545
7546 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7547 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7548 switch (attributes->get_vector_len()) {
7549 case AVX_128bit:
7550 case AVX_256bit:
7551 attributes->set_is_legacy_mode();
7552 break;
7553 }
7554 }
7555
7556 // For pure EVEX check and see if this instruction
7557 // is allowed in legacy mode and has resources which will
7558 // fit in it. Pure EVEX instructions will use set_is_evex_instruction in their definition,
7559 // else that field is set when we encode to EVEX
7560 if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7561 !_is_managed && !attributes->is_evex_instruction()) {
7562 if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7563 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7564 if (check_register_bank) {
7565 // check nds_enc and xreg_enc for upper bank usage
7566 if (nds_enc < 16 && xreg_enc < 16) {
7567 attributes->set_is_legacy_mode();
7568 }
7569 } else {
7570 attributes->set_is_legacy_mode();
7571 }
7572 }
7573 }
7574
7575 _is_managed = false;
7576 if (UseAVX > 2 && !attributes->is_legacy_mode())
7577 {
7578 bool evex_r = (xreg_enc >= 16);
7579 bool evex_v;
7580 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
7581 if (adr.isxmmindex()) {
7582 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
7583 } else {
7584 evex_v = (nds_enc >= 16);
7585 }
7586 attributes->set_is_evex_instruction();
7587 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7588 } else {
7589 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7590 attributes->set_rex_vex_w(false);
7591 }
7592 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7593 }
7594 }
7595
7596 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7597 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7598 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7599 bool vex_x = false;
7600 set_attributes(attributes);
7601 attributes->set_current_assembler(this);
7602 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7603
7604 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7605 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
|