1330
1331 void Assembler::andl(Register dst, int32_t imm32) {
1332 prefix(dst);
1333 emit_arith(0x81, 0xE0, dst, imm32);
1334 }
1335
1336 void Assembler::andl(Register dst, Address src) {
1337 InstructionMark im(this);
1338 prefix(src, dst);
1339 emit_int8(0x23);
1340 emit_operand(dst, src);
1341 }
1342
1343 void Assembler::andl(Register dst, Register src) {
1344 (void) prefix_and_encode(dst->encoding(), src->encoding());
1345 emit_arith(0x23, 0xC0, dst, src);
1346 }
1347
1348 void Assembler::andnl(Register dst, Register src1, Register src2) {
1349 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1350 int encode = vex_prefix_0F38_and_encode(dst, src1, src2, false);
1351 emit_int8((unsigned char)0xF2);
1352 emit_int8((unsigned char)(0xC0 | encode));
1353 }
1354
1355 void Assembler::andnl(Register dst, Register src1, Address src2) {
1356 InstructionMark im(this);
1357 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1358 vex_prefix_0F38(dst, src1, src2, false);
1359 emit_int8((unsigned char)0xF2);
1360 emit_operand(dst, src2);
1361 }
1362
1363 void Assembler::bsfl(Register dst, Register src) {
1364 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1365 emit_int8(0x0F);
1366 emit_int8((unsigned char)0xBC);
1367 emit_int8((unsigned char)(0xC0 | encode));
1368 }
1369
1370 void Assembler::bsrl(Register dst, Register src) {
1371 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1372 emit_int8(0x0F);
1373 emit_int8((unsigned char)0xBD);
1374 emit_int8((unsigned char)(0xC0 | encode));
1375 }
1376
1377 void Assembler::bswapl(Register reg) { // bswap
1378 int encode = prefix_and_encode(reg->encoding());
1379 emit_int8(0x0F);
1380 emit_int8((unsigned char)(0xC8 | encode));
1381 }
1382
1383 void Assembler::blsil(Register dst, Register src) {
1384 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1385 int encode = vex_prefix_0F38_and_encode(rbx, dst, src, false);
1386 emit_int8((unsigned char)0xF3);
1387 emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389
1390 void Assembler::blsil(Register dst, Address src) {
1391 InstructionMark im(this);
1392 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1393 vex_prefix_0F38(rbx, dst, src, false);
1394 emit_int8((unsigned char)0xF3);
1395 emit_operand(rbx, src);
1396 }
1397
1398 void Assembler::blsmskl(Register dst, Register src) {
1399 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1400 int encode = vex_prefix_0F38_and_encode(rdx, dst, src, false);
1401 emit_int8((unsigned char)0xF3);
1402 emit_int8((unsigned char)(0xC0 | encode));
1403 }
1404
1405 void Assembler::blsmskl(Register dst, Address src) {
1406 InstructionMark im(this);
1407 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1408 vex_prefix_0F38(rdx, dst, src, false);
1409 emit_int8((unsigned char)0xF3);
1410 emit_operand(rdx, src);
1411 }
1412
1413 void Assembler::blsrl(Register dst, Register src) {
1414 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1415 int encode = vex_prefix_0F38_and_encode(rcx, dst, src, false);
1416 emit_int8((unsigned char)0xF3);
1417 emit_int8((unsigned char)(0xC0 | encode));
1418 }
1419
1420 void Assembler::blsrl(Register dst, Address src) {
1421 InstructionMark im(this);
1422 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1423 vex_prefix_0F38(rcx, dst, src, false);
1424 emit_int8((unsigned char)0xF3);
1425 emit_operand(rcx, src);
1426 }
1427
1428 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1429 // suspect disp32 is always good
1430 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1431
1432 if (L.is_bound()) {
1433 const int long_size = 5;
1434 int offs = (int)( target(L) - pc() );
1435 assert(offs <= 0, "assembler error");
1436 InstructionMark im(this);
1437 // 1110 1000 #32-bit disp
1438 emit_int8((unsigned char)0xE8);
1439 emit_data(offs - long_size, rtype, operand);
1440 } else {
1441 InstructionMark im(this);
1442 // 1110 1000 #32-bit disp
1443 L.add_patch_at(code(), locator());
3082 false, AVX_128bit, (VM_Version::supports_avx512bw() == false));
3083 emit_int8(0x70);
3084 emit_operand(dst, src);
3085 emit_int8(mode & 0xFF);
3086 }
3087
3088 void Assembler::psrldq(XMMRegister dst, int shift) {
3089 // Shift 128 bit value in xmm register by number of bytes.
3090 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3091 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, true, VEX_OPCODE_0F,
3092 false, AVX_128bit, (VM_Version::supports_avx512bw() == false));
3093 emit_int8(0x73);
3094 emit_int8((unsigned char)(0xC0 | encode));
3095 emit_int8(shift);
3096 }
3097
3098 void Assembler::ptest(XMMRegister dst, Address src) {
3099 assert(VM_Version::supports_sse4_1(), "");
3100 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3101 InstructionMark im(this);
3102 simd_prefix(dst, src, VEX_SIMD_66, false, VEX_OPCODE_0F_38);
3103 emit_int8(0x17);
3104 emit_operand(dst, src);
3105 }
3106
3107 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3108 assert(VM_Version::supports_sse4_1(), "");
3109 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
3110 false, VEX_OPCODE_0F_38);
3111 emit_int8(0x17);
3112 emit_int8((unsigned char)(0xC0 | encode));
3113 }
3114
3115 void Assembler::vptest(XMMRegister dst, Address src) {
3116 assert(VM_Version::supports_avx(), "");
3117 InstructionMark im(this);
3118 int vector_len = AVX_256bit;
3119 assert(dst != xnoreg, "sanity");
3120 int dst_enc = dst->encoding();
3121 // swap src<->dst for encoding
3122 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector_len);
3123 emit_int8(0x17);
3124 emit_operand(dst, src);
3125 }
3126
3127 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3128 assert(VM_Version::supports_avx(), "");
3129 int vector_len = AVX_256bit;
3130 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
3131 vector_len, VEX_OPCODE_0F_38);
3132 emit_int8(0x17);
3133 emit_int8((unsigned char)(0xC0 | encode));
3134 }
3135
3136 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3137 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3138 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3139 if (VM_Version::supports_evex()) {
3140 tuple_type = EVEX_FVM;
3141 }
3142 emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
3143 }
3144
3145 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3146 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3147 emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
3148 }
3149
3150 void Assembler::punpckldq(XMMRegister dst, Address src) {
3151 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4955 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
4956 VM_Version::supports_avx512dq(), vector_len, false, false);
4957 emit_int8(0x19);
4958 emit_int8((unsigned char)(0xC0 | encode));
4959 // 0x01 - extract from bits 255:128
4960 // 0x02 - extract from bits 383:256
4961 // 0x03 - extract from bits 511:384
4962 emit_int8(value & 0x3);
4963 }
4964
4965 // duplicate 4-bytes integer data from src into 8 locations in dest
4966 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
4967 assert(VM_Version::supports_avx2(), "");
4968 int vector_len = AVX_256bit;
4969 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
4970 vector_len, VEX_OPCODE_0F_38, false);
4971 emit_int8(0x58);
4972 emit_int8((unsigned char)(0xC0 | encode));
4973 }
4974
4975 // duplicate 4-bytes integer data from src into 8 locations in dest
4976 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
4977 assert(VM_Version::supports_evex(), "");
4978 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
4979 vector_len, VEX_OPCODE_0F_38, false);
4980 emit_int8(0x58);
4981 emit_int8((unsigned char)(0xC0 | encode));
4982 }
4983
4984 // Carry-Less Multiplication Quadword
4985 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
4986 assert(VM_Version::supports_clmul(), "");
4987 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, false,
4988 VEX_OPCODE_0F_3A, false, AVX_128bit, true);
4989 emit_int8(0x44);
4990 emit_int8((unsigned char)(0xC0 | encode));
4991 emit_int8((unsigned char)mask);
4992 }
4993
4994 // Carry-Less Multiplication Quadword
4995 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
4996 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
4997 int vector_len = AVX_128bit;
4998 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66,
4999 vector_len, VEX_OPCODE_0F_3A, true);
5000 emit_int8(0x44);
5001 emit_int8((unsigned char)(0xC0 | encode));
5002 emit_int8((unsigned char)mask);
5003 }
5574 // confine pre opcode extensions in pp bits to lower two bits
5575 // of form {66, F3, F2}
5576 byte3 |= pre;
5577 emit_int8(byte3);
5578
5579 // P2: byte 4 as zL'Lbv'aaa
5580 int byte4 = (no_mask_reg) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
5581 // EVEX.v` for extending EVEX.vvvv or VIDX
5582 byte4 |= (evex_v ? 0: EVEX_V);
5583 // third EXEC.b for broadcast actions
5584 byte4 |= (is_extended_context ? EVEX_Rb : 0);
5585 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
5586 byte4 |= ((vector_len) & 0x3) << 5;
5587 // last is EVEX.z for zero/merge actions
5588 byte4 |= (is_merge_context ? EVEX_Z : 0);
5589 emit_int8(byte4);
5590 }
5591
5592 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre,
5593 VexOpcode opc, bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg) {
5594 bool vex_r = (xreg_enc >= 8);
5595 bool vex_b = adr.base_needs_rex();
5596 bool vex_x = adr.index_needs_rex();
5597 avx_vector_len = vector_len;
5598
5599 // if vector length is turned off, revert to AVX for vectors smaller than AVX_512bit
5600 if (VM_Version::supports_avx512vl() == false) {
5601 switch (vector_len) {
5602 case AVX_128bit:
5603 case AVX_256bit:
5604 legacy_mode = true;
5605 break;
5606 }
5607 }
5608
5609 if ((UseAVX > 2) && (legacy_mode == false))
5610 {
5611 bool evex_r = (xreg_enc >= 16);
5612 bool evex_v = (nds_enc >= 16);
5613 is_evex_instruction = true;
5614 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
5615 } else {
5616 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
5617 }
5618 }
5619
5620 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
5621 bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg ) {
5622 bool vex_r = (dst_enc >= 8);
5623 bool vex_b = (src_enc >= 8);
5624 bool vex_x = false;
5625 avx_vector_len = vector_len;
5626
5627 // if vector length is turned off, revert to AVX for vectors smaller than AVX_512bit
5628 if (VM_Version::supports_avx512vl() == false) {
5629 switch (vector_len) {
5630 case AVX_128bit:
5631 case AVX_256bit:
5632 legacy_mode = true;
5633 break;
5634 }
5635 }
5636
5637 if ((UseAVX > 2) && (legacy_mode == false))
5638 {
5639 bool evex_r = (dst_enc >= 16);
5640 bool evex_v = (nds_enc >= 16);
5641 // can use vex_x as bank extender on rm encoding
5642 vex_x = (src_enc >= 16);
5643 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6248
6249 void Assembler::andq(Register dst, int32_t imm32) {
6250 (void) prefixq_and_encode(dst->encoding());
6251 emit_arith(0x81, 0xE0, dst, imm32);
6252 }
6253
6254 void Assembler::andq(Register dst, Address src) {
6255 InstructionMark im(this);
6256 prefixq(src, dst);
6257 emit_int8(0x23);
6258 emit_operand(dst, src);
6259 }
6260
6261 void Assembler::andq(Register dst, Register src) {
6262 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6263 emit_arith(0x23, 0xC0, dst, src);
6264 }
6265
6266 void Assembler::andnq(Register dst, Register src1, Register src2) {
6267 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6268 int encode = vex_prefix_0F38_and_encode_q(dst, src1, src2);
6269 emit_int8((unsigned char)0xF2);
6270 emit_int8((unsigned char)(0xC0 | encode));
6271 }
6272
6273 void Assembler::andnq(Register dst, Register src1, Address src2) {
6274 if (VM_Version::supports_evex()) {
6275 tuple_type = EVEX_T1S;
6276 input_size_in_bits = EVEX_64bit;
6277 }
6278 InstructionMark im(this);
6279 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6280 vex_prefix_0F38_q(dst, src1, src2);
6281 emit_int8((unsigned char)0xF2);
6282 emit_operand(dst, src2);
6283 }
6284
6285 void Assembler::bsfq(Register dst, Register src) {
6286 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6287 emit_int8(0x0F);
6288 emit_int8((unsigned char)0xBC);
6289 emit_int8((unsigned char)(0xC0 | encode));
6290 }
6291
6292 void Assembler::bsrq(Register dst, Register src) {
6293 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6294 emit_int8(0x0F);
6295 emit_int8((unsigned char)0xBD);
6296 emit_int8((unsigned char)(0xC0 | encode));
6297 }
6298
6299 void Assembler::bswapq(Register reg) {
6300 int encode = prefixq_and_encode(reg->encoding());
6301 emit_int8(0x0F);
6302 emit_int8((unsigned char)(0xC8 | encode));
6303 }
6304
6305 void Assembler::blsiq(Register dst, Register src) {
6306 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6307 int encode = vex_prefix_0F38_and_encode_q(rbx, dst, src);
6308 emit_int8((unsigned char)0xF3);
6309 emit_int8((unsigned char)(0xC0 | encode));
6310 }
6311
6312 void Assembler::blsiq(Register dst, Address src) {
6313 InstructionMark im(this);
6314 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6315 vex_prefix_0F38_q(rbx, dst, src);
6316 emit_int8((unsigned char)0xF3);
6317 emit_operand(rbx, src);
6318 }
6319
6320 void Assembler::blsmskq(Register dst, Register src) {
6321 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6322 int encode = vex_prefix_0F38_and_encode_q(rdx, dst, src);
6323 emit_int8((unsigned char)0xF3);
6324 emit_int8((unsigned char)(0xC0 | encode));
6325 }
6326
6327 void Assembler::blsmskq(Register dst, Address src) {
6328 InstructionMark im(this);
6329 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6330 vex_prefix_0F38_q(rdx, dst, src);
6331 emit_int8((unsigned char)0xF3);
6332 emit_operand(rdx, src);
6333 }
6334
6335 void Assembler::blsrq(Register dst, Register src) {
6336 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6337 int encode = vex_prefix_0F38_and_encode_q(rcx, dst, src);
6338 emit_int8((unsigned char)0xF3);
6339 emit_int8((unsigned char)(0xC0 | encode));
6340 }
6341
6342 void Assembler::blsrq(Register dst, Address src) {
6343 InstructionMark im(this);
6344 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6345 vex_prefix_0F38_q(rcx, dst, src);
6346 emit_int8((unsigned char)0xF3);
6347 emit_operand(rcx, src);
6348 }
6349
6350 void Assembler::cdqq() {
6351 prefix(REX_W);
6352 emit_int8((unsigned char)0x99);
6353 }
6354
6355 void Assembler::clflush(Address adr) {
6356 prefix(adr);
6357 emit_int8(0x0F);
6358 emit_int8((unsigned char)0xAE);
6359 emit_operand(rdi, adr);
6360 }
6361
6362 void Assembler::cmovq(Condition cc, Register dst, Register src) {
6363 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6364 emit_int8(0x0F);
6365 emit_int8(0x40 | cc);
|
1330
1331 void Assembler::andl(Register dst, int32_t imm32) {
1332 prefix(dst);
1333 emit_arith(0x81, 0xE0, dst, imm32);
1334 }
1335
1336 void Assembler::andl(Register dst, Address src) {
1337 InstructionMark im(this);
1338 prefix(src, dst);
1339 emit_int8(0x23);
1340 emit_operand(dst, src);
1341 }
1342
1343 void Assembler::andl(Register dst, Register src) {
1344 (void) prefix_and_encode(dst->encoding(), src->encoding());
1345 emit_arith(0x23, 0xC0, dst, src);
1346 }
1347
1348 void Assembler::andnl(Register dst, Register src1, Register src2) {
1349 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1350 int encode = vex_prefix_0F38_and_encode_legacy(dst, src1, src2, false);
1351 emit_int8((unsigned char)0xF2);
1352 emit_int8((unsigned char)(0xC0 | encode));
1353 }
1354
1355 void Assembler::andnl(Register dst, Register src1, Address src2) {
1356 InstructionMark im(this);
1357 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1358 vex_prefix_0F38_legacy(dst, src1, src2, false);
1359 emit_int8((unsigned char)0xF2);
1360 emit_operand(dst, src2);
1361 }
1362
1363 void Assembler::bsfl(Register dst, Register src) {
1364 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1365 emit_int8(0x0F);
1366 emit_int8((unsigned char)0xBC);
1367 emit_int8((unsigned char)(0xC0 | encode));
1368 }
1369
1370 void Assembler::bsrl(Register dst, Register src) {
1371 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1372 emit_int8(0x0F);
1373 emit_int8((unsigned char)0xBD);
1374 emit_int8((unsigned char)(0xC0 | encode));
1375 }
1376
1377 void Assembler::bswapl(Register reg) { // bswap
1378 int encode = prefix_and_encode(reg->encoding());
1379 emit_int8(0x0F);
1380 emit_int8((unsigned char)(0xC8 | encode));
1381 }
1382
1383 void Assembler::blsil(Register dst, Register src) {
1384 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1385 int encode = vex_prefix_0F38_and_encode_legacy(rbx, dst, src, false);
1386 emit_int8((unsigned char)0xF3);
1387 emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389
1390 void Assembler::blsil(Register dst, Address src) {
1391 InstructionMark im(this);
1392 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1393 vex_prefix_0F38_legacy(rbx, dst, src, false);
1394 emit_int8((unsigned char)0xF3);
1395 emit_operand(rbx, src);
1396 }
1397
1398 void Assembler::blsmskl(Register dst, Register src) {
1399 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1400 int encode = vex_prefix_0F38_and_encode_legacy(rdx, dst, src, false);
1401 emit_int8((unsigned char)0xF3);
1402 emit_int8((unsigned char)(0xC0 | encode));
1403 }
1404
1405 void Assembler::blsmskl(Register dst, Address src) {
1406 InstructionMark im(this);
1407 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1408 vex_prefix_0F38(rdx, dst, src, false);
1409 emit_int8((unsigned char)0xF3);
1410 emit_operand(rdx, src);
1411 }
1412
1413 void Assembler::blsrl(Register dst, Register src) {
1414 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1415 int encode = vex_prefix_0F38_and_encode_legacy(rcx, dst, src, false);
1416 emit_int8((unsigned char)0xF3);
1417 emit_int8((unsigned char)(0xC0 | encode));
1418 }
1419
1420 void Assembler::blsrl(Register dst, Address src) {
1421 InstructionMark im(this);
1422 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1423 vex_prefix_0F38_legacy(rcx, dst, src, false);
1424 emit_int8((unsigned char)0xF3);
1425 emit_operand(rcx, src);
1426 }
1427
1428 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1429 // suspect disp32 is always good
1430 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1431
1432 if (L.is_bound()) {
1433 const int long_size = 5;
1434 int offs = (int)( target(L) - pc() );
1435 assert(offs <= 0, "assembler error");
1436 InstructionMark im(this);
1437 // 1110 1000 #32-bit disp
1438 emit_int8((unsigned char)0xE8);
1439 emit_data(offs - long_size, rtype, operand);
1440 } else {
1441 InstructionMark im(this);
1442 // 1110 1000 #32-bit disp
1443 L.add_patch_at(code(), locator());
3082 false, AVX_128bit, (VM_Version::supports_avx512bw() == false));
3083 emit_int8(0x70);
3084 emit_operand(dst, src);
3085 emit_int8(mode & 0xFF);
3086 }
3087
3088 void Assembler::psrldq(XMMRegister dst, int shift) {
3089 // Shift 128 bit value in xmm register by number of bytes.
3090 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3091 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, true, VEX_OPCODE_0F,
3092 false, AVX_128bit, (VM_Version::supports_avx512bw() == false));
3093 emit_int8(0x73);
3094 emit_int8((unsigned char)(0xC0 | encode));
3095 emit_int8(shift);
3096 }
3097
3098 void Assembler::ptest(XMMRegister dst, Address src) {
3099 assert(VM_Version::supports_sse4_1(), "");
3100 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3101 InstructionMark im(this);
3102 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, false,
3103 VEX_OPCODE_0F_38, false, AVX_128bit, true);
3104 emit_int8(0x17);
3105 emit_operand(dst, src);
3106 }
3107
3108 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3109 assert(VM_Version::supports_sse4_1(), "");
3110 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, false,
3111 VEX_OPCODE_0F_38, false, AVX_128bit, true);
3112 emit_int8(0x17);
3113 emit_int8((unsigned char)(0xC0 | encode));
3114 }
3115
3116 void Assembler::vptest(XMMRegister dst, Address src) {
3117 assert(VM_Version::supports_avx(), "");
3118 InstructionMark im(this);
3119 int vector_len = AVX_256bit;
3120 assert(dst != xnoreg, "sanity");
3121 int dst_enc = dst->encoding();
3122 // swap src<->dst for encoding
3123 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector_len, true, false);
3124 emit_int8(0x17);
3125 emit_operand(dst, src);
3126 }
3127
3128 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3129 assert(VM_Version::supports_avx(), "");
3130 int vector_len = AVX_256bit;
3131 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
3132 vector_len, VEX_OPCODE_0F_38, true, false);
3133 emit_int8(0x17);
3134 emit_int8((unsigned char)(0xC0 | encode));
3135 }
3136
3137 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3138 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3139 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3140 if (VM_Version::supports_evex()) {
3141 tuple_type = EVEX_FVM;
3142 }
3143 emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
3144 }
3145
3146 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3147 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3148 emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
3149 }
3150
3151 void Assembler::punpckldq(XMMRegister dst, Address src) {
3152 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4956 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
4957 VM_Version::supports_avx512dq(), vector_len, false, false);
4958 emit_int8(0x19);
4959 emit_int8((unsigned char)(0xC0 | encode));
4960 // 0x01 - extract from bits 255:128
4961 // 0x02 - extract from bits 383:256
4962 // 0x03 - extract from bits 511:384
4963 emit_int8(value & 0x3);
4964 }
4965
4966 // duplicate 4-bytes integer data from src into 8 locations in dest
4967 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
4968 assert(VM_Version::supports_avx2(), "");
4969 int vector_len = AVX_256bit;
4970 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
4971 vector_len, VEX_OPCODE_0F_38, false);
4972 emit_int8(0x58);
4973 emit_int8((unsigned char)(0xC0 | encode));
4974 }
4975
4976 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
4977 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
4978 assert(VM_Version::supports_evex(), "");
4979 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
4980 vector_len, VEX_OPCODE_0F_38, false);
4981 emit_int8(0x78);
4982 emit_int8((unsigned char)(0xC0 | encode));
4983 }
4984
4985 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
4986 assert(VM_Version::supports_evex(), "");
4987 tuple_type = EVEX_T1S;
4988 input_size_in_bits = EVEX_8bit;
4989 InstructionMark im(this);
4990 assert(dst != xnoreg, "sanity");
4991 int dst_enc = dst->encoding();
4992 // swap src<->dst for encoding
4993 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector_len);
4994 emit_int8(0x78);
4995 emit_operand(dst, src);
4996 }
4997
4998 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
4999 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5000 assert(VM_Version::supports_evex(), "");
5001 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
5002 vector_len, VEX_OPCODE_0F_38, false);
5003 emit_int8(0x79);
5004 emit_int8((unsigned char)(0xC0 | encode));
5005 }
5006
5007 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5008 assert(VM_Version::supports_evex(), "");
5009 tuple_type = EVEX_T1S;
5010 input_size_in_bits = EVEX_16bit;
5011 InstructionMark im(this);
5012 assert(dst != xnoreg, "sanity");
5013 int dst_enc = dst->encoding();
5014 // swap src<->dst for encoding
5015 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector_len);
5016 emit_int8(0x79);
5017 emit_operand(dst, src);
5018 }
5019
5020 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5021 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5022 assert(VM_Version::supports_evex(), "");
5023 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
5024 vector_len, VEX_OPCODE_0F_38, false);
5025 emit_int8(0x58);
5026 emit_int8((unsigned char)(0xC0 | encode));
5027 }
5028
5029 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5030 assert(VM_Version::supports_evex(), "");
5031 tuple_type = EVEX_T1S;
5032 input_size_in_bits = EVEX_32bit;
5033 InstructionMark im(this);
5034 assert(dst != xnoreg, "sanity");
5035 int dst_enc = dst->encoding();
5036 // swap src<->dst for encoding
5037 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector_len);
5038 emit_int8(0x58);
5039 emit_operand(dst, src);
5040 }
5041
5042 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5043 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5044 assert(VM_Version::supports_evex(), "");
5045 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5046 VEX_OPCODE_0F_38, true, vector_len, false, false);
5047 emit_int8(0x59);
5048 emit_int8((unsigned char)(0xC0 | encode));
5049 }
5050
5051 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5052 assert(VM_Version::supports_evex(), "");
5053 tuple_type = EVEX_T1S;
5054 input_size_in_bits = EVEX_64bit;
5055 InstructionMark im(this);
5056 assert(dst != xnoreg, "sanity");
5057 int dst_enc = dst->encoding();
5058 // swap src<->dst for encoding
5059 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, true, vector_len);
5060 emit_int8(0x59);
5061 emit_operand(dst, src);
5062 }
5063
5064 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5065 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5066 assert(VM_Version::supports_evex(), "");
5067 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5068 VEX_OPCODE_0F_38, false, vector_len, false, false);
5069 emit_int8(0x18);
5070 emit_int8((unsigned char)(0xC0 | encode));
5071 }
5072
5073 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5074 assert(VM_Version::supports_evex(), "");
5075 tuple_type = EVEX_T1S;
5076 input_size_in_bits = EVEX_32bit;
5077 InstructionMark im(this);
5078 assert(dst != xnoreg, "sanity");
5079 int dst_enc = dst->encoding();
5080 // swap src<->dst for encoding
5081 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector_len);
5082 emit_int8(0x18);
5083 emit_operand(dst, src);
5084 }
5085
5086 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5087 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5088 assert(VM_Version::supports_evex(), "");
5089 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5090 VEX_OPCODE_0F_38, true, vector_len, false, false);
5091 emit_int8(0x19);
5092 emit_int8((unsigned char)(0xC0 | encode));
5093 }
5094
5095 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5096 assert(VM_Version::supports_evex(), "");
5097 tuple_type = EVEX_T1S;
5098 input_size_in_bits = EVEX_64bit;
5099 InstructionMark im(this);
5100 assert(dst != xnoreg, "sanity");
5101 int dst_enc = dst->encoding();
5102 // swap src<->dst for encoding
5103 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, true, vector_len);
5104 emit_int8(0x19);
5105 emit_operand(dst, src);
5106 }
5107
5108 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5109 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
5110 assert(VM_Version::supports_evex(), "");
5111 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5112 VEX_OPCODE_0F_38, false, vector_len, false, false);
5113 emit_int8(0x7A);
5114 emit_int8((unsigned char)(0xC0 | encode));
5115 }
5116
5117 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5118 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
5119 assert(VM_Version::supports_evex(), "");
5120 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5121 VEX_OPCODE_0F_38, false, vector_len, false, false);
5122 emit_int8(0x7B);
5123 emit_int8((unsigned char)(0xC0 | encode));
5124 }
5125
5126 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5127 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
5128 assert(VM_Version::supports_evex(), "");
5129 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5130 VEX_OPCODE_0F_38, false, vector_len, false, false);
5131 emit_int8(0x7C);
5132 emit_int8((unsigned char)(0xC0 | encode));
5133 }
5134
5135 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5136 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
5137 assert(VM_Version::supports_evex(), "");
5138 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66,
5139 VEX_OPCODE_0F_38, true, vector_len, false, false);
5140 emit_int8(0x7C);
5141 emit_int8((unsigned char)(0xC0 | encode));
5142 }
5143
5144 // Carry-Less Multiplication Quadword
5145 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5146 assert(VM_Version::supports_clmul(), "");
5147 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, false,
5148 VEX_OPCODE_0F_3A, false, AVX_128bit, true);
5149 emit_int8(0x44);
5150 emit_int8((unsigned char)(0xC0 | encode));
5151 emit_int8((unsigned char)mask);
5152 }
5153
5154 // Carry-Less Multiplication Quadword
5155 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5156 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5157 int vector_len = AVX_128bit;
5158 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66,
5159 vector_len, VEX_OPCODE_0F_3A, true);
5160 emit_int8(0x44);
5161 emit_int8((unsigned char)(0xC0 | encode));
5162 emit_int8((unsigned char)mask);
5163 }
5734 // confine pre opcode extensions in pp bits to lower two bits
5735 // of form {66, F3, F2}
5736 byte3 |= pre;
5737 emit_int8(byte3);
5738
5739 // P2: byte 4 as zL'Lbv'aaa
5740 int byte4 = (no_mask_reg) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
5741 // EVEX.v` for extending EVEX.vvvv or VIDX
5742 byte4 |= (evex_v ? 0: EVEX_V);
5743 // third EXEC.b for broadcast actions
5744 byte4 |= (is_extended_context ? EVEX_Rb : 0);
5745 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
5746 byte4 |= ((vector_len) & 0x3) << 5;
5747 // last is EVEX.z for zero/merge actions
5748 byte4 |= (is_merge_context ? EVEX_Z : 0);
5749 emit_int8(byte4);
5750 }
5751
5752 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre,
5753 VexOpcode opc, bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg) {
5754 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
5755 bool vex_b = adr.base_needs_rex();
5756 bool vex_x = adr.index_needs_rex();
5757 avx_vector_len = vector_len;
5758
5759 // if vector length is turned off, revert to AVX for vectors smaller than AVX_512bit
5760 if (VM_Version::supports_avx512vl() == false) {
5761 switch (vector_len) {
5762 case AVX_128bit:
5763 case AVX_256bit:
5764 legacy_mode = true;
5765 break;
5766 }
5767 }
5768
5769 if ((UseAVX > 2) && (legacy_mode == false))
5770 {
5771 bool evex_r = (xreg_enc >= 16);
5772 bool evex_v = (nds_enc >= 16);
5773 is_evex_instruction = true;
5774 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
5775 } else {
5776 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
5777 }
5778 }
5779
5780 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
5781 bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg ) {
5782 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
5783 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
5784 bool vex_x = false;
5785 avx_vector_len = vector_len;
5786
5787 // if vector length is turned off, revert to AVX for vectors smaller than AVX_512bit
5788 if (VM_Version::supports_avx512vl() == false) {
5789 switch (vector_len) {
5790 case AVX_128bit:
5791 case AVX_256bit:
5792 legacy_mode = true;
5793 break;
5794 }
5795 }
5796
5797 if ((UseAVX > 2) && (legacy_mode == false))
5798 {
5799 bool evex_r = (dst_enc >= 16);
5800 bool evex_v = (nds_enc >= 16);
5801 // can use vex_x as bank extender on rm encoding
5802 vex_x = (src_enc >= 16);
5803 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6408
6409 void Assembler::andq(Register dst, int32_t imm32) {
6410 (void) prefixq_and_encode(dst->encoding());
6411 emit_arith(0x81, 0xE0, dst, imm32);
6412 }
6413
6414 void Assembler::andq(Register dst, Address src) {
6415 InstructionMark im(this);
6416 prefixq(src, dst);
6417 emit_int8(0x23);
6418 emit_operand(dst, src);
6419 }
6420
6421 void Assembler::andq(Register dst, Register src) {
6422 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6423 emit_arith(0x23, 0xC0, dst, src);
6424 }
6425
6426 void Assembler::andnq(Register dst, Register src1, Register src2) {
6427 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6428 int encode = vex_prefix_0F38_and_encode_q_legacy(dst, src1, src2);
6429 emit_int8((unsigned char)0xF2);
6430 emit_int8((unsigned char)(0xC0 | encode));
6431 }
6432
6433 void Assembler::andnq(Register dst, Register src1, Address src2) {
6434 InstructionMark im(this);
6435 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6436 vex_prefix_0F38_q_legacy(dst, src1, src2);
6437 emit_int8((unsigned char)0xF2);
6438 emit_operand(dst, src2);
6439 }
6440
6441 void Assembler::bsfq(Register dst, Register src) {
6442 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6443 emit_int8(0x0F);
6444 emit_int8((unsigned char)0xBC);
6445 emit_int8((unsigned char)(0xC0 | encode));
6446 }
6447
6448 void Assembler::bsrq(Register dst, Register src) {
6449 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6450 emit_int8(0x0F);
6451 emit_int8((unsigned char)0xBD);
6452 emit_int8((unsigned char)(0xC0 | encode));
6453 }
6454
6455 void Assembler::bswapq(Register reg) {
6456 int encode = prefixq_and_encode(reg->encoding());
6457 emit_int8(0x0F);
6458 emit_int8((unsigned char)(0xC8 | encode));
6459 }
6460
6461 void Assembler::blsiq(Register dst, Register src) {
6462 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6463 int encode = vex_prefix_0F38_and_encode_q_legacy(rbx, dst, src);
6464 emit_int8((unsigned char)0xF3);
6465 emit_int8((unsigned char)(0xC0 | encode));
6466 }
6467
6468 void Assembler::blsiq(Register dst, Address src) {
6469 InstructionMark im(this);
6470 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6471 vex_prefix_0F38_q_legacy(rbx, dst, src);
6472 emit_int8((unsigned char)0xF3);
6473 emit_operand(rbx, src);
6474 }
6475
6476 void Assembler::blsmskq(Register dst, Register src) {
6477 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6478 int encode = vex_prefix_0F38_and_encode_q_legacy(rdx, dst, src);
6479 emit_int8((unsigned char)0xF3);
6480 emit_int8((unsigned char)(0xC0 | encode));
6481 }
6482
6483 void Assembler::blsmskq(Register dst, Address src) {
6484 InstructionMark im(this);
6485 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6486 vex_prefix_0F38_q_legacy(rdx, dst, src);
6487 emit_int8((unsigned char)0xF3);
6488 emit_operand(rdx, src);
6489 }
6490
6491 void Assembler::blsrq(Register dst, Register src) {
6492 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6493 int encode = vex_prefix_0F38_and_encode_q_legacy(rcx, dst, src);
6494 emit_int8((unsigned char)0xF3);
6495 emit_int8((unsigned char)(0xC0 | encode));
6496 }
6497
6498 void Assembler::blsrq(Register dst, Address src) {
6499 InstructionMark im(this);
6500 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6501 vex_prefix_0F38_q_legacy(rcx, dst, src);
6502 emit_int8((unsigned char)0xF3);
6503 emit_operand(rcx, src);
6504 }
6505
6506 void Assembler::cdqq() {
6507 prefix(REX_W);
6508 emit_int8((unsigned char)0x99);
6509 }
6510
6511 void Assembler::clflush(Address adr) {
6512 prefix(adr);
6513 emit_int8(0x0F);
6514 emit_int8((unsigned char)0xAE);
6515 emit_operand(rdi, adr);
6516 }
6517
6518 void Assembler::cmovq(Condition cc, Register dst, Register src) {
6519 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6520 emit_int8(0x0F);
6521 emit_int8(0x40 | cc);
|