< prev index next >

src/cpu/x86/vm/assembler_x86.cpp

Print this page




 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx += 2 + ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;


 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && _is_evex_instruction) {
 398     switch (_tuple_type) {


 399     case EVEX_FV:
 400       if ((_evex_encoding & VEX_W) == VEX_W) {
 401         mod_idx += 2 + ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 402       } else {
 403         mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 404       }
 405       break;
 406 
 407     case EVEX_HV:
 408       mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 409       break;
 410 
 411     case EVEX_FVM:
 412       break;
 413 
 414     case EVEX_T1S:
 415       switch (_input_size_in_bits) {
 416       case EVEX_8bit:
 417         break;
 418 
 419       case EVEX_16bit:
 420         mod_idx = 1;
 421         break;
 422 
 423       case EVEX_32bit:
 424         mod_idx = 2;
 425         break;
 426 
 427       case EVEX_64bit:
 428         mod_idx = 3;
 429         break;
 430       }
 431       break;
 432 
 433     case EVEX_T1F:
 434     case EVEX_T2:
 435     case EVEX_T4:
 436       mod_idx = (_input_size_in_bits == EVEX_64bit) ? 1 : 0;
 437       break;
 438 
 439     case EVEX_T8:
 440       break;
 441 
 442     case EVEX_HVM:
 443       break;
 444 
 445     case EVEX_QVM:
 446       break;
 447 
 448     case EVEX_OVM:
 449       break;
 450 
 451     case EVEX_M128:
 452       break;
 453 
 454     case EVEX_DUP:
 455       break;
 456 
 457     default:
 458       assert(0, "no valid evex tuple_table entry");
 459       break;
 460     }
 461 
 462     if (_avx_vector_len >= AVX_128bit && _avx_vector_len <= AVX_512bit) {
 463       int disp_factor = tuple_table[_tuple_type + mod_idx][_avx_vector_len];

 464       if ((disp % disp_factor) == 0) {
 465         int new_disp = disp / disp_factor;
 466         if (is8bit(new_disp)) {
 467           disp = new_disp;
 468         }
 469       } else {
 470         return false;
 471       }
 472     }
 473   }
 474   return is8bit(disp);
 475 }
 476 
 477 
 478 void Assembler::emit_operand(Register reg, Register base, Register index,
 479                              Address::ScaleFactor scale, int disp,
 480                              RelocationHolder const& rspec,
 481                              int rip_relative_correction) {
 482   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 483 


 574       // at the start of the instruction. That needs more correction here.
 575       // intptr_t disp = target - next_ip;
 576       assert(inst_mark() != NULL, "must be inside InstructionMark");
 577       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 578       int64_t adjusted = disp;
 579       // Do rip-rel adjustment for 64bit
 580       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 581       assert(is_simm32(adjusted),
 582              "must be 32bit offset (RIP relative address)");
 583       emit_data((int32_t) adjusted, rspec, disp32_operand);
 584 
 585     } else {
 586       // 32bit never did this, did everything as the rip-rel/disp code above
 587       // [disp] ABSOLUTE
 588       // [00 reg 100][00 100 101] disp32
 589       emit_int8(0x04 | regenc);
 590       emit_int8(0x25);
 591       emit_data(disp, rspec, disp32_operand);
 592     }
 593   }
 594   _is_evex_instruction = false;
 595 }
 596 
 597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 598                              Address::ScaleFactor scale, int disp,
 599                              RelocationHolder const& rspec) {
 600   if (UseAVX > 2) {
 601     int xreg_enc = reg->encoding();
 602     if (xreg_enc > 15) {
 603       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 604       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 605       return;
 606     }
 607   }
 608   emit_operand((Register)reg, base, index, scale, disp, rspec);
 609 }
 610 
 611 // Secret local extension to Assembler::WhichOperand:
 612 #define end_pc_operand (_WhichOperand_limit)
 613 
 614 address Assembler::locate_operand(address inst, WhichOperand which) {


 753       tail_size = 1;
 754     case 0x38: // ptest, pmovzxbw
 755       ip++; // skip opcode
 756       debug_only(has_disp32 = true); // has both kinds of operands!
 757       break;
 758 
 759     case 0x70: // pshufd r, r/a, #8
 760       debug_only(has_disp32 = true); // has both kinds of operands!
 761     case 0x73: // psrldq r, #8
 762       tail_size = 1;
 763       break;
 764 
 765     case 0x12: // movlps
 766     case 0x28: // movaps
 767     case 0x2E: // ucomiss
 768     case 0x2F: // comiss
 769     case 0x54: // andps
 770     case 0x55: // andnps
 771     case 0x56: // orps
 772     case 0x57: // xorps
 773     case 0x59: //mulpd
 774     case 0x6E: // movd
 775     case 0x7E: // movd
 776     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 777       debug_only(has_disp32 = true);
 778       break;
 779 
 780     case 0xAD: // shrd r, a, %cl
 781     case 0xAF: // imul r, a
 782     case 0xBE: // movsbl r, a (movsxb)
 783     case 0xBF: // movswl r, a (movsxw)
 784     case 0xB6: // movzbl r, a (movzxb)
 785     case 0xB7: // movzwl r, a (movzxw)
 786     case REP16(0x40): // cmovl cc, r, a
 787     case 0xB0: // cmpxchgb
 788     case 0xB1: // cmpxchg
 789     case 0xC1: // xaddl
 790     case 0xC7: // cmpxchg8
 791     case REP16(0x90): // setcc a
 792       debug_only(has_disp32 = true);
 793       // fall out of the switch to decode the address


1217   emit_int8(0x0F);
1218   emit_int8(0x1F);
1219   emit_int8((unsigned char)0x80);
1220                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1221   emit_int32(0);   // 32-bits offset (4 bytes)
1222 }
1223 
1224 void Assembler::addr_nop_8() {
1225   assert(UseAddressNop, "no CPU support");
1226   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1227   emit_int8(0x0F);
1228   emit_int8(0x1F);
1229   emit_int8((unsigned char)0x84);
1230                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1231   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1232   emit_int32(0);   // 32-bits offset (4 bytes)
1233 }
1234 
1235 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1236   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1237   if (VM_Version::supports_evex()) {
1238     emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
1239   } else {
1240     emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
1241   }
1242 }
1243 
1244 void Assembler::addsd(XMMRegister dst, Address src) {
1245   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1246   if (VM_Version::supports_evex()) {
1247     _tuple_type = EVEX_T1S;
1248     _input_size_in_bits = EVEX_64bit;
1249     emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
1250   } else {
1251     emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
1252   }
1253 }
1254 
1255 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1256   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1257   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);



1258 }
1259 
1260 void Assembler::addss(XMMRegister dst, Address src) {
1261   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1262   if (VM_Version::supports_evex()) {
1263     _tuple_type = EVEX_T1S;
1264     _input_size_in_bits = EVEX_32bit;
1265   }
1266   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);

1267 }
1268 
1269 void Assembler::aesdec(XMMRegister dst, Address src) {
1270   assert(VM_Version::supports_aes(), "");
1271   InstructionMark im(this);
1272   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1273               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1274   emit_int8((unsigned char)0xDE);
1275   emit_operand(dst, src);
1276 }
1277 
1278 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1279   assert(VM_Version::supports_aes(), "");
1280   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1281                                       VEX_OPCODE_0F_38,  /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1282   emit_int8((unsigned char)0xDE);
1283   emit_int8(0xC0 | encode);
1284 }
1285 
1286 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1287   assert(VM_Version::supports_aes(), "");
1288   InstructionMark im(this);
1289   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1290               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit,  /* legacy_mode */ true);
1291   emit_int8((unsigned char)0xDF);
1292   emit_operand(dst, src);
1293 }
1294 
1295 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1296   assert(VM_Version::supports_aes(), "");
1297   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1298                                       VEX_OPCODE_0F_38,  /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1299   emit_int8((unsigned char)0xDF);
1300   emit_int8((unsigned char)(0xC0 | encode));
1301 }
1302 
1303 void Assembler::aesenc(XMMRegister dst, Address src) {
1304   assert(VM_Version::supports_aes(), "");
1305   InstructionMark im(this);
1306   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1307               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1308   emit_int8((unsigned char)0xDC);
1309   emit_operand(dst, src);
1310 }
1311 
1312 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1313   assert(VM_Version::supports_aes(), "");
1314   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1315                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1316   emit_int8((unsigned char)0xDC);
1317   emit_int8(0xC0 | encode);
1318 }
1319 
1320 void Assembler::aesenclast(XMMRegister dst, Address src) {
1321   assert(VM_Version::supports_aes(), "");
1322   InstructionMark im(this);
1323   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1324               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit,  /* legacy_mode */ true);
1325   emit_int8((unsigned char)0xDD);
1326   emit_operand(dst, src);
1327 }
1328 
1329 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1330   assert(VM_Version::supports_aes(), "");
1331   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1332                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1333   emit_int8((unsigned char)0xDD);
1334   emit_int8((unsigned char)(0xC0 | encode));
1335 }
1336 
1337 void Assembler::andl(Address dst, int32_t imm32) {
1338   InstructionMark im(this);
1339   prefix(dst);
1340   emit_int8((unsigned char)0x81);
1341   emit_operand(rsp, dst, 4);
1342   emit_int32(imm32);
1343 }
1344 
1345 void Assembler::andl(Register dst, int32_t imm32) {
1346   prefix(dst);
1347   emit_arith(0x81, 0xE0, dst, imm32);
1348 }
1349 
1350 void Assembler::andl(Register dst, Address src) {
1351   InstructionMark im(this);
1352   prefix(src, dst);
1353   emit_int8(0x23);
1354   emit_operand(dst, src);
1355 }
1356 
1357 void Assembler::andl(Register dst, Register src) {
1358   (void) prefix_and_encode(dst->encoding(), src->encoding());
1359   emit_arith(0x23, 0xC0, dst, src);
1360 }
1361 
1362 void Assembler::andnl(Register dst, Register src1, Register src2) {
1363   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1364   int encode = vex_prefix_0F38_and_encode_legacy(dst, src1, src2);

1365   emit_int8((unsigned char)0xF2);
1366   emit_int8((unsigned char)(0xC0 | encode));
1367 }
1368 
1369 void Assembler::andnl(Register dst, Register src1, Address src2) {
1370   InstructionMark im(this);
1371   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1372   vex_prefix_0F38_legacy(dst, src1, src2);


1373   emit_int8((unsigned char)0xF2);
1374   emit_operand(dst, src2);
1375 }
1376 
1377 void Assembler::bsfl(Register dst, Register src) {
1378   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1379   emit_int8(0x0F);
1380   emit_int8((unsigned char)0xBC);
1381   emit_int8((unsigned char)(0xC0 | encode));
1382 }
1383 
1384 void Assembler::bsrl(Register dst, Register src) {
1385   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1386   emit_int8(0x0F);
1387   emit_int8((unsigned char)0xBD);
1388   emit_int8((unsigned char)(0xC0 | encode));
1389 }
1390 
1391 void Assembler::bswapl(Register reg) { // bswap
1392   int encode = prefix_and_encode(reg->encoding());
1393   emit_int8(0x0F);
1394   emit_int8((unsigned char)(0xC8 | encode));
1395 }
1396 
1397 void Assembler::blsil(Register dst, Register src) {
1398   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1399   int encode = vex_prefix_0F38_and_encode_legacy(rbx, dst, src);

1400   emit_int8((unsigned char)0xF3);
1401   emit_int8((unsigned char)(0xC0 | encode));
1402 }
1403 
1404 void Assembler::blsil(Register dst, Address src) {
1405   InstructionMark im(this);
1406   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1407   vex_prefix_0F38_legacy(rbx, dst, src);


1408   emit_int8((unsigned char)0xF3);
1409   emit_operand(rbx, src);
1410 }
1411 
1412 void Assembler::blsmskl(Register dst, Register src) {
1413   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1414   int encode = vex_prefix_0F38_and_encode_legacy(rdx, dst, src);

1415   emit_int8((unsigned char)0xF3);
1416   emit_int8((unsigned char)(0xC0 | encode));
1417 }
1418 
1419 void Assembler::blsmskl(Register dst, Address src) {
1420   InstructionMark im(this);
1421   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1422   vex_prefix_0F38_legacy(rdx, dst, src);


1423   emit_int8((unsigned char)0xF3);
1424   emit_operand(rdx, src);
1425 }
1426 
1427 void Assembler::blsrl(Register dst, Register src) {
1428   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1429   int encode = vex_prefix_0F38_and_encode_legacy(rcx, dst, src);

1430   emit_int8((unsigned char)0xF3);
1431   emit_int8((unsigned char)(0xC0 | encode));
1432 }
1433 
1434 void Assembler::blsrl(Register dst, Address src) {
1435   InstructionMark im(this);
1436   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1437   vex_prefix_0F38_legacy(rcx, dst, src);


1438   emit_int8((unsigned char)0xF3);
1439   emit_operand(rcx, src);
1440 }
1441 
1442 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1443   // suspect disp32 is always good
1444   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1445 
1446   if (L.is_bound()) {
1447     const int long_size = 5;
1448     int offs = (int)( target(L) - pc() );
1449     assert(offs <= 0, "assembler error");
1450     InstructionMark im(this);
1451     // 1110 1000 #32-bit disp
1452     emit_int8((unsigned char)0xE8);
1453     emit_data(offs - long_size, rtype, operand);
1454   } else {
1455     InstructionMark im(this);
1456     // 1110 1000 #32-bit disp
1457     L.add_patch_at(code(), locator());


1564   emit_int8(0x0F);
1565   emit_int8((unsigned char)0xB1);
1566   emit_operand(reg, adr);
1567 }
1568 
1569 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1570 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1571 // The ZF is set if the compared values were equal, and cleared otherwise.
1572 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1573   InstructionMark im(this);
1574   prefix(adr, reg, true);
1575   emit_int8(0x0F);
1576   emit_int8((unsigned char)0xB0);
1577   emit_operand(reg, adr);
1578 }
1579 
1580 void Assembler::comisd(XMMRegister dst, Address src) {
1581   // NOTE: dbx seems to decode this as comiss even though the
1582   // 0x66 is there. Strangly ucomisd comes out correct
1583   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1584   if (VM_Version::supports_evex()) {
1585     _tuple_type = EVEX_T1S;
1586     _input_size_in_bits = EVEX_64bit;
1587     emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
1588   } else {
1589     emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1590   }
1591 }
1592 
1593 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1594   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1595   if (VM_Version::supports_evex()) {
1596     emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
1597   } else {
1598     emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1599   }
1600 }
1601 
1602 void Assembler::comiss(XMMRegister dst, Address src) {
1603   if (VM_Version::supports_evex()) {
1604     _tuple_type = EVEX_T1S;
1605     _input_size_in_bits = EVEX_32bit;
1606   }
1607   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1608   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);





1609 }
1610 
1611 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1612   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1613   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);



1614 }
1615 
1616 void Assembler::cpuid() {
1617   emit_int8(0x0F);
1618   emit_int8((unsigned char)0xA2);
1619 }
1620 
1621 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1622 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1623 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1624 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1625 //
1626 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1627 //
1628 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1629 //
1630 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1631 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1632   assert(VM_Version::supports_sse4_2(), "");
1633   int8_t w = 0x01;


1682   case 2:
1683   case 4:
1684     break;
1685   LP64_ONLY(case 8:)
1686     // This instruction is not valid in 32 bits
1687     p = REX_W;
1688     break;
1689   default:
1690     assert(0, "Unsupported value for a sizeInBytes argument");
1691     break;
1692   }
1693   LP64_ONLY(prefix(crc, adr, p);)
1694   emit_int8((int8_t)0x0F);
1695   emit_int8(0x38);
1696   emit_int8((int8_t)(0xF0 | w));
1697   emit_operand(crc, adr);
1698 }
1699 
1700 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1701   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1702   emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3, /* no_mask_reg */ false, /* legacy_mode */ true);



1703 }
1704 
1705 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1706   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1707   emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ true);



1708 }
1709 
1710 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1711   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1712   if (VM_Version::supports_evex()) {
1713     emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
1714   } else {
1715     emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1716   }
1717 }
1718 
1719 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1720   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1721   if (VM_Version::supports_evex()) {
1722     _tuple_type = EVEX_T1F;
1723     _input_size_in_bits = EVEX_64bit;
1724     emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
1725   } else {
1726     emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1727   }
1728 }
1729 
1730 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1731   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1732   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VM_Version::supports_evex());

1733   emit_int8(0x2A);
1734   emit_int8((unsigned char)(0xC0 | encode));
1735 }
1736 
1737 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1738   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1739   if (VM_Version::supports_evex()) {
1740     _tuple_type = EVEX_T1S;
1741     _input_size_in_bits = EVEX_32bit;
1742     emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
1743   } else {
1744     emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
1745   }
1746 }
1747 
1748 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1749   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1750   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);

1751   emit_int8(0x2A);
1752   emit_int8((unsigned char)(0xC0 | encode));
1753 }
1754 
1755 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1756   if (VM_Version::supports_evex()) {
1757     _tuple_type = EVEX_T1S;
1758     _input_size_in_bits = EVEX_32bit;
1759   }
1760   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1761   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);





1762 }
1763 
1764 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1765   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1766   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);

1767   emit_int8(0x2A);
1768   emit_int8((unsigned char)(0xC0 | encode));
1769 }
1770 
1771 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1772   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1773   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);



1774 }
1775 
1776 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1777   if (VM_Version::supports_evex()) {
1778     _tuple_type = EVEX_T1S;
1779     _input_size_in_bits = EVEX_32bit;
1780   }
1781   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1782   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);





1783 }
1784 
1785 
1786 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1787   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1788   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);

1789   emit_int8(0x2C);
1790   emit_int8((unsigned char)(0xC0 | encode));
1791 }
1792 
1793 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1794   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1795   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);

1796   emit_int8(0x2C);
1797   emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799 
1800 void Assembler::decl(Address dst) {
1801   // Don't use it directly. Use MacroAssembler::decrement() instead.
1802   InstructionMark im(this);
1803   prefix(dst);
1804   emit_int8((unsigned char)0xFF);
1805   emit_operand(rcx, dst);
1806 }
1807 
1808 void Assembler::divsd(XMMRegister dst, Address src) {
1809   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1810   if (VM_Version::supports_evex()) {
1811     _tuple_type = EVEX_T1S;
1812     _input_size_in_bits = EVEX_64bit;
1813     emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
1814   } else {
1815     emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1816   }
1817 }
1818 
1819 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1820   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1821   if (VM_Version::supports_evex()) {
1822     emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
1823   } else {
1824     emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1825   }
1826 }
1827 
1828 void Assembler::divss(XMMRegister dst, Address src) {
1829   if (VM_Version::supports_evex()) {
1830     _tuple_type = EVEX_T1S;
1831     _input_size_in_bits = EVEX_32bit;
1832   }
1833   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1834   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);





1835 }
1836 
1837 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1838   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1839   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);



1840 }
1841 
1842 void Assembler::emms() {
1843   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1844   emit_int8(0x0F);
1845   emit_int8(0x77);
1846 }
1847 
1848 void Assembler::hlt() {
1849   emit_int8((unsigned char)0xF4);
1850 }
1851 
1852 void Assembler::idivl(Register src) {
1853   int encode = prefix_and_encode(src->encoding());
1854   emit_int8((unsigned char)0xF7);
1855   emit_int8((unsigned char)(0xF8 | encode));
1856 }
1857 
1858 void Assembler::divl(Register src) { // Unsigned
1859   int encode = prefix_and_encode(src->encoding());


2065   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2066   emit_int8(0x0F);
2067   emit_int8((unsigned char)0xBD);
2068   emit_int8((unsigned char)(0xC0 | encode));
2069 }
2070 
2071 // Emit mfence instruction
2072 void Assembler::mfence() {
2073   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2074   emit_int8(0x0F);
2075   emit_int8((unsigned char)0xAE);
2076   emit_int8((unsigned char)0xF0);
2077 }
2078 
2079 void Assembler::mov(Register dst, Register src) {
2080   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2081 }
2082 
2083 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2084   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2085   if (VM_Version::supports_avx512novl()) {
2086     int vector_len = AVX_512bit;
2087     int dst_enc = dst->encoding();
2088     int src_enc = src->encoding();
2089     int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F,
2090                                        /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2091     emit_int8(0x28);
2092     emit_int8((unsigned char)(0xC0 | encode));
2093   } else if (VM_Version::supports_evex()) {
2094     emit_simd_arith_nonds_q(0x28, dst, src, VEX_SIMD_66);
2095   } else {
2096     emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
2097   }
2098 }
2099 
2100 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2101   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2102   if (VM_Version::supports_avx512novl()) {
2103     int vector_len = AVX_512bit;
2104     int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, vector_len);
2105     emit_int8(0x28);
2106     emit_int8((unsigned char)(0xC0 | encode));
2107   } else {
2108     emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
2109   }
2110 }
2111 
2112 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2113   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2114   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, /* no_mask_reg */ true);

2115   emit_int8(0x16);
2116   emit_int8((unsigned char)(0xC0 | encode));
2117 }
2118 
2119 void Assembler::movb(Register dst, Address src) {
2120   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2121   InstructionMark im(this);
2122   prefix(src, dst, true);
2123   emit_int8((unsigned char)0x8A);
2124   emit_operand(dst, src);
2125 }
2126 
2127 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2128   _instruction_uses_vl = true;
2129   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2130   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, /* no_mask_reg */ false, VEX_OPCODE_0F,
2131                                       /* rex_w */ VM_Version::supports_evex(), AVX_128bit, /* legacy_mode */ false);

2132   emit_int8(0x12);
2133   emit_int8(0xC0 | encode);
2134 
2135 }
2136 
2137 void Assembler::kmovql(KRegister dst, KRegister src) {
2138   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2139   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE,
2140                                       /* no_mask_reg */ true, VEX_OPCODE_0F, /* rex_w */ true);
2141   emit_int8((unsigned char)0x90);
2142   emit_int8((unsigned char)(0xC0 | encode));
2143 }
2144 
2145 void Assembler::kmovql(KRegister dst, Address src) {
2146   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2147   int dst_enc = dst->encoding();
2148   int nds_enc = 0;
2149   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_NONE,
2150              VEX_OPCODE_0F, /* vex_w */  true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
2151   emit_int8((unsigned char)0x90);
2152   emit_operand((Register)dst, src);
2153 }
2154 
2155 void Assembler::kmovql(Address dst, KRegister src) {
2156   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2157   int src_enc = src->encoding();
2158   int nds_enc = 0;
2159   vex_prefix(dst, nds_enc, src_enc, VEX_SIMD_NONE,
2160              VEX_OPCODE_0F, /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
2161   emit_int8((unsigned char)0x90);
2162   emit_operand((Register)src, dst);
2163 }
2164 
2165 void Assembler::kmovql(KRegister dst, Register src) {
2166   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2167   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2168   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true,
2169                                       VEX_OPCODE_0F, /* legacy_mode */ !_legacy_mode_bw);
2170   emit_int8((unsigned char)0x92);
2171   emit_int8((unsigned char)(0xC0 | encode));
2172 }
2173 
2174 void Assembler::kmovdl(KRegister dst, Register src) {
2175   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2176   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2177   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true);

2178   emit_int8((unsigned char)0x92);
2179   emit_int8((unsigned char)(0xC0 | encode));
2180 }
2181 
2182 void Assembler::kmovwl(KRegister dst, Register src) {
2183   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2184   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, /* no_mask_reg */ true);

2185   emit_int8((unsigned char)0x92);
2186   emit_int8((unsigned char)(0xC0 | encode));
2187 }
2188 
2189 void Assembler::movb(Address dst, int imm8) {
2190   InstructionMark im(this);
2191    prefix(dst);
2192   emit_int8((unsigned char)0xC6);
2193   emit_operand(rax, dst, 1);
2194   emit_int8(imm8);
2195 }
2196 
2197 
2198 void Assembler::movb(Address dst, Register src) {
2199   assert(src->has_byte_register(), "must have byte register");
2200   InstructionMark im(this);
2201   prefix(dst, src, true);
2202   emit_int8((unsigned char)0x88);
2203   emit_operand(src, dst);
2204 }
2205 
2206 void Assembler::movdl(XMMRegister dst, Register src) {
2207   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2208   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);

2209   emit_int8(0x6E);
2210   emit_int8((unsigned char)(0xC0 | encode));
2211 }
2212 
2213 void Assembler::movdl(Register dst, XMMRegister src) {
2214   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

2215   // swap src/dst to get correct prefix
2216   int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
2217   emit_int8(0x7E);
2218   emit_int8((unsigned char)(0xC0 | encode));
2219 }
2220 
2221 void Assembler::movdl(XMMRegister dst, Address src) {
2222   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2223   if (VM_Version::supports_evex()) {
2224     _tuple_type = EVEX_T1S;
2225     _input_size_in_bits = EVEX_32bit;
2226   }
2227   InstructionMark im(this);
2228   simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);


2229   emit_int8(0x6E);
2230   emit_operand(dst, src);
2231 }
2232 
2233 void Assembler::movdl(Address dst, XMMRegister src) {
2234   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2235   if (VM_Version::supports_evex()) {
2236     _tuple_type = EVEX_T1S;
2237     _input_size_in_bits = EVEX_32bit;
2238   }
2239   InstructionMark im(this);
2240   simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);


2241   emit_int8(0x7E);
2242   emit_operand(src, dst);
2243 }
2244 
2245 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2246   _instruction_uses_vl = true;
2247   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2248   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);




2249 }
2250 
2251 void Assembler::movdqa(XMMRegister dst, Address src) {
2252   _instruction_uses_vl = true;
2253   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2254   if (VM_Version::supports_evex()) {
2255     _tuple_type = EVEX_FVM;
2256   }
2257   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);


2258 }
2259 
2260 void Assembler::movdqu(XMMRegister dst, Address src) {
2261   _instruction_uses_vl = true;
2262   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2263   if (VM_Version::supports_evex()) {
2264     _tuple_type = EVEX_FVM;
2265   }
2266   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);


2267 }
2268 
2269 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2270   _instruction_uses_vl = true;
2271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2272   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);



2273 }
2274 
2275 void Assembler::movdqu(Address dst, XMMRegister src) {
2276   _instruction_uses_vl = true;
2277   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2278   if (VM_Version::supports_evex()) {
2279     _tuple_type = EVEX_FVM;
2280   }
2281   InstructionMark im(this);
2282   simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);


2283   emit_int8(0x7F);
2284   emit_operand(src, dst);
2285 }
2286 
2287 // Move Unaligned 256bit Vector
2288 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2289   _instruction_uses_vl = true;
2290   assert(UseAVX > 0, "");
2291   int vector_len = AVX_256bit;
2292   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2293   emit_int8(0x6F);
2294   emit_int8((unsigned char)(0xC0 | encode));
2295 }
2296 
2297 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2298   _instruction_uses_vl = true;
2299   assert(UseAVX > 0, "");
2300   if (VM_Version::supports_evex()) {
2301     _tuple_type = EVEX_FVM;
2302   }
2303   InstructionMark im(this);
2304   int vector_len = AVX_256bit;
2305   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);

2306   emit_int8(0x6F);
2307   emit_operand(dst, src);
2308 }
2309 
2310 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2311   _instruction_uses_vl = true;
2312   assert(UseAVX > 0, "");
2313   if (VM_Version::supports_evex()) {
2314     _tuple_type = EVEX_FVM;
2315   }
2316   InstructionMark im(this);
2317   int vector_len = AVX_256bit;

2318   // swap src<->dst for encoding
2319   assert(src != xnoreg, "sanity");
2320   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2321   emit_int8(0x7F);
2322   emit_operand(src, dst);
2323 }
2324 
2325 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2326 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2327   _instruction_uses_vl = true;
2328   assert(UseAVX > 0, "");
2329   int src_enc = src->encoding();
2330   int dst_enc = dst->encoding();
2331   int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
2332                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2333   emit_int8(0x6F);
2334   emit_int8((unsigned char)(0xC0 | encode));
2335 }
2336 
2337 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2338   _instruction_uses_vl = true;
2339   assert(UseAVX > 0, "");
2340   InstructionMark im(this);
2341   if (VM_Version::supports_evex()) {
2342     _tuple_type = EVEX_FVM;
2343   }
2344   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2345   emit_int8(0x6F);
2346   emit_operand(dst, src);
2347 }
2348 
2349 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2350   _instruction_uses_vl = true;
2351   assert(UseAVX > 0, "");
2352   InstructionMark im(this);
2353   assert(src != xnoreg, "sanity");
2354   if (VM_Version::supports_evex()) {
2355     _tuple_type = EVEX_FVM;
2356   }
2357   // swap src<->dst for encoding
2358   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2359   emit_int8(0x7F);
2360   emit_operand(src, dst);
2361 }
2362 
2363 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2364   _instruction_uses_vl = true;
2365   assert(UseAVX > 0, "");
2366   int src_enc = src->encoding();
2367   int dst_enc = dst->encoding();
2368   int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
2369                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2370   emit_int8(0x6F);
2371   emit_int8((unsigned char)(0xC0 | encode));
2372 }
2373 
2374 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2375   _instruction_uses_vl = true;
2376   assert(UseAVX > 2, "");
2377   InstructionMark im(this);
2378   _tuple_type = EVEX_FVM;
2379   vex_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, vector_len);

2380   emit_int8(0x6F);
2381   emit_operand(dst, src);
2382 }
2383 
2384 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2385   _instruction_uses_vl = true;
2386   assert(UseAVX > 2, "");
2387   InstructionMark im(this);
2388   assert(src != xnoreg, "sanity");
2389   _tuple_type = EVEX_FVM;
2390   // swap src<->dst for encoding
2391   vex_prefix_q(src, xnoreg, dst, VEX_SIMD_F3, vector_len);

2392   emit_int8(0x7F);
2393   emit_operand(src, dst);
2394 }
2395 
2396 // Uses zero extension on 64bit
2397 
2398 void Assembler::movl(Register dst, int32_t imm32) {
2399   int encode = prefix_and_encode(dst->encoding());
2400   emit_int8((unsigned char)(0xB8 | encode));
2401   emit_int32(imm32);
2402 }
2403 
2404 void Assembler::movl(Register dst, Register src) {
2405   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2406   emit_int8((unsigned char)0x8B);
2407   emit_int8((unsigned char)(0xC0 | encode));
2408 }
2409 
2410 void Assembler::movl(Register dst, Address src) {
2411   InstructionMark im(this);


2417 void Assembler::movl(Address dst, int32_t imm32) {
2418   InstructionMark im(this);
2419   prefix(dst);
2420   emit_int8((unsigned char)0xC7);
2421   emit_operand(rax, dst, 4);
2422   emit_int32(imm32);
2423 }
2424 
2425 void Assembler::movl(Address dst, Register src) {
2426   InstructionMark im(this);
2427   prefix(dst, src);
2428   emit_int8((unsigned char)0x89);
2429   emit_operand(src, dst);
2430 }
2431 
2432 // New cpus require to use movsd and movss to avoid partial register stall
2433 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2434 // The selection is done in MacroAssembler::movdbl() and movflt().
2435 void Assembler::movlpd(XMMRegister dst, Address src) {
2436   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2437   if (VM_Version::supports_evex()) {
2438     _tuple_type = EVEX_T1S;
2439     _input_size_in_bits = EVEX_32bit;
2440     emit_simd_arith_q(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2441   } else {
2442     emit_simd_arith(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2443   }
2444 }
2445 
2446 void Assembler::movq( MMXRegister dst, Address src ) {
2447   assert( VM_Version::supports_mmx(), "" );
2448   emit_int8(0x0F);
2449   emit_int8(0x6F);
2450   emit_operand(dst, src);
2451 }
2452 
2453 void Assembler::movq( Address dst, MMXRegister src ) {
2454   assert( VM_Version::supports_mmx(), "" );
2455   emit_int8(0x0F);
2456   emit_int8(0x7F);
2457   // workaround gcc (3.2.1-7a) bug
2458   // In that version of gcc with only an emit_operand(MMX, Address)
2459   // gcc will tail jump and try and reverse the parameters completely
2460   // obliterating dst in the process. By having a version available
2461   // that doesn't need to swap the args at the tail jump the bug is
2462   // avoided.
2463   emit_operand(dst, src);
2464 }
2465 
2466 void Assembler::movq(XMMRegister dst, Address src) {
2467   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2468   InstructionMark im(this);
2469   if (VM_Version::supports_evex()) {
2470     _tuple_type = EVEX_T1S;
2471     _input_size_in_bits = EVEX_64bit;
2472     simd_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2473   } else {
2474     simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2475   }
2476   emit_int8(0x7E);
2477   emit_operand(dst, src);
2478 }
2479 
2480 void Assembler::movq(Address dst, XMMRegister src) {
2481   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2482   InstructionMark im(this);
2483   if (VM_Version::supports_evex()) {
2484     _tuple_type = EVEX_T1S;
2485     _input_size_in_bits = EVEX_64bit;
2486     simd_prefix(src, xnoreg, dst, VEX_SIMD_66, /* no_mask_reg */ true,
2487                 VEX_OPCODE_0F, /* rex_w */ true);
2488   } else {
2489     simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2490   }
2491   emit_int8((unsigned char)0xD6);
2492   emit_operand(src, dst);
2493 }
2494 
2495 void Assembler::movsbl(Register dst, Address src) { // movsxb
2496   InstructionMark im(this);
2497   prefix(src, dst);
2498   emit_int8(0x0F);
2499   emit_int8((unsigned char)0xBE);
2500   emit_operand(dst, src);
2501 }
2502 
2503 void Assembler::movsbl(Register dst, Register src) { // movsxb
2504   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2505   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2506   emit_int8(0x0F);
2507   emit_int8((unsigned char)0xBE);
2508   emit_int8((unsigned char)(0xC0 | encode));
2509 }
2510 
2511 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2512   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2513   if (VM_Version::supports_evex()) {
2514     emit_simd_arith_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
2515   } else {
2516     emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
2517   }
2518 }
2519 
2520 void Assembler::movsd(XMMRegister dst, Address src) {
2521   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2522   if (VM_Version::supports_evex()) {
2523     _tuple_type = EVEX_T1S;
2524     _input_size_in_bits = EVEX_64bit;
2525     emit_simd_arith_nonds_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
2526   } else {
2527     emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
2528   }
2529 }
2530 
2531 void Assembler::movsd(Address dst, XMMRegister src) {
2532   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2533   InstructionMark im(this);
2534   if (VM_Version::supports_evex()) {
2535     _tuple_type = EVEX_T1S;
2536     _input_size_in_bits = EVEX_64bit;
2537     simd_prefix_q(src, xnoreg, dst, VEX_SIMD_F2);
2538   } else {
2539     simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, /* no_mask_reg */ false);
2540   }
2541   emit_int8(0x11);
2542   emit_operand(src, dst);
2543 }
2544 
2545 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2546   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2547   emit_simd_arith(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);



2548 }
2549 
2550 void Assembler::movss(XMMRegister dst, Address src) {
2551   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2552   if (VM_Version::supports_evex()) {
2553     _tuple_type = EVEX_T1S;
2554     _input_size_in_bits = EVEX_32bit;
2555   }
2556   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);

2557 }
2558 
2559 void Assembler::movss(Address dst, XMMRegister src) {
2560   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2561   if (VM_Version::supports_evex()) {
2562     _tuple_type = EVEX_T1S;
2563     _input_size_in_bits = EVEX_32bit;
2564   }
2565   InstructionMark im(this);
2566   simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);


2567   emit_int8(0x11);
2568   emit_operand(src, dst);
2569 }
2570 
2571 void Assembler::movswl(Register dst, Address src) { // movsxw
2572   InstructionMark im(this);
2573   prefix(src, dst);
2574   emit_int8(0x0F);
2575   emit_int8((unsigned char)0xBF);
2576   emit_operand(dst, src);
2577 }
2578 
2579 void Assembler::movswl(Register dst, Register src) { // movsxw
2580   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2581   emit_int8(0x0F);
2582   emit_int8((unsigned char)0xBF);
2583   emit_int8((unsigned char)(0xC0 | encode));
2584 }
2585 
2586 void Assembler::movw(Address dst, int imm16) {


2638   emit_int8(0x0F);
2639   emit_int8((unsigned char)0xB7);
2640   emit_int8(0xC0 | encode);
2641 }
2642 
2643 void Assembler::mull(Address src) {
2644   InstructionMark im(this);
2645   prefix(src);
2646   emit_int8((unsigned char)0xF7);
2647   emit_operand(rsp, src);
2648 }
2649 
2650 void Assembler::mull(Register src) {
2651   int encode = prefix_and_encode(src->encoding());
2652   emit_int8((unsigned char)0xF7);
2653   emit_int8((unsigned char)(0xE0 | encode));
2654 }
2655 
2656 void Assembler::mulsd(XMMRegister dst, Address src) {
2657   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2658   if (VM_Version::supports_evex()) {
2659     _tuple_type = EVEX_T1S;
2660     _input_size_in_bits = EVEX_64bit;
2661     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
2662   } else {
2663     emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2664   }
2665 }
2666 
2667 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2668   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2669   if (VM_Version::supports_evex()) {
2670     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
2671   } else {
2672     emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2673   }
2674 }
2675 
2676 void Assembler::mulss(XMMRegister dst, Address src) {
2677   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2678   if (VM_Version::supports_evex()) {
2679     _tuple_type = EVEX_T1S;
2680     _input_size_in_bits = EVEX_32bit;
2681   }
2682   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);

2683 }
2684 
2685 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2686   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2687   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);



2688 }
2689 
2690 void Assembler::negl(Register dst) {
2691   int encode = prefix_and_encode(dst->encoding());
2692   emit_int8((unsigned char)0xF7);
2693   emit_int8((unsigned char)(0xD8 | encode));
2694 }
2695 
2696 void Assembler::nop(int i) {
2697 #ifdef ASSERT
2698   assert(i > 0, " ");
2699   // The fancy nops aren't currently recognized by debuggers making it a
2700   // pain to disassemble code while debugging. If asserts are on clearly
2701   // speed is not an issue so simply use the single byte traditional nop
2702   // to do alignment.
2703 
2704   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2705   return;
2706 
2707 #endif // ASSERT


2968   prefix(src, dst);
2969   emit_int8(0x0B);
2970   emit_operand(dst, src);
2971 }
2972 
2973 void Assembler::orl(Register dst, Register src) {
2974   (void) prefix_and_encode(dst->encoding(), src->encoding());
2975   emit_arith(0x0B, 0xC0, dst, src);
2976 }
2977 
2978 void Assembler::orl(Address dst, Register src) {
2979   InstructionMark im(this);
2980   prefix(dst, src);
2981   emit_int8(0x09);
2982   emit_operand(src, dst);
2983 }
2984 
2985 void Assembler::packuswb(XMMRegister dst, Address src) {
2986   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2987   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2988   if (VM_Version::supports_evex()) {
2989     _tuple_type = EVEX_FV;
2990     _input_size_in_bits = EVEX_32bit;
2991   }
2992   emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);

2993 }
2994 
2995 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2996   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2997   emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



2998 }
2999 
3000 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3001   assert(UseAVX > 0, "some form of AVX must be enabled");
3002   emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);




3003 }
3004 
3005 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3006   _instruction_uses_vl = true;
3007   assert(VM_Version::supports_avx2(), "");
3008   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3009                                       VEX_OPCODE_0F_3A, /* rex_w */ true, vector_len);
3010   emit_int8(0x00);
3011   emit_int8(0xC0 | encode);
3012   emit_int8(imm8);
3013 }
3014 
3015 void Assembler::pause() {
3016   emit_int8((unsigned char)0xF3);
3017   emit_int8((unsigned char)0x90);
3018 }
3019 
3020 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3021   assert(VM_Version::supports_sse4_2(), "");
3022   InstructionMark im(this);
3023   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_3A,
3024               /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3025   emit_int8(0x61);
3026   emit_operand(dst, src);
3027   emit_int8(imm8);
3028 }
3029 
3030 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3031   assert(VM_Version::supports_sse4_2(), "");
3032   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3033                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3034   emit_int8(0x61);
3035   emit_int8((unsigned char)(0xC0 | encode));
3036   emit_int8(imm8);
3037 }
3038 
3039 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3040   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3041   emit_simd_arith(0x75, dst, src, VEX_SIMD_66,
3042                   false, (VM_Version::supports_avx512dq() == false));


3043 }
3044 
3045 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3046   assert(UseAVX > 0, "some form of AVX must be enabled");
3047   emit_vex_arith(0x75, dst, nds, src, VEX_SIMD_66, vector_len,
3048                  false, (VM_Version::supports_avx512dq() == false));




3049 }
3050 
3051 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3052   assert(VM_Version::supports_sse2(), "");
3053   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, true, VEX_OPCODE_0F,
3054                                       false, AVX_128bit, (VM_Version::supports_avx512dq() == false));
3055   emit_int8((unsigned char)0xD7);
3056   emit_int8((unsigned char)(0xC0 | encode));
3057 }
3058 
3059 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3060   assert(VM_Version::supports_avx2(), "");
3061   int vector_len = AVX_256bit;
3062   int encode = vex_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66,
3063                                      vector_len, VEX_OPCODE_0F, true, false);
3064   emit_int8((unsigned char)0xD7);
3065   emit_int8((unsigned char)(0xC0 | encode));
3066 }
3067 
3068 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3069   assert(VM_Version::supports_sse4_1(), "");
3070   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3071                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3072   emit_int8(0x16);
3073   emit_int8((unsigned char)(0xC0 | encode));
3074   emit_int8(imm8);
3075 }
3076 
3077 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3078   assert(VM_Version::supports_sse4_1(), "");
3079   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */  true,
3080                                       VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3081   emit_int8(0x16);
3082   emit_int8((unsigned char)(0xC0 | encode));
3083   emit_int8(imm8);
3084 }
3085 
3086 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3087   assert(VM_Version::supports_sse2(), "");
3088   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3089                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3090   emit_int8((unsigned char)0xC5);
3091   emit_int8((unsigned char)(0xC0 | encode));
3092   emit_int8(imm8);
3093 }
3094 
3095 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3096   assert(VM_Version::supports_sse4_1(), "");
3097   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3098                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3099   emit_int8(0x22);
3100   emit_int8((unsigned char)(0xC0 | encode));
3101   emit_int8(imm8);
3102 }
3103 
3104 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3105   assert(VM_Version::supports_sse4_1(), "");
3106   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3107                                       VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3108   emit_int8(0x22);
3109   emit_int8((unsigned char)(0xC0 | encode));
3110   emit_int8(imm8);
3111 }
3112 
3113 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3114   assert(VM_Version::supports_sse2(), "");
3115   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3116                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3117   emit_int8((unsigned char)0xC4);
3118   emit_int8((unsigned char)(0xC0 | encode));
3119   emit_int8(imm8);
3120 }
3121 
3122 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3123   assert(VM_Version::supports_sse4_1(), "");
3124   if (VM_Version::supports_evex()) {
3125     _tuple_type = EVEX_HVM;
3126   }
3127   InstructionMark im(this);
3128   simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);


3129   emit_int8(0x30);
3130   emit_operand(dst, src);
3131 }
3132 
3133 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3134   assert(VM_Version::supports_sse4_1(), "");
3135   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);

3136   emit_int8(0x30);
3137   emit_int8((unsigned char)(0xC0 | encode));
3138 }
3139 
3140 void Assembler::vpmovzxbw(XMMRegister dst, Address src) {
3141   assert(VM_Version::supports_avx(), "");
3142   InstructionMark im(this);
3143   bool vector256 = true;
3144   assert(dst != xnoreg, "sanity");
3145   int dst_enc = dst->encoding();
3146   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);

3147   emit_int8(0x30);
3148   emit_operand(dst, src);
3149 }
3150 
3151 // generic
3152 void Assembler::pop(Register dst) {
3153   int encode = prefix_and_encode(dst->encoding());
3154   emit_int8(0x58 | encode);
3155 }
3156 
3157 void Assembler::popcntl(Register dst, Address src) {
3158   assert(VM_Version::supports_popcnt(), "must support");
3159   InstructionMark im(this);
3160   emit_int8((unsigned char)0xF3);
3161   prefix(src, dst);
3162   emit_int8(0x0F);
3163   emit_int8((unsigned char)0xB8);
3164   emit_operand(dst, src);
3165 }
3166 


3229   InstructionMark im(this);
3230   prefetch_prefix(src);
3231   emit_int8(0x18);
3232   emit_operand(rbx, src); // 3, src
3233 }
3234 
3235 void Assembler::prefetchw(Address src) {
3236   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3237   InstructionMark im(this);
3238   prefetch_prefix(src);
3239   emit_int8(0x0D);
3240   emit_operand(rcx, src); // 1, src
3241 }
3242 
3243 void Assembler::prefix(Prefix p) {
3244   emit_int8(p);
3245 }
3246 
3247 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3248   assert(VM_Version::supports_ssse3(), "");
3249   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
3250                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3251   emit_int8(0x00);
3252   emit_int8((unsigned char)(0xC0 | encode));
3253 }
3254 
3255 void Assembler::pshufb(XMMRegister dst, Address src) {
3256   assert(VM_Version::supports_ssse3(), "");
3257   if (VM_Version::supports_evex()) {
3258     _tuple_type = EVEX_FVM;
3259   }
3260   InstructionMark im(this);
3261   simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
3262               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);

3263   emit_int8(0x00);
3264   emit_operand(dst, src);
3265 }
3266 
3267 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3268   _instruction_uses_vl = true;
3269   assert(isByte(mode), "invalid value");
3270   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3271   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);




3272   emit_int8(mode & 0xFF);
3273 }
3274 
3275 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3276   _instruction_uses_vl = true;
3277   assert(isByte(mode), "invalid value");
3278   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3279   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3280   if (VM_Version::supports_evex()) {
3281     _tuple_type = EVEX_FV;
3282     _input_size_in_bits = EVEX_32bit;
3283   }
3284   InstructionMark im(this);
3285   simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false);


3286   emit_int8(0x70);
3287   emit_operand(dst, src);
3288   emit_int8(mode & 0xFF);
3289 }
3290 
3291 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3292   assert(isByte(mode), "invalid value");
3293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3294   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



3295   emit_int8(mode & 0xFF);
3296 }
3297 
3298 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3299   assert(isByte(mode), "invalid value");
3300   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3301   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3302   if (VM_Version::supports_evex()) {
3303     _tuple_type = EVEX_FVM;
3304   }
3305   InstructionMark im(this);
3306   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, /* no_mask_reg */ false,
3307               VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);

3308   emit_int8(0x70);
3309   emit_operand(dst, src);
3310   emit_int8(mode & 0xFF);
3311 }
3312 
3313 void Assembler::psrldq(XMMRegister dst, int shift) {
3314   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3315   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

3316   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3317   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
3318                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3319   emit_int8(0x73);
3320   emit_int8((unsigned char)(0xC0 | encode));
3321   emit_int8(shift);
3322 }
3323 
3324 void Assembler::pslldq(XMMRegister dst, int shift) {
3325   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3326   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

3327   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3328   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
3329                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3330   emit_int8(0x73);
3331   emit_int8((unsigned char)(0xC0 | encode));
3332   emit_int8(shift);
3333 }
3334 
3335 void Assembler::ptest(XMMRegister dst, Address src) {
3336   assert(VM_Version::supports_sse4_1(), "");
3337   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3338   InstructionMark im(this);
3339   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3340               VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3341   emit_int8(0x17);
3342   emit_operand(dst, src);
3343 }
3344 
3345 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3346   assert(VM_Version::supports_sse4_1(), "");
3347   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3348                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3349   emit_int8(0x17);
3350   emit_int8((unsigned char)(0xC0 | encode));
3351 }
3352 
3353 void Assembler::vptest(XMMRegister dst, Address src) {
3354   assert(VM_Version::supports_avx(), "");
3355   InstructionMark im(this);
3356   int vector_len = AVX_256bit;
3357   assert(dst != xnoreg, "sanity");
3358   int dst_enc = dst->encoding();
3359   // swap src<->dst for encoding
3360   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* rex_w */ false,
3361              vector_len, /* legacy_mode  */ true, /* no_mask_reg */ false);
3362   emit_int8(0x17);
3363   emit_operand(dst, src);
3364 }
3365 
3366 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3367   assert(VM_Version::supports_avx(), "");
3368   int vector_len = AVX_256bit;
3369   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
3370   emit_int8(0x17);
3371   emit_int8((unsigned char)(0xC0 | encode));
3372 }
3373 
3374 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3375   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3376   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3377   if (VM_Version::supports_evex()) {
3378     _tuple_type = EVEX_FVM;
3379   }
3380   emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);


3381 }
3382 
3383 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3384   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3385   emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);



3386 }
3387 
3388 void Assembler::punpckldq(XMMRegister dst, Address src) {
3389   _instruction_uses_vl = true;
3390   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3391   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3392   if (VM_Version::supports_evex()) {
3393     _tuple_type = EVEX_FV;
3394     _input_size_in_bits = EVEX_32bit;
3395   }
3396   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);

3397 }
3398 
3399 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3400   _instruction_uses_vl = true;
3401   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3402   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);



3403 }
3404 
3405 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3406   _instruction_uses_vl = true;
3407   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3408   if (VM_Version::supports_evex()) {
3409     emit_simd_arith_q(0x6C, dst, src, VEX_SIMD_66);
3410   } else {
3411     emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
3412   }
3413 }
3414 
3415 void Assembler::push(int32_t imm32) {
3416   // in 64bits we push 64bits onto the stack but only
3417   // take a 32bit immediate
3418   emit_int8(0x68);
3419   emit_int32(imm32);
3420 }
3421 
3422 void Assembler::push(Register src) {
3423   int encode = prefix_and_encode(src->encoding());
3424 
3425   emit_int8(0x50 | encode);
3426 }
3427 
3428 void Assembler::pushf() {
3429   emit_int8((unsigned char)0x9C);
3430 }
3431 
3432 #ifndef _LP64 // no 32bit push/pop on amd64


3437   emit_int8((unsigned char)0xFF);
3438   emit_operand(rsi, src);
3439 }
3440 #endif
3441 
3442 void Assembler::rcll(Register dst, int imm8) {
3443   assert(isShiftCount(imm8), "illegal shift count");
3444   int encode = prefix_and_encode(dst->encoding());
3445   if (imm8 == 1) {
3446     emit_int8((unsigned char)0xD1);
3447     emit_int8((unsigned char)(0xD0 | encode));
3448   } else {
3449     emit_int8((unsigned char)0xC1);
3450     emit_int8((unsigned char)0xD0 | encode);
3451     emit_int8(imm8);
3452   }
3453 }
3454 
3455 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3456   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3457   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);

3458   emit_int8(0x53);
3459   emit_int8(0xC0 | encode);
3460 }
3461 
3462 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3463   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3464   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);

3465   emit_int8(0x53);
3466   emit_int8(0xC0 | encode);
3467 }
3468 
3469 void Assembler::rdtsc() {
3470   emit_int8((unsigned char)0x0F);
3471   emit_int8((unsigned char)0x31);
3472 }
3473 
3474 // copies data from [esi] to [edi] using rcx pointer sized words
3475 // generic
3476 void Assembler::rep_mov() {
3477   emit_int8((unsigned char)0xF3);
3478   // MOVSQ
3479   LP64_ONLY(prefix(REX_W));
3480   emit_int8((unsigned char)0xA5);
3481 }
3482 
3483 // sets rcx bytes with rax, value at [edi]
3484 void Assembler::rep_stosb() {
3485   emit_int8((unsigned char)0xF3); // REP
3486   LP64_ONLY(prefix(REX_W));


3605   assert(isShiftCount(imm8), "illegal shift count");
3606   int encode = prefix_and_encode(dst->encoding());
3607   emit_int8((unsigned char)0xC1);
3608   emit_int8((unsigned char)(0xE8 | encode));
3609   emit_int8(imm8);
3610 }
3611 
3612 void Assembler::shrl(Register dst) {
3613   int encode = prefix_and_encode(dst->encoding());
3614   emit_int8((unsigned char)0xD3);
3615   emit_int8((unsigned char)(0xE8 | encode));
3616 }
3617 
3618 // copies a single word from [esi] to [edi]
3619 void Assembler::smovl() {
3620   emit_int8((unsigned char)0xA5);
3621 }
3622 
3623 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3624   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3625   if (VM_Version::supports_evex()) {
3626     emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
3627   } else {
3628     emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
3629   }
3630 }
3631 
3632 void Assembler::sqrtsd(XMMRegister dst, Address src) {
3633   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3634   if (VM_Version::supports_evex()) {
3635     _tuple_type = EVEX_T1S;
3636     _input_size_in_bits = EVEX_64bit;
3637     emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
3638   } else {
3639     emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
3640   }
3641 }
3642 
3643 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3644   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3645   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);



3646 }
3647 
3648 void Assembler::std() {
3649   emit_int8((unsigned char)0xFD);
3650 }
3651 
3652 void Assembler::sqrtss(XMMRegister dst, Address src) {
3653   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3654   if (VM_Version::supports_evex()) {
3655     _tuple_type = EVEX_T1S;
3656     _input_size_in_bits = EVEX_32bit;
3657   }
3658   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);

3659 }
3660 
3661 void Assembler::stmxcsr( Address dst) {
3662   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3663   InstructionMark im(this);
3664   prefix(dst);
3665   emit_int8(0x0F);
3666   emit_int8((unsigned char)0xAE);
3667   emit_operand(as_Register(3), dst);
3668 }
3669 
3670 void Assembler::subl(Address dst, int32_t imm32) {
3671   InstructionMark im(this);
3672   prefix(dst);
3673   emit_arith_operand(0x81, rbp, dst, imm32);
3674 }
3675 
3676 void Assembler::subl(Address dst, Register src) {
3677   InstructionMark im(this);
3678   prefix(dst, src);


3688 // Force generation of a 4 byte immediate value even if it fits into 8bit
3689 void Assembler::subl_imm32(Register dst, int32_t imm32) {
3690   prefix(dst);
3691   emit_arith_imm32(0x81, 0xE8, dst, imm32);
3692 }
3693 
3694 void Assembler::subl(Register dst, Address src) {
3695   InstructionMark im(this);
3696   prefix(src, dst);
3697   emit_int8(0x2B);
3698   emit_operand(dst, src);
3699 }
3700 
3701 void Assembler::subl(Register dst, Register src) {
3702   (void) prefix_and_encode(dst->encoding(), src->encoding());
3703   emit_arith(0x2B, 0xC0, dst, src);
3704 }
3705 
3706 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3707   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3708   if (VM_Version::supports_evex()) {
3709     emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
3710   } else {
3711     emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
3712   }
3713 }
3714 
3715 void Assembler::subsd(XMMRegister dst, Address src) {
3716   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3717   if (VM_Version::supports_evex()) {
3718     _tuple_type = EVEX_T1S;
3719     _input_size_in_bits = EVEX_64bit;
3720   }
3721   if (VM_Version::supports_evex()) {
3722     emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
3723   } else {
3724     emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
3725   }
3726 }
3727 
3728 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3729   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3730   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);



3731 }
3732 
3733 void Assembler::subss(XMMRegister dst, Address src) {
3734   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3735   if (VM_Version::supports_evex()) {
3736     _tuple_type = EVEX_T1S;
3737     _input_size_in_bits = EVEX_32bit;
3738   }
3739   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);

3740 }
3741 
3742 void Assembler::testb(Register dst, int imm8) {
3743   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
3744   (void) prefix_and_encode(dst->encoding(), true);
3745   emit_arith_b(0xF6, 0xC0, dst, imm8);
3746 }
3747 
3748 void Assembler::testl(Register dst, int32_t imm32) {
3749   // not using emit_arith because test
3750   // doesn't support sign-extension of
3751   // 8bit operands
3752   int encode = dst->encoding();
3753   if (encode == 0) {
3754     emit_int8((unsigned char)0xA9);
3755   } else {
3756     encode = prefix_and_encode(encode);
3757     emit_int8((unsigned char)0xF7);
3758     emit_int8((unsigned char)(0xC0 | encode));
3759   }


3775 void Assembler::tzcntl(Register dst, Register src) {
3776   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3777   emit_int8((unsigned char)0xF3);
3778   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3779   emit_int8(0x0F);
3780   emit_int8((unsigned char)0xBC);
3781   emit_int8((unsigned char)0xC0 | encode);
3782 }
3783 
3784 void Assembler::tzcntq(Register dst, Register src) {
3785   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3786   emit_int8((unsigned char)0xF3);
3787   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3788   emit_int8(0x0F);
3789   emit_int8((unsigned char)0xBC);
3790   emit_int8((unsigned char)(0xC0 | encode));
3791 }
3792 
3793 void Assembler::ucomisd(XMMRegister dst, Address src) {
3794   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3795   if (VM_Version::supports_evex()) {
3796     _tuple_type = EVEX_T1S;
3797     _input_size_in_bits = EVEX_64bit;
3798     emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
3799   } else {
3800     emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
3801   }
3802 }
3803 
3804 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3805   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3806   if (VM_Version::supports_evex()) {
3807     emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
3808   } else {
3809     emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
3810   }
3811 }
3812 
3813 void Assembler::ucomiss(XMMRegister dst, Address src) {
3814   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3815   if (VM_Version::supports_evex()) {
3816     _tuple_type = EVEX_T1S;
3817     _input_size_in_bits = EVEX_32bit;
3818   }
3819   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);

3820 }
3821 
3822 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3823   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3824   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);



3825 }
3826 
3827 void Assembler::xabort(int8_t imm8) {
3828   emit_int8((unsigned char)0xC6);
3829   emit_int8((unsigned char)0xF8);
3830   emit_int8((unsigned char)(imm8 & 0xFF));
3831 }
3832 
3833 void Assembler::xaddl(Address dst, Register src) {
3834   InstructionMark im(this);
3835   prefix(dst, src);
3836   emit_int8(0x0F);
3837   emit_int8((unsigned char)0xC1);
3838   emit_operand(src, dst);
3839 }
3840 
3841 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3842   InstructionMark im(this);
3843   relocate(rtype);
3844   if (abort.is_bound()) {


3886   emit_arith(0x81, 0xF0, dst, imm32);
3887 }
3888 
3889 void Assembler::xorl(Register dst, Address src) {
3890   InstructionMark im(this);
3891   prefix(src, dst);
3892   emit_int8(0x33);
3893   emit_operand(dst, src);
3894 }
3895 
3896 void Assembler::xorl(Register dst, Register src) {
3897   (void) prefix_and_encode(dst->encoding(), src->encoding());
3898   emit_arith(0x33, 0xC0, dst, src);
3899 }
3900 
3901 
3902 // AVX 3-operands scalar float-point arithmetic instructions
3903 
3904 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3905   assert(VM_Version::supports_avx(), "");
3906   if (VM_Version::supports_evex()) {
3907     _tuple_type = EVEX_T1S;
3908     _input_size_in_bits = EVEX_64bit;
3909     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3910   } else {
3911     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3912   }
3913 }
3914 
3915 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3916   assert(VM_Version::supports_avx(), "");
3917   if (VM_Version::supports_evex()) {
3918     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3919   } else {
3920     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3921   }
3922 }
3923 
3924 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3925   assert(VM_Version::supports_avx(), "");
3926   if (VM_Version::supports_evex()) {
3927     _tuple_type = EVEX_T1S;
3928     _input_size_in_bits = EVEX_32bit;
3929   }
3930   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);


3931 }
3932 
3933 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3934   assert(VM_Version::supports_avx(), "");
3935   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);




3936 }
3937 
3938 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3939   assert(VM_Version::supports_avx(), "");
3940   if (VM_Version::supports_evex()) {
3941     _tuple_type = EVEX_T1S;
3942     _input_size_in_bits = EVEX_64bit;
3943     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3944   } else {
3945     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3946   }
3947 }
3948 
3949 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3950   assert(VM_Version::supports_avx(), "");
3951   if (VM_Version::supports_evex()) {
3952     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3953   } else {
3954     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3955   }
3956 }
3957 
3958 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3959   assert(VM_Version::supports_avx(), "");
3960   if (VM_Version::supports_evex()) {
3961     _tuple_type = EVEX_T1S;
3962     _input_size_in_bits = EVEX_32bit;
3963   }
3964   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);


3965 }
3966 
3967 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3968   assert(VM_Version::supports_avx(), "");
3969   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);




3970 }
3971 
3972 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3973   assert(VM_Version::supports_avx(), "");
3974   if (VM_Version::supports_evex()) {
3975     _tuple_type = EVEX_T1S;
3976     _input_size_in_bits = EVEX_64bit;
3977     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3978   } else {
3979     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3980   }
3981 }
3982 
3983 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3984   assert(VM_Version::supports_avx(), "");
3985   if (VM_Version::supports_evex()) {
3986     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3987   } else {
3988     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3989   }
3990 }
3991 
3992 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
3993   assert(VM_Version::supports_avx(), "");
3994   if (VM_Version::supports_evex()) {
3995     _tuple_type = EVEX_T1S;
3996     _input_size_in_bits = EVEX_32bit;
3997   }
3998   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);


3999 }
4000 
4001 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4002   assert(VM_Version::supports_avx(), "");
4003   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);




4004 }
4005 
4006 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4007   assert(VM_Version::supports_avx(), "");
4008   if (VM_Version::supports_evex()) {
4009     _tuple_type = EVEX_T1S;
4010     _input_size_in_bits = EVEX_64bit;
4011     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
4012   } else {
4013     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
4014   }
4015 }
4016 
4017 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4018   assert(VM_Version::supports_avx(), "");
4019   if (VM_Version::supports_evex()) {
4020     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
4021   } else {
4022     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
4023   }
4024 }
4025 
4026 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4027   assert(VM_Version::supports_avx(), "");
4028   if (VM_Version::supports_evex()) {
4029     _tuple_type = EVEX_T1S;
4030     _input_size_in_bits = EVEX_32bit;
4031   }
4032   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);


4033 }
4034 
4035 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4036   assert(VM_Version::supports_avx(), "");
4037   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);




4038 }
4039 
4040 //====================VECTOR ARITHMETIC=====================================
4041 
4042 // Float-point vector arithmetic
4043 
4044 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4045   _instruction_uses_vl = true;
4046   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4047   if (VM_Version::supports_evex()) {
4048     emit_simd_arith_q(0x58, dst, src, VEX_SIMD_66);
4049   } else {
4050     emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
4051   }
4052 }
4053 
4054 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4055   _instruction_uses_vl = true;
4056   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4057   emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);



4058 }
4059 
4060 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4061   _instruction_uses_vl = true;
4062   assert(VM_Version::supports_avx(), "");
4063   if (VM_Version::supports_evex()) {
4064     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4065   } else {
4066     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4067   }
4068 }
4069 
4070 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4071   _instruction_uses_vl = true;
4072   assert(VM_Version::supports_avx(), "");
4073   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);




4074 }
4075 
4076 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4077   _instruction_uses_vl = true;
4078   assert(VM_Version::supports_avx(), "");
4079   if (VM_Version::supports_evex()) {
4080     _tuple_type = EVEX_FV;
4081     _input_size_in_bits = EVEX_64bit;
4082     emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4083   } else {
4084     emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4085   }
4086 }
4087 
4088 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4089   _instruction_uses_vl = true;
4090   assert(VM_Version::supports_avx(), "");
4091   if (VM_Version::supports_evex()) {
4092     _tuple_type = EVEX_FV;
4093     _input_size_in_bits = EVEX_32bit;
4094   }
4095   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);


4096 }
4097 
4098 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4099   _instruction_uses_vl = true;
4100   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4101   if (VM_Version::supports_evex()) {
4102     emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_66);
4103   } else {
4104     emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
4105   }
4106 }
4107 
4108 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4109   _instruction_uses_vl = true;
4110   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4111   emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);



4112 }
4113 
4114 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4115   _instruction_uses_vl = true;
4116   assert(VM_Version::supports_avx(), "");
4117   if (VM_Version::supports_evex()) {
4118     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4119   } else {
4120     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4121   }
4122 }
4123 
4124 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4125   _instruction_uses_vl = true;
4126   assert(VM_Version::supports_avx(), "");
4127   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);




4128 }
4129 
4130 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4131   _instruction_uses_vl = true;
4132   assert(VM_Version::supports_avx(), "");
4133   if (VM_Version::supports_evex()) {
4134     _tuple_type = EVEX_FV;
4135     _input_size_in_bits = EVEX_64bit;
4136     emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4137   } else {
4138     emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4139   }
4140 }
4141 
4142 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4143   _instruction_uses_vl = true;
4144   assert(VM_Version::supports_avx(), "");
4145   if (VM_Version::supports_evex()) {
4146     _tuple_type = EVEX_FV;
4147     _input_size_in_bits = EVEX_32bit;
4148   }
4149   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);


4150 }
4151 
4152 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4153   _instruction_uses_vl = true;
4154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4155   if (VM_Version::supports_evex()) {
4156     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
4157   } else {
4158     emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
4159   }
4160 }
4161 
4162 void Assembler::mulpd(XMMRegister dst, Address src) {
4163   _instruction_uses_vl = true;
4164   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4165   if (VM_Version::supports_evex()) {
4166     emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
4167   } else {
4168     emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
4169   }

4170 }
4171 
4172 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4173   _instruction_uses_vl = true;
4174   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4175   emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);



4176 }
4177 
4178 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4179   _instruction_uses_vl = true;
4180   assert(VM_Version::supports_avx(), "");
4181   if (VM_Version::supports_evex()) {
4182     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4183   } else {
4184     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4185   }
4186 }
4187 
4188 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4189   _instruction_uses_vl = true;
4190   assert(VM_Version::supports_avx(), "");
4191   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);




4192 }
4193 
4194 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4195   _instruction_uses_vl = true;
4196   assert(VM_Version::supports_avx(), "");
4197   if (VM_Version::supports_evex()) {
4198     _tuple_type = EVEX_FV;
4199     _input_size_in_bits = EVEX_64bit;
4200     emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4201   } else {
4202     emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4203   }
4204 }
4205 
4206 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4207   _instruction_uses_vl = true;
4208   assert(VM_Version::supports_avx(), "");
4209   if (VM_Version::supports_evex()) {
4210     _tuple_type = EVEX_FV;
4211     _input_size_in_bits = EVEX_32bit;
4212   }
4213   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);


4214 }
4215 
4216 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4217   _instruction_uses_vl = true;
4218   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4219   if (VM_Version::supports_evex()) {
4220     emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_66);
4221   } else {
4222     emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
4223   }
4224 }
4225 
4226 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4227   _instruction_uses_vl = true;
4228   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4229   emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);



4230 }
4231 
4232 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4233   _instruction_uses_vl = true;
4234   assert(VM_Version::supports_avx(), "");
4235   if (VM_Version::supports_evex()) {
4236     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4237   } else {
4238     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4239   }
4240 }
4241 
4242 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4243   _instruction_uses_vl = true;
4244   assert(VM_Version::supports_avx(), "");
4245   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);




4246 }
4247 
4248 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4249   _instruction_uses_vl = true;
4250   assert(VM_Version::supports_avx(), "");
4251   if (VM_Version::supports_evex()) {
4252     _tuple_type = EVEX_FV;
4253     _input_size_in_bits = EVEX_64bit;
4254     emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4255   } else {
4256     emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4257   }
4258 }
4259 
4260 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4261   _instruction_uses_vl = true;
4262   assert(VM_Version::supports_avx(), "");
4263   if (VM_Version::supports_evex()) {
4264     _tuple_type = EVEX_FV;
4265     _input_size_in_bits = EVEX_32bit;
4266   }
4267   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);


4268 }
4269 
4270 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4271   _instruction_uses_vl = true;
4272   assert(VM_Version::supports_avx(), "");
4273   if (VM_Version::supports_evex()) {
4274     emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4275   } else {
4276     emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4277   }
4278 }
4279 
4280 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4281   _instruction_uses_vl = true;
4282   assert(VM_Version::supports_avx(), "");
4283   if (VM_Version::supports_evex()) {
4284     _tuple_type = EVEX_FV;
4285     _input_size_in_bits = EVEX_64bit;
4286     emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4287   } else {
4288     emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4289   }
4290 }
4291 
4292 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4294   if (VM_Version::supports_avx512dq()) {
4295     emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
4296   } else {
4297     emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4298   }
4299 }
4300 
4301 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4302   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4303   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);



4304 }
4305 
4306 void Assembler::andps(XMMRegister dst, Address src) {
4307   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4308   if (VM_Version::supports_evex()) {
4309     _tuple_type = EVEX_FV;
4310     _input_size_in_bits = EVEX_32bit;
4311   }
4312   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);

4313 }
4314 
4315 void Assembler::andpd(XMMRegister dst, Address src) {
4316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4317   if (VM_Version::supports_avx512dq()) {
4318     _tuple_type = EVEX_FV;
4319     _input_size_in_bits = EVEX_64bit;
4320     emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
4321   } else {
4322     emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4323   }
4324 }
4325 
4326 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4327   assert(VM_Version::supports_avx(), "");
4328   if (VM_Version::supports_avx512dq()) {
4329     emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
4330   } else {
4331     emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4332   }
4333 }
4334 
4335 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4336   assert(VM_Version::supports_avx(), "");
4337   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false,  /* legacy_mode */ _legacy_mode_dq);




4338 }
4339 
4340 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4341   assert(VM_Version::supports_avx(), "");
4342   if (VM_Version::supports_avx512dq()) {
4343     _tuple_type = EVEX_FV;
4344     _input_size_in_bits = EVEX_64bit;
4345     emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
4346   } else {
4347     emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4348   }
4349 }
4350 
4351 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4352   assert(VM_Version::supports_avx(), "");
4353   if (VM_Version::supports_evex()) {
4354     _tuple_type = EVEX_FV;
4355     _input_size_in_bits = EVEX_32bit;
4356   }
4357   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);


4358 }
4359 
4360 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4361   _instruction_uses_vl = true;
4362   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4363   if (VM_Version::supports_evex()) {
4364     emit_simd_arith_q(0x15, dst, src, VEX_SIMD_66);
4365   } else {
4366     emit_simd_arith(0x15, dst, src, VEX_SIMD_66);
4367   }
4368 }
4369 
4370 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4371   _instruction_uses_vl = true;
4372   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4373   if (VM_Version::supports_evex()) {
4374     emit_simd_arith_q(0x14, dst, src, VEX_SIMD_66);
4375   } else {
4376     emit_simd_arith(0x14, dst, src, VEX_SIMD_66);
4377   }
4378 }
4379 
4380 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4381   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4382   if (VM_Version::supports_avx512dq()) {
4383     emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
4384   } else {
4385     emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4386   }
4387 }
4388 
4389 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4390   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4391   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);



4392 }
4393 
4394 void Assembler::xorpd(XMMRegister dst, Address src) {
4395   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4396   if (VM_Version::supports_avx512dq()) {
4397     _tuple_type = EVEX_FV;
4398     _input_size_in_bits = EVEX_64bit;
4399     emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
4400   } else {
4401     emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4402   }
4403 }
4404 
4405 void Assembler::xorps(XMMRegister dst, Address src) {
4406   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4407   if (VM_Version::supports_evex()) {
4408     _tuple_type = EVEX_FV;
4409     _input_size_in_bits = EVEX_32bit;
4410   }
4411   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);

4412 }
4413 
4414 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4415   assert(VM_Version::supports_avx(), "");
4416   if (VM_Version::supports_avx512dq()) {
4417     emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
4418   } else {
4419     emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4420   }
4421 }
4422 
4423 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4424   assert(VM_Version::supports_avx(), "");
4425   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);




4426 }
4427 
4428 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4429   assert(VM_Version::supports_avx(), "");
4430   if (VM_Version::supports_avx512dq()) {
4431     _tuple_type = EVEX_FV;
4432     _input_size_in_bits = EVEX_64bit;
4433     emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
4434   } else {
4435     emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4436   }
4437 }
4438 
4439 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4440   assert(VM_Version::supports_avx(), "");
4441   if (VM_Version::supports_evex()) {
4442     _tuple_type = EVEX_FV;
4443     _input_size_in_bits = EVEX_32bit;
4444   }
4445   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);


4446 }
4447 
4448 // Integer vector arithmetic
4449 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4450   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4451          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4452   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);


4453   emit_int8(0x01);
4454   emit_int8((unsigned char)(0xC0 | encode));
4455 }
4456 
4457 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4458   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4459          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4460   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);


4461   emit_int8(0x02);
4462   emit_int8((unsigned char)(0xC0 | encode));
4463 }
4464 
4465 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4466   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4467   emit_simd_arith(0xFC, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4468 }
4469 
4470 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4471   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4472   emit_simd_arith(0xFD, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4473 }
4474 
4475 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
4476   _instruction_uses_vl = true;
4477   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4478   emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);



4479 }
4480 
4481 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
4482   _instruction_uses_vl = true;
4483   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4484   if (VM_Version::supports_evex()) {
4485     emit_simd_arith_q(0xD4, dst, src, VEX_SIMD_66);
4486   } else {
4487     emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
4488   }
4489 }
4490 
4491 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
4492   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4493   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
4494                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
4495   emit_int8(0x01);
4496   emit_int8((unsigned char)(0xC0 | encode));
4497 }
4498 
4499 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
4500   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4501   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
4502                                       VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
4503   emit_int8(0x02);
4504   emit_int8((unsigned char)(0xC0 | encode));
4505 }
4506 
4507 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4508   assert(UseAVX > 0, "requires some form of AVX");
4509   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);




4510 }
4511 
4512 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4513   assert(UseAVX > 0, "requires some form of AVX");
4514   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);




4515 }
4516 
4517 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4518   _instruction_uses_vl = true;
4519   assert(UseAVX > 0, "requires some form of AVX");
4520   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);




4521 }
4522 
4523 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4524   _instruction_uses_vl = true;
4525   assert(UseAVX > 0, "requires some form of AVX");
4526   if (VM_Version::supports_evex()) {
4527     emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4528   } else {
4529     emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4530   }
4531 }
4532 
4533 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4534   assert(UseAVX > 0, "requires some form of AVX");
4535   if (VM_Version::supports_evex()) {
4536     _tuple_type = EVEX_FVM;
4537   }
4538   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4539 }
4540 
4541 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4542   assert(UseAVX > 0, "requires some form of AVX");
4543   if (VM_Version::supports_evex()) {
4544     _tuple_type = EVEX_FVM;
4545   }
4546   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4547 }
4548 
4549 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4550   _instruction_uses_vl = true;
4551   assert(UseAVX > 0, "requires some form of AVX");
4552   if (VM_Version::supports_evex()) {
4553     _tuple_type = EVEX_FV;
4554     _input_size_in_bits = EVEX_32bit;
4555   }
4556   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);


4557 }
4558 
4559 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4560   _instruction_uses_vl = true;
4561   assert(UseAVX > 0, "requires some form of AVX");
4562   if (VM_Version::supports_evex()) {
4563     _tuple_type = EVEX_FV;
4564     _input_size_in_bits = EVEX_64bit;
4565     emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4566   } else {
4567     emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4568   }
4569 }
4570 
4571 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
4572   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4573   emit_simd_arith(0xF8, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4574 }
4575 
4576 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
4577   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4578   emit_simd_arith(0xF9, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4579 }
4580 
4581 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
4582   _instruction_uses_vl = true;
4583   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4584   emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);

4585 }
4586 
4587 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
4588   _instruction_uses_vl = true;
4589   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4590   if (VM_Version::supports_evex()) {
4591     emit_simd_arith_q(0xFB, dst, src, VEX_SIMD_66);
4592   } else {
4593     emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
4594   }
4595 }
4596 
4597 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4598   assert(UseAVX > 0, "requires some form of AVX");
4599   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);




4600 }
4601 
4602 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4603   assert(UseAVX > 0, "requires some form of AVX");
4604   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);




4605 }
4606 
4607 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4608   _instruction_uses_vl = true;
4609   assert(UseAVX > 0, "requires some form of AVX");
4610   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);




4611 }
4612 
4613 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4614   _instruction_uses_vl = true;
4615   assert(UseAVX > 0, "requires some form of AVX");
4616   if (VM_Version::supports_evex()) {
4617     emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4618   } else {
4619     emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4620   }
4621 }
4622 
4623 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4624   assert(UseAVX > 0, "requires some form of AVX");
4625   if (VM_Version::supports_evex()) {
4626     _tuple_type = EVEX_FVM;
4627   }
4628   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4629 }
4630 
4631 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4632   assert(UseAVX > 0, "requires some form of AVX");
4633   if (VM_Version::supports_evex()) {
4634     _tuple_type = EVEX_FVM;
4635   }
4636   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4637 }
4638 
4639 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4640   _instruction_uses_vl = true;
4641   assert(UseAVX > 0, "requires some form of AVX");
4642   if (VM_Version::supports_evex()) {
4643     _tuple_type = EVEX_FV;
4644     _input_size_in_bits = EVEX_32bit;
4645   }
4646   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);


4647 }
4648 
4649 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4650   _instruction_uses_vl = true;
4651   assert(UseAVX > 0, "requires some form of AVX");
4652   if (VM_Version::supports_evex()) {
4653     _tuple_type = EVEX_FV;
4654     _input_size_in_bits = EVEX_64bit;
4655     emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4656   } else {
4657     emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4658   }
4659 }
4660 
4661 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
4662   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4663   emit_simd_arith(0xD5, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4664 }
4665 
4666 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
4667   _instruction_uses_vl = true;
4668   assert(VM_Version::supports_sse4_1(), "");
4669   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66,
4670                                       /* no_mask_reg */ false, VEX_OPCODE_0F_38);
4671   emit_int8(0x40);
4672   emit_int8((unsigned char)(0xC0 | encode));
4673 }
4674 
4675 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4676   assert(UseAVX > 0, "requires some form of AVX");
4677   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);




4678 }
4679 
4680 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4681   _instruction_uses_vl = true;
4682   assert(UseAVX > 0, "requires some form of AVX");
4683   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);


4684   emit_int8(0x40);
4685   emit_int8((unsigned char)(0xC0 | encode));
4686 }
4687 
4688 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4689   assert(UseAVX > 2, "requires some form of AVX");
4690   int src_enc = src->encoding();
4691   int dst_enc = dst->encoding();
4692   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4693   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_38,
4694                                      /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false);
4695   emit_int8(0x40);
4696   emit_int8((unsigned char)(0xC0 | encode));
4697 }
4698 
4699 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4700   assert(UseAVX > 0, "requires some form of AVX");
4701   if (VM_Version::supports_evex()) {
4702     _tuple_type = EVEX_FVM;
4703   }
4704   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4705 }
4706 
4707 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4708   _instruction_uses_vl = true;
4709   assert(UseAVX > 0, "requires some form of AVX");
4710   if (VM_Version::supports_evex()) {
4711     _tuple_type = EVEX_FV;
4712     _input_size_in_bits = EVEX_32bit;
4713   }
4714   InstructionMark im(this);
4715   int dst_enc = dst->encoding();

4716   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4717   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
4718              VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
4719   emit_int8(0x40);
4720   emit_operand(dst, src);
4721 }
4722 
4723 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4724   assert(UseAVX > 0, "requires some form of AVX");
4725   if (VM_Version::supports_evex()) {
4726     _tuple_type = EVEX_FV;
4727     _input_size_in_bits = EVEX_64bit;
4728   }
4729   InstructionMark im(this);
4730   int dst_enc = dst->encoding();

4731   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4732   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
4733              VEX_OPCODE_0F_38, /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq);
4734   emit_int8(0x40);
4735   emit_operand(dst, src);
4736 }
4737 
4738 // Shift packed integers left by specified number of bits.
4739 void Assembler::psllw(XMMRegister dst, int shift) {
4740   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4741   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4742   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F,
4743                                       /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4744   emit_int8(0x71);
4745   emit_int8((unsigned char)(0xC0 | encode));
4746   emit_int8(shift & 0xFF);
4747 }
4748 
4749 void Assembler::pslld(XMMRegister dst, int shift) {
4750   _instruction_uses_vl = true;
4751   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4752   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4753   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4754   emit_int8(0x72);
4755   emit_int8((unsigned char)(0xC0 | encode));
4756   emit_int8(shift & 0xFF);
4757 }
4758 
4759 void Assembler::psllq(XMMRegister dst, int shift) {
4760   _instruction_uses_vl = true;
4761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4762   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4763   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ true);
4764   emit_int8(0x73);
4765   emit_int8((unsigned char)(0xC0 | encode));
4766   emit_int8(shift & 0xFF);
4767 }
4768 
4769 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
4770   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4771   emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4772 }
4773 
4774 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
4775   _instruction_uses_vl = true;
4776   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4777   emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);



4778 }
4779 
4780 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
4781   _instruction_uses_vl = true;
4782   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4783   if (VM_Version::supports_evex()) {
4784     emit_simd_arith_q(0xF3, dst, shift, VEX_SIMD_66);
4785   } else {
4786     emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
4787   }
4788 }
4789 
4790 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4791   assert(UseAVX > 0, "requires some form of AVX");

4792   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4793   emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);


4794   emit_int8(shift & 0xFF);
4795 }
4796 
4797 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4798   _instruction_uses_vl = true;
4799   assert(UseAVX > 0, "requires some form of AVX");


4800   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4801   emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector_len);


4802   emit_int8(shift & 0xFF);
4803 }
4804 
4805 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4806   _instruction_uses_vl = true;
4807   assert(UseAVX > 0, "requires some form of AVX");

4808   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4809   if (VM_Version::supports_evex()) {
4810     emit_vex_arith_q(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
4811   } else {
4812     emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
4813   }
4814   emit_int8(shift & 0xFF);
4815 }
4816 
4817 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4818   assert(UseAVX > 0, "requires some form of AVX");
4819   emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4820 }
4821 
4822 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4823   _instruction_uses_vl = true;
4824   assert(UseAVX > 0, "requires some form of AVX");
4825   emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector_len);



4826 }
4827 
4828 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4829   _instruction_uses_vl = true;
4830   assert(UseAVX > 0, "requires some form of AVX");
4831   if (VM_Version::supports_evex()) {
4832     emit_vex_arith_q(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
4833   } else {
4834     emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
4835   }
4836 }
4837 
4838 // Shift packed integers logically right by specified number of bits.
4839 void Assembler::psrlw(XMMRegister dst, int shift) {
4840   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4841   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4842   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4843                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4844   emit_int8(0x71);
4845   emit_int8((unsigned char)(0xC0 | encode));
4846   emit_int8(shift & 0xFF);
4847 }
4848 
4849 void Assembler::psrld(XMMRegister dst, int shift) {
4850   _instruction_uses_vl = true;
4851   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4852   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4853   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4854   emit_int8(0x72);
4855   emit_int8((unsigned char)(0xC0 | encode));
4856   emit_int8(shift & 0xFF);
4857 }
4858 
4859 void Assembler::psrlq(XMMRegister dst, int shift) {
4860   _instruction_uses_vl = true;
4861   // Do not confuse it with psrldq SSE2 instruction which
4862   // shifts 128 bit value in xmm register by number of bytes.
4863   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4864   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4865   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4866                                       VEX_OPCODE_0F, /* rex_w */ VM_Version::supports_evex());
4867   emit_int8(0x73);
4868   emit_int8((unsigned char)(0xC0 | encode));
4869   emit_int8(shift & 0xFF);
4870 }
4871 
4872 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
4873   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4874   emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4875 }
4876 
4877 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
4878   _instruction_uses_vl = true;
4879   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4880   emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);



4881 }
4882 
4883 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
4884   _instruction_uses_vl = true;
4885   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4886   if (VM_Version::supports_evex()) {
4887     emit_simd_arith_q(0xD3, dst, shift, VEX_SIMD_66);
4888   } else {
4889     emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
4890   }
4891 }
4892 
4893 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4894   assert(UseAVX > 0, "requires some form of AVX");

4895   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4896   emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);


4897   emit_int8(shift & 0xFF);
4898 }
4899 
4900 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4901   _instruction_uses_vl = true;
4902   assert(UseAVX > 0, "requires some form of AVX");

4903   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4904   emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector_len);


4905   emit_int8(shift & 0xFF);
4906 }
4907 
4908 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4909   _instruction_uses_vl = true;
4910   assert(UseAVX > 0, "requires some form of AVX");

4911   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4912   if (VM_Version::supports_evex()) {
4913     emit_vex_arith_q(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
4914   } else {
4915     emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
4916   }
4917   emit_int8(shift & 0xFF);
4918 }
4919 
4920 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4921   assert(UseAVX > 0, "requires some form of AVX");
4922   emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4923 }
4924 
4925 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4926   _instruction_uses_vl = true;
4927   assert(UseAVX > 0, "requires some form of AVX");
4928   emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector_len);



4929 }
4930 
4931 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4932   _instruction_uses_vl = true;
4933   assert(UseAVX > 0, "requires some form of AVX");
4934   if (VM_Version::supports_evex()) {
4935     emit_vex_arith_q(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
4936   } else {
4937     emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
4938   }
4939 }
4940 
4941 // Shift packed integers arithmetically right by specified number of bits.
4942 void Assembler::psraw(XMMRegister dst, int shift) {
4943   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4944   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4945   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4946                                       VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4947   emit_int8(0x71);
4948   emit_int8((unsigned char)(0xC0 | encode));
4949   emit_int8(shift & 0xFF);
4950 }
4951 
4952 void Assembler::psrad(XMMRegister dst, int shift) {
4953   _instruction_uses_vl = true;
4954   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

4955   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
4956   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4957   emit_int8(0x72);
4958   emit_int8((unsigned char)(0xC0 | encode));
4959   emit_int8(shift & 0xFF);
4960 }
4961 
4962 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
4963   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4964   emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4965 }
4966 
4967 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
4968   _instruction_uses_vl = true;
4969   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4970   emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);



4971 }
4972 
4973 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4974   assert(UseAVX > 0, "requires some form of AVX");

4975   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4976   emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);


4977   emit_int8(shift & 0xFF);
4978 }
4979 
4980 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4981   _instruction_uses_vl = true;
4982   assert(UseAVX > 0, "requires some form of AVX");

4983   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4984   emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector_len);


4985   emit_int8(shift & 0xFF);
4986 }
4987 
4988 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4989   assert(UseAVX > 0, "requires some form of AVX");
4990   emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);



4991 }
4992 
4993 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4994   _instruction_uses_vl = true;
4995   assert(UseAVX > 0, "requires some form of AVX");
4996   emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector_len);



4997 }
4998 
4999 
5000 // logical operations packed integers
5001 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5002   _instruction_uses_vl = true;
5003   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5004   emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);



5005 }
5006 
5007 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5008   _instruction_uses_vl = true;
5009   assert(UseAVX > 0, "requires some form of AVX");
5010   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);




5011 }
5012 
5013 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5014   _instruction_uses_vl = true;
5015   assert(UseAVX > 0, "requires some form of AVX");
5016   if (VM_Version::supports_evex()) {
5017     _tuple_type = EVEX_FV;
5018     _input_size_in_bits = EVEX_32bit;
5019   }
5020   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);


5021 }
5022 
5023 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5024   _instruction_uses_vl = true;
5025   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5026   if (VM_Version::supports_evex()) {
5027     emit_simd_arith_q(0xDF, dst, src, VEX_SIMD_66);
5028   }
5029   else {
5030     emit_simd_arith(0xDF, dst, src, VEX_SIMD_66);
5031   }
5032 }
5033 
5034 void Assembler::por(XMMRegister dst, XMMRegister src) {
5035   _instruction_uses_vl = true;
5036   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5037   emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);



5038 }
5039 
5040 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5041   _instruction_uses_vl = true;
5042   assert(UseAVX > 0, "requires some form of AVX");
5043   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);




5044 }
5045 
5046 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5047   _instruction_uses_vl = true;
5048   assert(UseAVX > 0, "requires some form of AVX");
5049   if (VM_Version::supports_evex()) {
5050     _tuple_type = EVEX_FV;
5051     _input_size_in_bits = EVEX_32bit;
5052   }
5053   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);


5054 }
5055 
5056 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5057   _instruction_uses_vl = true;
5058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5059   emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);



5060 }
5061 
5062 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5063   _instruction_uses_vl = true;
5064   assert(UseAVX > 0, "requires some form of AVX");
5065   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);




5066 }
5067 
5068 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5069   _instruction_uses_vl = true;
5070   assert(UseAVX > 0, "requires some form of AVX");
5071   if (VM_Version::supports_evex()) {
5072     _tuple_type = EVEX_FV;
5073     _input_size_in_bits = EVEX_32bit;
5074   }
5075   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);


5076 }
5077 
5078 
5079 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5080   assert(VM_Version::supports_avx(), "");
5081   int vector_len = AVX_256bit;
5082   if (VM_Version::supports_evex()) {
5083     vector_len = AVX_512bit;
5084   }
5085   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5086   emit_int8(0x18);
5087   emit_int8((unsigned char)(0xC0 | encode));
5088   // 0x00 - insert into lower 128 bits
5089   // 0x01 - insert into upper 128 bits
5090   emit_int8(0x01);
5091 }
5092 
5093 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5094   assert(VM_Version::supports_evex(), "");
5095   int vector_len = AVX_512bit;
5096   int src_enc = src->encoding();
5097   int dst_enc = dst->encoding();
5098   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5099   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5100                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5101   emit_int8(0x1A);
5102   emit_int8((unsigned char)(0xC0 | encode));
5103   // 0x00 - insert into lower 256 bits
5104   // 0x01 - insert into upper 256 bits
5105   emit_int8(0x01);
5106 }
5107 
5108 void Assembler::vinsertf64x4h(XMMRegister dst, Address src) {
5109   assert(VM_Version::supports_evex(), "");
5110   _tuple_type = EVEX_T4;
5111   _input_size_in_bits = EVEX_64bit;
5112   InstructionMark im(this);
5113   int vector_len = AVX_512bit;
5114   assert(dst != xnoreg, "sanity");
5115   int dst_enc = dst->encoding();


5116   // swap src<->dst for encoding
5117   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ true, vector_len);
5118   emit_int8(0x1A);
5119   emit_operand(dst, src);

5120   // 0x01 - insert into upper 128 bits
5121   emit_int8(0x01);
5122 }
5123 
5124 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5125   assert(VM_Version::supports_evex(), "");
5126   int vector_len = AVX_512bit;
5127   int src_enc = src->encoding();
5128   int dst_enc = dst->encoding();
5129   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5130   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5131                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5132   emit_int8(0x18);
5133   emit_int8((unsigned char)(0xC0 | encode));
5134   // 0x00 - insert into q0 128 bits (0..127)
5135   // 0x01 - insert into q1 128 bits (128..255)
5136   // 0x02 - insert into q2 128 bits (256..383)
5137   // 0x03 - insert into q3 128 bits (384..511)
5138   emit_int8(value & 0x3);
5139 }
5140 
5141 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
5142   assert(VM_Version::supports_evex(), "");
5143   _tuple_type = EVEX_T4;
5144   _input_size_in_bits = EVEX_32bit;
5145   InstructionMark im(this);
5146   int vector_len = AVX_512bit;
5147   assert(dst != xnoreg, "sanity");
5148   int dst_enc = dst->encoding();



5149   // swap src<->dst for encoding
5150   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5151   emit_int8(0x18);
5152   emit_operand(dst, src);
5153   // 0x00 - insert into q0 128 bits (0..127)
5154   // 0x01 - insert into q1 128 bits (128..255)
5155   // 0x02 - insert into q2 128 bits (256..383)
5156   // 0x03 - insert into q3 128 bits (384..511)
5157   emit_int8(value & 0x3);
5158 }
5159 
5160 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
5161   assert(VM_Version::supports_avx(), "");
5162   int vector_len = AVX_256bit;
5163   if (VM_Version::supports_evex()) {
5164     _tuple_type = EVEX_T4;
5165     _input_size_in_bits = EVEX_32bit;
5166     vector_len = AVX_512bit;
5167   }
5168   InstructionMark im(this);
5169   assert(dst != xnoreg, "sanity");
5170   int dst_enc = dst->encoding();



5171   // swap src<->dst for encoding
5172   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5173   emit_int8(0x18);
5174   emit_operand(dst, src);
5175   // 0x01 - insert into upper 128 bits
5176   emit_int8(0x01);
5177 }
5178 
5179 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
5180   assert(VM_Version::supports_avx(), "");
5181   int vector_len = AVX_256bit;
5182   if (VM_Version::supports_evex()) {
5183     vector_len = AVX_512bit;
5184   }
5185   int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5186   emit_int8(0x19);
5187   emit_int8((unsigned char)(0xC0 | encode));
5188   // 0x00 - insert into lower 128 bits
5189   // 0x01 - insert into upper 128 bits
5190   emit_int8(0x01);
5191 }
5192 
5193 void Assembler::vextractf128h(Address dst, XMMRegister src) {
5194   assert(VM_Version::supports_avx(), "");
5195   int vector_len = AVX_256bit;
5196   if (VM_Version::supports_evex()) {
5197     _tuple_type = EVEX_T4;
5198     _input_size_in_bits = EVEX_32bit;
5199     vector_len = AVX_512bit;
5200   }
5201   InstructionMark im(this);
5202   assert(src != xnoreg, "sanity");
5203   int src_enc = src->encoding();
5204   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);



5205   emit_int8(0x19);
5206   emit_operand(src, dst);
5207   // 0x01 - extract from upper 128 bits
5208   emit_int8(0x01);
5209 }
5210 
5211 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5212   assert(VM_Version::supports_avx2(), "");
5213   int vector_len = AVX_256bit;
5214   if (VM_Version::supports_evex()) {
5215     vector_len = AVX_512bit;
5216   }
5217   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5218   emit_int8(0x38);
5219   emit_int8((unsigned char)(0xC0 | encode));
5220   // 0x00 - insert into lower 128 bits
5221   // 0x01 - insert into upper 128 bits
5222   emit_int8(0x01);
5223 }
5224 
5225 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5226   assert(VM_Version::supports_evex(), "");
5227   int vector_len = AVX_512bit;
5228   int src_enc = src->encoding();
5229   int dst_enc = dst->encoding();
5230   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5231   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5232                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_reg_mask */ false);
5233   emit_int8(0x38);
5234   emit_int8((unsigned char)(0xC0 | encode));
5235   // 0x00 - insert into lower 256 bits
5236   // 0x01 - insert into upper 256 bits
5237   emit_int8(0x01);
5238 }
5239 
5240 void Assembler::vinserti128h(XMMRegister dst, Address src) {
5241   assert(VM_Version::supports_avx2(), "");
5242   int vector_len = AVX_256bit;
5243   if (VM_Version::supports_evex()) {
5244     _tuple_type = EVEX_T4;
5245     _input_size_in_bits = EVEX_32bit;
5246     vector_len = AVX_512bit;
5247   }
5248   InstructionMark im(this);
5249   assert(dst != xnoreg, "sanity");
5250   int dst_enc = dst->encoding();



5251   // swap src<->dst for encoding
5252   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5253   emit_int8(0x38);
5254   emit_operand(dst, src);
5255   // 0x01 - insert into upper 128 bits
5256   emit_int8(0x01);
5257 }
5258 
5259 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
5260   assert(VM_Version::supports_avx(), "");
5261   int vector_len = AVX_256bit;
5262   if (VM_Version::supports_evex()) {
5263     vector_len = AVX_512bit;
5264   }
5265   int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5266   emit_int8(0x39);
5267   emit_int8((unsigned char)(0xC0 | encode));
5268   // 0x00 - insert into lower 128 bits
5269   // 0x01 - insert into upper 128 bits
5270   emit_int8(0x01);
5271 }
5272 
5273 void Assembler::vextracti128h(Address dst, XMMRegister src) {
5274   assert(VM_Version::supports_avx2(), "");
5275   int vector_len = AVX_256bit;
5276   if (VM_Version::supports_evex()) {
5277     _tuple_type = EVEX_T4;
5278     _input_size_in_bits = EVEX_32bit;
5279     vector_len = AVX_512bit;
5280   }
5281   InstructionMark im(this);
5282   assert(src != xnoreg, "sanity");
5283   int src_enc = src->encoding();
5284   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);



5285   emit_int8(0x39);
5286   emit_operand(src, dst);
5287   // 0x01 - extract from upper 128 bits
5288   emit_int8(0x01);
5289 }
5290 
5291 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src) {
5292   assert(VM_Version::supports_evex(), "");
5293   int vector_len = AVX_512bit;
5294   int src_enc = src->encoding();
5295   int dst_enc = dst->encoding();
5296   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5297                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5298   emit_int8(0x3B);
5299   emit_int8((unsigned char)(0xC0 | encode));

5300   // 0x01 - extract from upper 256 bits
5301   emit_int8(0x01);
5302 }
5303 
5304 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
5305   assert(VM_Version::supports_evex(), "");
5306   int vector_len = AVX_512bit;
5307   int src_enc = src->encoding();
5308   int dst_enc = dst->encoding();
5309   int encode;
5310   if (VM_Version::supports_avx512dq()) {
5311     encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5312                                    /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5313   } else {
5314     encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5315                                    /* vex_w */ false, vector_len, /* legacy_mode */ true, /* no_mask_reg */ false);
5316   }
5317   emit_int8(0x39);
5318   emit_int8((unsigned char)(0xC0 | encode));
5319   // 0x01 - extract from bits 255:128
5320   // 0x02 - extract from bits 383:256
5321   // 0x03 - extract from bits 511:384
5322   emit_int8(value & 0x3);
5323 }
5324 
5325 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src) {
5326   assert(VM_Version::supports_evex(), "");
5327   int vector_len = AVX_512bit;
5328   int src_enc = src->encoding();
5329   int dst_enc = dst->encoding();
5330   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5331                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5332   emit_int8(0x1B);
5333   emit_int8((unsigned char)(0xC0 | encode));

5334   // 0x01 - extract from upper 256 bits
5335   emit_int8(0x01);
5336 }
5337 
5338 void Assembler::vextractf64x4h(Address dst, XMMRegister src) {
5339   assert(VM_Version::supports_evex(), "");
5340   _tuple_type = EVEX_T4;
5341   _input_size_in_bits = EVEX_64bit;
5342   InstructionMark im(this);
5343   int vector_len = AVX_512bit;
5344   assert(src != xnoreg, "sanity");
5345   int src_enc = src->encoding();
5346   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5347              /* vex_w */ true, vector_len);

5348   emit_int8(0x1B);
5349   emit_operand(src, dst);

5350   // 0x01 - extract from upper 256 bits
5351   emit_int8(0x01);
5352 }
5353 
5354 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
5355   assert(VM_Version::supports_evex(), "");
5356   int vector_len = AVX_512bit;
5357   int src_enc = src->encoding();
5358   int dst_enc = dst->encoding();
5359   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5360                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5361   emit_int8(0x19);
5362   emit_int8((unsigned char)(0xC0 | encode));
5363   // 0x00 - extract from bits 127:0
5364   // 0x01 - extract from bits 255:128
5365   // 0x02 - extract from bits 383:256
5366   // 0x03 - extract from bits 511:384
5367   emit_int8(value & 0x3);
5368 }
5369 
5370 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
5371   assert(VM_Version::supports_evex(), "");
5372   _tuple_type = EVEX_T4;
5373   _input_size_in_bits = EVEX_32bit;
5374   InstructionMark im(this);
5375   int vector_len = AVX_512bit;
5376   assert(src != xnoreg, "sanity");
5377   int src_enc = src->encoding();
5378   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);


5379   emit_int8(0x19);
5380   emit_operand(src, dst);
5381   // 0x00 - extract from bits 127:0
5382   // 0x01 - extract from bits 255:128
5383   // 0x02 - extract from bits 383:256
5384   // 0x03 - extract from bits 511:384
5385   emit_int8(value & 0x3);
5386 }
5387 
5388 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
5389   assert(VM_Version::supports_evex(), "");
5390   int vector_len = AVX_512bit;
5391   int src_enc = src->encoding();
5392   int dst_enc = dst->encoding();
5393   int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5394                                      /* vex_w */ !_legacy_mode_dq, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5395   emit_int8(0x19);
5396   emit_int8((unsigned char)(0xC0 | encode));
5397   // 0x01 - extract from bits 255:128
5398   // 0x02 - extract from bits 383:256
5399   // 0x03 - extract from bits 511:384
5400   emit_int8(value & 0x3);
5401 }
5402 
5403 // duplicate 4-bytes integer data from src into 8 locations in dest
5404 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
5405   _instruction_uses_vl = true;
5406   assert(UseAVX > 1, "");
5407   int vector_len = AVX_256bit;
5408   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5409   emit_int8(0x58);
5410   emit_int8((unsigned char)(0xC0 | encode));
5411 }
5412 
5413 // duplicate 2-bytes integer data from src into 16 locations in dest
5414 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
5415   assert(VM_Version::supports_avx2(), "");
5416   bool vector_len = AVX_256bit;
5417   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66,
5418                                      vector_len, VEX_OPCODE_0F_38, false);
5419   emit_int8(0x79);
5420   emit_int8((unsigned char)(0xC0 | encode));
5421 }
5422 
5423 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5424 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
5425   _instruction_uses_vl = true;
5426   assert(UseAVX > 1, "");
5427   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5428   emit_int8(0x78);
5429   emit_int8((unsigned char)(0xC0 | encode));
5430 }
5431 
5432 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
5433   _instruction_uses_vl = true;
5434   assert(UseAVX > 1, "");
5435   _tuple_type = EVEX_T1S;
5436   _input_size_in_bits = EVEX_8bit;
5437   InstructionMark im(this);
5438   assert(dst != xnoreg, "sanity");
5439   int dst_enc = dst->encoding();


5440   // swap src<->dst for encoding
5441   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5442   emit_int8(0x78);
5443   emit_operand(dst, src);
5444 }
5445 
5446 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5447 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5448   _instruction_uses_vl = true;
5449   assert(UseAVX > 1, "");
5450   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5451   emit_int8(0x79);
5452   emit_int8((unsigned char)(0xC0 | encode));
5453 }
5454 
5455 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5456   _instruction_uses_vl = true;
5457   assert(UseAVX > 1, "");
5458   _tuple_type = EVEX_T1S;
5459   _input_size_in_bits = EVEX_16bit;
5460   InstructionMark im(this);
5461   assert(dst != xnoreg, "sanity");
5462   int dst_enc = dst->encoding();


5463   // swap src<->dst for encoding
5464   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5465   emit_int8(0x79);
5466   emit_operand(dst, src);
5467 }
5468 
5469 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5470 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5471   _instruction_uses_vl = true;
5472   assert(UseAVX > 1, "");
5473   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5474   emit_int8(0x58);
5475   emit_int8((unsigned char)(0xC0 | encode));
5476 }
5477 
5478 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5479   _instruction_uses_vl = true;
5480   assert(UseAVX > 1, "");
5481   _tuple_type = EVEX_T1S;
5482   _input_size_in_bits = EVEX_32bit;
5483   InstructionMark im(this);
5484   assert(dst != xnoreg, "sanity");
5485   int dst_enc = dst->encoding();


5486   // swap src<->dst for encoding
5487   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5488   emit_int8(0x58);
5489   emit_operand(dst, src);
5490 }
5491 
5492 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5493 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5494   _instruction_uses_vl = true;
5495   assert(UseAVX > 1, "");
5496   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5497                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5498   emit_int8(0x59);
5499   emit_int8((unsigned char)(0xC0 | encode));
5500 }
5501 
5502 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5503   _instruction_uses_vl = true;
5504   assert(UseAVX > 1, "");
5505   _tuple_type = EVEX_T1S;
5506   _input_size_in_bits = EVEX_64bit;
5507   InstructionMark im(this);
5508   assert(dst != xnoreg, "sanity");
5509   int dst_enc = dst->encoding();


5510   // swap src<->dst for encoding
5511   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
5512   emit_int8(0x59);
5513   emit_operand(dst, src);
5514 }
5515 
5516 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5517 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5518   _instruction_uses_vl = true;
5519   assert(UseAVX > 1, "");
5520   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5521                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5522   emit_int8(0x18);
5523   emit_int8((unsigned char)(0xC0 | encode));
5524 }
5525 
5526 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5527   assert(UseAVX > 1, "");
5528   _tuple_type = EVEX_T1S;
5529   _input_size_in_bits = EVEX_32bit;
5530   InstructionMark im(this);
5531   assert(dst != xnoreg, "sanity");
5532   int dst_enc = dst->encoding();


5533   // swap src<->dst for encoding
5534   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5535   emit_int8(0x18);
5536   emit_operand(dst, src);
5537 }
5538 
5539 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5540 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5541   _instruction_uses_vl = true;
5542   assert(UseAVX > 1, "");
5543   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5544                                      /*vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5545   emit_int8(0x19);
5546   emit_int8((unsigned char)(0xC0 | encode));
5547 }
5548 
5549 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5550   _instruction_uses_vl = true;
5551   assert(UseAVX > 1, "");
5552   _tuple_type = EVEX_T1S;
5553   _input_size_in_bits = EVEX_64bit;
5554   InstructionMark im(this);
5555   assert(dst != xnoreg, "sanity");
5556   int dst_enc = dst->encoding();


5557   // swap src<->dst for encoding
5558   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
5559   emit_int8(0x19);
5560   emit_operand(dst, src);
5561 }
5562 
5563 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5564 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
5565   _instruction_uses_vl = true;
5566   assert(VM_Version::supports_evex(), "");
5567   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5568                                      /*vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5569   emit_int8(0x7A);
5570   emit_int8((unsigned char)(0xC0 | encode));
5571 }
5572 
5573 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5574 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
5575   _instruction_uses_vl = true;
5576   assert(VM_Version::supports_evex(), "");
5577   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5578                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5579   emit_int8(0x7B);
5580   emit_int8((unsigned char)(0xC0 | encode));
5581 }
5582 
5583 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5584 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
5585   _instruction_uses_vl = true;
5586   assert(VM_Version::supports_evex(), "");
5587   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5588                                      /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5589   emit_int8(0x7C);
5590   emit_int8((unsigned char)(0xC0 | encode));
5591 }
5592 
5593 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5594 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
5595   _instruction_uses_vl = true;
5596   assert(VM_Version::supports_evex(), "");
5597   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5598                                      /* vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5599   emit_int8(0x7C);
5600   emit_int8((unsigned char)(0xC0 | encode));
5601 }
5602 
5603 // Carry-Less Multiplication Quadword
5604 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5605   assert(VM_Version::supports_clmul(), "");
5606   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
5607                                       VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
5608   emit_int8(0x44);
5609   emit_int8((unsigned char)(0xC0 | encode));
5610   emit_int8((unsigned char)mask);
5611 }
5612 
5613 // Carry-Less Multiplication Quadword
5614 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5615   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5616   int vector_len = AVX_128bit;
5617   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* legacy_mode */ true);

5618   emit_int8(0x44);
5619   emit_int8((unsigned char)(0xC0 | encode));
5620   emit_int8((unsigned char)mask);
5621 }
5622 
5623 void Assembler::vzeroupper() {
5624   assert(VM_Version::supports_avx(), "");
5625   if (UseAVX < 3)
5626   {
5627     (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
5628     emit_int8(0x77);
5629   }
5630 }
5631 
5632 
5633 #ifndef _LP64
5634 // 32bit only pieces of the assembler
5635 
5636 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5637   // NO PREFIX AS NEVER 64BIT
5638   InstructionMark im(this);
5639   emit_int8((unsigned char)0x81);
5640   emit_int8((unsigned char)(0xF8 | src1->encoding()));
5641   emit_data(imm32, rspec, 0);
5642 }
5643 
5644 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5645   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
5646   InstructionMark im(this);
5647   emit_int8((unsigned char)0x81);
5648   emit_operand(rdi, src1);
5649   emit_data(imm32, rspec, 0);


6113     emit_int8(simd_pre[pre]);
6114   }
6115   if (rex_w) {
6116     prefixq(adr, xreg);
6117   } else {
6118     prefix(adr, xreg);
6119   }
6120   if (opc > 0) {
6121     emit_int8(0x0F);
6122     int opc2 = simd_opc[opc];
6123     if (opc2 > 0) {
6124       emit_int8(opc2);
6125     }
6126   }
6127 }
6128 
6129 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6130   if (pre > 0) {
6131     emit_int8(simd_pre[pre]);
6132   }
6133   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
6134                           prefix_and_encode(dst_enc, src_enc);
6135   if (opc > 0) {
6136     emit_int8(0x0F);
6137     int opc2 = simd_opc[opc];
6138     if (opc2 > 0) {
6139       emit_int8(opc2);
6140     }
6141   }
6142   return encode;
6143 }
6144 
6145 
6146 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, int vector_len) {


6147   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6148     prefix(VEX_3bytes);
6149 
6150     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6151     byte1 = (~byte1) & 0xE0;
6152     byte1 |= opc;
6153     emit_int8(byte1);
6154 
6155     int byte2 = ((~nds_enc) & 0xf) << 3;
6156     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6157     emit_int8(byte2);
6158   } else {
6159     prefix(VEX_2bytes);
6160 
6161     int byte1 = vex_r ? VEX_R : 0;
6162     byte1 = (~byte1) & 0x80;
6163     byte1 |= ((~nds_enc) & 0xf) << 3;
6164     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6165     emit_int8(byte1);
6166   }
6167 }
6168 
6169 // This is a 4 byte encoding
6170 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, bool evex_r, bool evex_v,
6171                             int nds_enc, VexSimdPrefix pre, VexOpcode opc,
6172                             bool is_extended_context, bool is_merge_context,
6173                             int vector_len, bool no_mask_reg ){
6174   // EVEX 0x62 prefix
6175   prefix(EVEX_4bytes);
6176   _evex_encoding = (vex_w ? VEX_W : 0) | (evex_r ? EVEX_Rb : 0);



6177 
6178   // P0: byte 2, initialized to RXBR`00mm
6179   // instead of not'd
6180   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6181   byte2 = (~byte2) & 0xF0;
6182   // confine opc opcode extensions in mm bits to lower two bits
6183   // of form {0F, 0F_38, 0F_3A}
6184   byte2 |= opc;
6185   emit_int8(byte2);
6186 
6187   // P1: byte 3 as Wvvvv1pp
6188   int byte3 = ((~nds_enc) & 0xf) << 3;
6189   // p[10] is always 1
6190   byte3 |= EVEX_F;
6191   byte3 |= (vex_w & 1) << 7;
6192   // confine pre opcode extensions in pp bits to lower two bits
6193   // of form {66, F3, F2}
6194   byte3 |= pre;
6195   emit_int8(byte3);
6196 
6197   // P2: byte 4 as zL'Lbv'aaa
6198   int byte4 = (no_mask_reg) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6199   // EVEX.v` for extending EVEX.vvvv or VIDX
6200   byte4 |= (evex_v ? 0: EVEX_V);
6201   // third EXEC.b for broadcast actions
6202   byte4 |= (is_extended_context ? EVEX_Rb : 0);
6203   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6204   byte4 |= ((vector_len) & 0x3) << 5;
6205   // last is EVEX.z for zero/merge actions
6206   byte4 |= (is_merge_context ? EVEX_Z : 0);
6207   emit_int8(byte4);
6208 }
6209 
6210 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre,
6211                            VexOpcode opc, bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg) {
6212   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6213   bool vex_b = adr.base_needs_rex();
6214   bool vex_x = adr.index_needs_rex();
6215   _avx_vector_len = vector_len;

6216 
6217   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6218   if (_legacy_mode_vl && _instruction_uses_vl) {
6219     switch (vector_len) {
6220     case AVX_128bit:
6221     case AVX_256bit:
6222       legacy_mode = true;
6223       break;
6224     }
6225   }
6226 
6227   if ((UseAVX > 2) && (legacy_mode == false))
6228   {
6229     bool evex_r = (xreg_enc >= 16);
6230     bool evex_v = (nds_enc >= 16);
6231     _is_evex_instruction = true;
6232     evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6233   } else {
6234     vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
6235   }
6236   _instruction_uses_vl = false;
6237 }
6238 
6239 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
6240                                      bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg ) {
6241   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6242   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6243   bool vex_x = false;
6244   _avx_vector_len = vector_len;

6245 
6246   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6247   if (_legacy_mode_vl && _instruction_uses_vl) {
6248     switch (vector_len) {
6249     case AVX_128bit:
6250     case AVX_256bit:
6251       legacy_mode = true;





6252       break;
6253     }
6254   }
6255 
6256   if ((UseAVX > 2) && (legacy_mode == false))
6257   {
6258     bool evex_r = (dst_enc >= 16);
6259     bool evex_v = (nds_enc >= 16);
6260     // can use vex_x as bank extender on rm encoding
6261     vex_x = (src_enc >= 16);
6262     evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);

6263   } else {
6264     vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
6265   }
6266 
6267   _instruction_uses_vl = false;
6268 
6269   // return modrm byte components for operands
6270   return (((dst_enc & 7) << 3) | (src_enc & 7));
6271 }
6272 
6273 
6274 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6275                             bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
6276   if (UseAVX > 0) {
6277     int xreg_enc = xreg->encoding();
6278     int  nds_enc = nds->is_valid() ? nds->encoding() : 0;
6279     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
6280   } else {
6281     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6282     rex_prefix(adr, xreg, pre, opc, rex_w);
6283   }
6284 }
6285 
6286 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6287                                       bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
6288   int dst_enc = dst->encoding();
6289   int src_enc = src->encoding();
6290   if (UseAVX > 0) {
6291     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6292     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
6293   } else {
6294     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6295     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
6296   }
6297 }
6298 
6299 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
6300                                       bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
6301   int dst_enc = dst->encoding();
6302   int src_enc = src->encoding();
6303   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6304   return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
6305 }
6306 
6307 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
6308                                       bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
6309   int dst_enc = dst->encoding();
6310   int src_enc = src->encoding();
6311   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6312   return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
6313 }
6314 
6315 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6316   InstructionMark im(this);
6317   simd_prefix(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6318   emit_int8(opcode);
6319   emit_operand(dst, src);
6320 }
6321 
6322 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg) {
6323   InstructionMark im(this);
6324   simd_prefix_q(dst, dst, src, pre, no_mask_reg);
6325   emit_int8(opcode);
6326   emit_operand(dst, src);
6327 }
6328 
6329 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6330   int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6331   emit_int8(opcode);
6332   emit_int8((unsigned char)(0xC0 | encode));
6333 }
6334 
6335 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
6336   int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, true, AVX_128bit);
6337   emit_int8(opcode);
6338   emit_int8((unsigned char)(0xC0 | encode));
6339 }
6340 
6341 // Versions with no second source register (non-destructive source).
6342 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
6343   InstructionMark im(this);
6344   simd_prefix(dst, xnoreg, src, pre, opNoRegMask);
6345   emit_int8(opcode);
6346   emit_operand(dst, src);
6347 }
6348 
6349 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
6350   InstructionMark im(this);
6351   simd_prefix_q(dst, xnoreg, src, pre, opNoRegMask);
6352   emit_int8(opcode);
6353   emit_operand(dst, src);
6354 }
6355 
6356 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6357   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6358   emit_int8(opcode);
6359   emit_int8((unsigned char)(0xC0 | encode));
6360 }
6361 
6362 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
6363   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, true);
6364   emit_int8(opcode);
6365   emit_int8((unsigned char)(0xC0 | encode));
6366 }
6367 
6368 // 3-operands AVX instructions
6369 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, Address src,
6370                                VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
6371   InstructionMark im(this);
6372   vex_prefix(dst, nds, src, pre, vector_len, no_mask_reg, legacy_mode);
6373   emit_int8(opcode);
6374   emit_operand(dst, src);
6375 }
6376 
6377 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds,
6378                                  Address src, VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
6379   InstructionMark im(this);
6380   vex_prefix_q(dst, nds, src, pre, vector_len, no_mask_reg);
6381   emit_int8(opcode);
6382   emit_operand(dst, src);
6383 }
6384 
6385 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
6386                                VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
6387   int encode = vex_prefix_and_encode(dst, nds, src, pre, vector_len, VEX_OPCODE_0F, legacy_mode, no_mask_reg);
6388   emit_int8(opcode);
6389   emit_int8((unsigned char)(0xC0 | encode));
6390 }
6391 
6392 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
6393                                  VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
6394   int src_enc = src->encoding();
6395   int dst_enc = dst->encoding();
6396   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6397   int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, VEX_OPCODE_0F, true, vector_len, false, no_mask_reg);
6398   emit_int8(opcode);
6399   emit_int8((unsigned char)(0xC0 | encode));
6400 }
6401 
6402 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
6403   assert(VM_Version::supports_avx(), "");
6404   assert(!VM_Version::supports_evex(), "");
6405   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F, /* no_mask_reg */ false);

6406   emit_int8((unsigned char)0xC2);
6407   emit_int8((unsigned char)(0xC0 | encode));
6408   emit_int8((unsigned char)(0xF & cop));
6409 }
6410 
6411 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
6412   assert(VM_Version::supports_avx(), "");
6413   assert(!VM_Version::supports_evex(), "");
6414   int encode = vex_prefix_and_encode(dst, nds, src1, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* no_mask_reg */ false);


6415   emit_int8((unsigned char)0x4B);
6416   emit_int8((unsigned char)(0xC0 | encode));
6417   int src2_enc = src2->encoding();
6418   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
6419 }
6420 
6421 
6422 #ifndef _LP64
6423 
6424 void Assembler::incl(Register dst) {
6425   // Don't use it directly. Use MacroAssembler::incrementl() instead.
6426   emit_int8(0x40 | dst->encoding());
6427 }
6428 
6429 void Assembler::lea(Register dst, Address src) {
6430   leal(dst, src);
6431 }
6432 
6433 void Assembler::mov_literal32(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
6434   InstructionMark im(this);


6931 
6932 void Assembler::andq(Register dst, int32_t imm32) {
6933   (void) prefixq_and_encode(dst->encoding());
6934   emit_arith(0x81, 0xE0, dst, imm32);
6935 }
6936 
6937 void Assembler::andq(Register dst, Address src) {
6938   InstructionMark im(this);
6939   prefixq(src, dst);
6940   emit_int8(0x23);
6941   emit_operand(dst, src);
6942 }
6943 
6944 void Assembler::andq(Register dst, Register src) {
6945   (void) prefixq_and_encode(dst->encoding(), src->encoding());
6946   emit_arith(0x23, 0xC0, dst, src);
6947 }
6948 
6949 void Assembler::andnq(Register dst, Register src1, Register src2) {
6950   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6951   int encode = vex_prefix_0F38_and_encode_q_legacy(dst, src1, src2);

6952   emit_int8((unsigned char)0xF2);
6953   emit_int8((unsigned char)(0xC0 | encode));
6954 }
6955 
6956 void Assembler::andnq(Register dst, Register src1, Address src2) {
6957   InstructionMark im(this);
6958   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6959   vex_prefix_0F38_q_legacy(dst, src1, src2);


6960   emit_int8((unsigned char)0xF2);
6961   emit_operand(dst, src2);
6962 }
6963 
6964 void Assembler::bsfq(Register dst, Register src) {
6965   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6966   emit_int8(0x0F);
6967   emit_int8((unsigned char)0xBC);
6968   emit_int8((unsigned char)(0xC0 | encode));
6969 }
6970 
6971 void Assembler::bsrq(Register dst, Register src) {
6972   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6973   emit_int8(0x0F);
6974   emit_int8((unsigned char)0xBD);
6975   emit_int8((unsigned char)(0xC0 | encode));
6976 }
6977 
6978 void Assembler::bswapq(Register reg) {
6979   int encode = prefixq_and_encode(reg->encoding());
6980   emit_int8(0x0F);
6981   emit_int8((unsigned char)(0xC8 | encode));
6982 }
6983 
6984 void Assembler::blsiq(Register dst, Register src) {
6985   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6986   int encode = vex_prefix_0F38_and_encode_q_legacy(rbx, dst, src);

6987   emit_int8((unsigned char)0xF3);
6988   emit_int8((unsigned char)(0xC0 | encode));
6989 }
6990 
6991 void Assembler::blsiq(Register dst, Address src) {
6992   InstructionMark im(this);
6993   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6994   vex_prefix_0F38_q_legacy(rbx, dst, src);


6995   emit_int8((unsigned char)0xF3);
6996   emit_operand(rbx, src);
6997 }
6998 
6999 void Assembler::blsmskq(Register dst, Register src) {
7000   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7001   int encode = vex_prefix_0F38_and_encode_q_legacy(rdx, dst, src);

7002   emit_int8((unsigned char)0xF3);
7003   emit_int8((unsigned char)(0xC0 | encode));
7004 }
7005 
7006 void Assembler::blsmskq(Register dst, Address src) {
7007   InstructionMark im(this);
7008   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7009   vex_prefix_0F38_q_legacy(rdx, dst, src);


7010   emit_int8((unsigned char)0xF3);
7011   emit_operand(rdx, src);
7012 }
7013 
7014 void Assembler::blsrq(Register dst, Register src) {
7015   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7016   int encode = vex_prefix_0F38_and_encode_q_legacy(rcx, dst, src);

7017   emit_int8((unsigned char)0xF3);
7018   emit_int8((unsigned char)(0xC0 | encode));
7019 }
7020 
7021 void Assembler::blsrq(Register dst, Address src) {
7022   InstructionMark im(this);
7023   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7024   vex_prefix_0F38_q_legacy(rcx, dst, src);


7025   emit_int8((unsigned char)0xF3);
7026   emit_operand(rcx, src);
7027 }
7028 
7029 void Assembler::cdqq() {
7030   prefix(REX_W);
7031   emit_int8((unsigned char)0x99);
7032 }
7033 
7034 void Assembler::clflush(Address adr) {
7035   prefix(adr);
7036   emit_int8(0x0F);
7037   emit_int8((unsigned char)0xAE);
7038   emit_operand(rdi, adr);
7039 }
7040 
7041 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7042   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7043   emit_int8(0x0F);
7044   emit_int8(0x40 | cc);


7078   emit_arith(0x3B, 0xC0, dst, src);
7079 }
7080 
7081 void Assembler::cmpq(Register dst, Address  src) {
7082   InstructionMark im(this);
7083   prefixq(src, dst);
7084   emit_int8(0x3B);
7085   emit_operand(dst, src);
7086 }
7087 
7088 void Assembler::cmpxchgq(Register reg, Address adr) {
7089   InstructionMark im(this);
7090   prefixq(adr, reg);
7091   emit_int8(0x0F);
7092   emit_int8((unsigned char)0xB1);
7093   emit_operand(reg, adr);
7094 }
7095 
7096 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7097   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7098   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);

7099   emit_int8(0x2A);
7100   emit_int8((unsigned char)(0xC0 | encode));
7101 }
7102 
7103 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7104   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7105   if (VM_Version::supports_evex()) {
7106     _tuple_type = EVEX_T1S;
7107     _input_size_in_bits = EVEX_32bit;
7108   }
7109   InstructionMark im(this);
7110   simd_prefix_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);


7111   emit_int8(0x2A);
7112   emit_operand(dst, src);
7113 }
7114 
7115 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7116   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7117   if (VM_Version::supports_evex()) {
7118     _tuple_type = EVEX_T1S;
7119     _input_size_in_bits = EVEX_32bit;
7120   }
7121   InstructionMark im(this);
7122   simd_prefix_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);


7123   emit_int8(0x2A);
7124   emit_operand(dst, src);
7125 }
7126 
7127 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7128   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7129   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);

7130   emit_int8(0x2C);
7131   emit_int8((unsigned char)(0xC0 | encode));
7132 }
7133 
7134 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7135   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7136   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);

7137   emit_int8(0x2C);
7138   emit_int8((unsigned char)(0xC0 | encode));
7139 }
7140 
7141 void Assembler::decl(Register dst) {
7142   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7143   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7144   int encode = prefix_and_encode(dst->encoding());
7145   emit_int8((unsigned char)0xFF);
7146   emit_int8((unsigned char)(0xC8 | encode));
7147 }
7148 
7149 void Assembler::decq(Register dst) {
7150   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7151   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7152   int encode = prefixq_and_encode(dst->encoding());
7153   emit_int8((unsigned char)0xFF);
7154   emit_int8(0xC8 | encode);
7155 }
7156 


7299 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7300   InstructionMark im(this);
7301   prefix(src1);
7302   emit_int8((unsigned char)0x81);
7303   emit_operand(rax, src1, 4);
7304   emit_data((int)imm32, rspec, narrow_oop_operand);
7305 }
7306 
7307 void Assembler::lzcntq(Register dst, Register src) {
7308   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7309   emit_int8((unsigned char)0xF3);
7310   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7311   emit_int8(0x0F);
7312   emit_int8((unsigned char)0xBD);
7313   emit_int8((unsigned char)(0xC0 | encode));
7314 }
7315 
7316 void Assembler::movdq(XMMRegister dst, Register src) {
7317   // table D-1 says MMX/SSE2
7318   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7319   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);

7320   emit_int8(0x6E);
7321   emit_int8((unsigned char)(0xC0 | encode));
7322 }
7323 
7324 void Assembler::movdq(Register dst, XMMRegister src) {
7325   // table D-1 says MMX/SSE2
7326   NOT_LP64(assert(VM_Version::supports_sse2(), ""));

7327   // swap src/dst to get correct prefix
7328   int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
7329   emit_int8(0x7E);
7330   emit_int8((unsigned char)(0xC0 | encode));
7331 }
7332 
7333 void Assembler::movq(Register dst, Register src) {
7334   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7335   emit_int8((unsigned char)0x8B);
7336   emit_int8((unsigned char)(0xC0 | encode));
7337 }
7338 
7339 void Assembler::movq(Register dst, Address src) {
7340   InstructionMark im(this);
7341   prefixq(src, dst);
7342   emit_int8((unsigned char)0x8B);
7343   emit_operand(dst, src);
7344 }
7345 
7346 void Assembler::movq(Address dst, Register src) {
7347   InstructionMark im(this);
7348   prefixq(dst, src);


7441   emit_int8((unsigned char)0x0F);
7442   emit_int8((unsigned char)0xB7);
7443   emit_int8((unsigned char)(0xC0 | encode));
7444 }
7445 
7446 void Assembler::mulq(Address src) {
7447   InstructionMark im(this);
7448   prefixq(src);
7449   emit_int8((unsigned char)0xF7);
7450   emit_operand(rsp, src);
7451 }
7452 
7453 void Assembler::mulq(Register src) {
7454   int encode = prefixq_and_encode(src->encoding());
7455   emit_int8((unsigned char)0xF7);
7456   emit_int8((unsigned char)(0xE0 | encode));
7457 }
7458 
7459 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
7460   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7461   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38,
7462                                     /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
7463   emit_int8((unsigned char)0xF6);
7464   emit_int8((unsigned char)(0xC0 | encode));
7465 }
7466 
7467 void Assembler::negq(Register dst) {
7468   int encode = prefixq_and_encode(dst->encoding());
7469   emit_int8((unsigned char)0xF7);
7470   emit_int8((unsigned char)(0xD8 | encode));
7471 }
7472 
7473 void Assembler::notq(Register dst) {
7474   int encode = prefixq_and_encode(dst->encoding());
7475   emit_int8((unsigned char)0xF7);
7476   emit_int8((unsigned char)(0xD0 | encode));
7477 }
7478 
7479 void Assembler::orq(Address dst, int32_t imm32) {
7480   InstructionMark im(this);
7481   prefixq(dst);
7482   emit_int8((unsigned char)0x81);


7604     emit_int8((unsigned char)(0xD8 | encode));
7605     emit_int8(imm8);
7606   }
7607 }
7608 
7609 void Assembler::rorq(Register dst, int imm8) {
7610   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7611   int encode = prefixq_and_encode(dst->encoding());
7612   if (imm8 == 1) {
7613     emit_int8((unsigned char)0xD1);
7614     emit_int8((unsigned char)(0xC8 | encode));
7615   } else {
7616     emit_int8((unsigned char)0xC1);
7617     emit_int8((unsigned char)(0xc8 | encode));
7618     emit_int8(imm8);
7619   }
7620 }
7621 
7622 void Assembler::rorxq(Register dst, Register src, int imm8) {
7623   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7624   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A,
7625                                      /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
7626   emit_int8((unsigned char)0xF0);
7627   emit_int8((unsigned char)(0xC0 | encode));
7628   emit_int8(imm8);
7629 }
7630 
7631 void Assembler::sarq(Register dst, int imm8) {
7632   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7633   int encode = prefixq_and_encode(dst->encoding());
7634   if (imm8 == 1) {
7635     emit_int8((unsigned char)0xD1);
7636     emit_int8((unsigned char)(0xF8 | encode));
7637   } else {
7638     emit_int8((unsigned char)0xC1);
7639     emit_int8((unsigned char)(0xF8 | encode));
7640     emit_int8(imm8);
7641   }
7642 }
7643 
7644 void Assembler::sarq(Register dst) {
7645   int encode = prefixq_and_encode(dst->encoding());




 296   }
 297 }
 298 
 299 
 300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 301   assert(isByte(op1) && isByte(op2), "wrong opcode");
 302   emit_int8(op1);
 303   emit_int8(op2 | encode(dst) << 3 | encode(src));
 304 }
 305 
 306 
 307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
 308                                            int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
 309   int mod_idx = 0;
 310   // We will test if the displacement fits the compressed format and if so
 311   // apply the compression to the displacment iff the result is8bit.
 312   if (VM_Version::supports_evex() && is_evex_inst) {
 313     switch (cur_tuple_type) {
 314     case EVEX_FV:
 315       if ((cur_encoding & VEX_W) == VEX_W) {
 316         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 317       } else {
 318         mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 319       }
 320       break;
 321 
 322     case EVEX_HV:
 323       mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 324       break;
 325 
 326     case EVEX_FVM:
 327       break;
 328 
 329     case EVEX_T1S:
 330       switch (in_size_in_bits) {
 331       case EVEX_8bit:
 332         break;
 333 
 334       case EVEX_16bit:
 335         mod_idx = 1;
 336         break;


 377     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 378       int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
 379       if ((disp % disp_factor) == 0) {
 380         int new_disp = disp / disp_factor;
 381         if ((-0x80 <= new_disp && new_disp < 0x80)) {
 382           disp = new_disp;
 383         }
 384       } else {
 385         return false;
 386       }
 387     }
 388   }
 389   return (-0x80 <= disp && disp < 0x80);
 390 }
 391 
 392 
 393 bool Assembler::emit_compressed_disp_byte(int &disp) {
 394   int mod_idx = 0;
 395   // We will test if the displacement fits the compressed format and if so
 396   // apply the compression to the displacment iff the result is8bit.
 397   if (VM_Version::supports_evex() && (_attributes != NULL) && _attributes->is_evex_instruction()) {
 398     int evex_encoding = _attributes->get_evex_encoding();
 399     int tuple_type = _attributes->get_tuple_type();
 400     switch (tuple_type) {
 401     case EVEX_FV:
 402       if ((evex_encoding & VEX_W) == VEX_W) {
 403         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
 404       } else {
 405         mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 406       }
 407       break;
 408 
 409     case EVEX_HV:
 410       mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
 411       break;
 412 
 413     case EVEX_FVM:
 414       break;
 415 
 416     case EVEX_T1S:
 417       switch (_attributes->get_input_size()) {
 418       case EVEX_8bit:
 419         break;
 420 
 421       case EVEX_16bit:
 422         mod_idx = 1;
 423         break;
 424 
 425       case EVEX_32bit:
 426         mod_idx = 2;
 427         break;
 428 
 429       case EVEX_64bit:
 430         mod_idx = 3;
 431         break;
 432       }
 433       break;
 434 
 435     case EVEX_T1F:
 436     case EVEX_T2:
 437     case EVEX_T4:
 438       mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
 439       break;
 440 
 441     case EVEX_T8:
 442       break;
 443 
 444     case EVEX_HVM:
 445       break;
 446 
 447     case EVEX_QVM:
 448       break;
 449 
 450     case EVEX_OVM:
 451       break;
 452 
 453     case EVEX_M128:
 454       break;
 455 
 456     case EVEX_DUP:
 457       break;
 458 
 459     default:
 460       assert(0, "no valid evex tuple_table entry");
 461       break;
 462     }
 463 
 464     int vector_len = _attributes->get_vector_len();
 465     if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
 466       int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
 467       if ((disp % disp_factor) == 0) {
 468         int new_disp = disp / disp_factor;
 469         if (is8bit(new_disp)) {
 470           disp = new_disp;
 471         }
 472       } else {
 473         return false;
 474       }
 475     }
 476   }
 477   return is8bit(disp);
 478 }
 479 
 480 
 481 void Assembler::emit_operand(Register reg, Register base, Register index,
 482                              Address::ScaleFactor scale, int disp,
 483                              RelocationHolder const& rspec,
 484                              int rip_relative_correction) {
 485   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 486 


 577       // at the start of the instruction. That needs more correction here.
 578       // intptr_t disp = target - next_ip;
 579       assert(inst_mark() != NULL, "must be inside InstructionMark");
 580       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 581       int64_t adjusted = disp;
 582       // Do rip-rel adjustment for 64bit
 583       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 584       assert(is_simm32(adjusted),
 585              "must be 32bit offset (RIP relative address)");
 586       emit_data((int32_t) adjusted, rspec, disp32_operand);
 587 
 588     } else {
 589       // 32bit never did this, did everything as the rip-rel/disp code above
 590       // [disp] ABSOLUTE
 591       // [00 reg 100][00 100 101] disp32
 592       emit_int8(0x04 | regenc);
 593       emit_int8(0x25);
 594       emit_data(disp, rspec, disp32_operand);
 595     }
 596   }

 597 }
 598 
 599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 600                              Address::ScaleFactor scale, int disp,
 601                              RelocationHolder const& rspec) {
 602   if (UseAVX > 2) {
 603     int xreg_enc = reg->encoding();
 604     if (xreg_enc > 15) {
 605       XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
 606       emit_operand((Register)new_reg, base, index, scale, disp, rspec);
 607       return;
 608     }
 609   }
 610   emit_operand((Register)reg, base, index, scale, disp, rspec);
 611 }
 612 
 613 // Secret local extension to Assembler::WhichOperand:
 614 #define end_pc_operand (_WhichOperand_limit)
 615 
 616 address Assembler::locate_operand(address inst, WhichOperand which) {


 755       tail_size = 1;
 756     case 0x38: // ptest, pmovzxbw
 757       ip++; // skip opcode
 758       debug_only(has_disp32 = true); // has both kinds of operands!
 759       break;
 760 
 761     case 0x70: // pshufd r, r/a, #8
 762       debug_only(has_disp32 = true); // has both kinds of operands!
 763     case 0x73: // psrldq r, #8
 764       tail_size = 1;
 765       break;
 766 
 767     case 0x12: // movlps
 768     case 0x28: // movaps
 769     case 0x2E: // ucomiss
 770     case 0x2F: // comiss
 771     case 0x54: // andps
 772     case 0x55: // andnps
 773     case 0x56: // orps
 774     case 0x57: // xorps
 775     case 0x59: // mulpd
 776     case 0x6E: // movd
 777     case 0x7E: // movd
 778     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 779       debug_only(has_disp32 = true);
 780       break;
 781 
 782     case 0xAD: // shrd r, a, %cl
 783     case 0xAF: // imul r, a
 784     case 0xBE: // movsbl r, a (movsxb)
 785     case 0xBF: // movswl r, a (movsxw)
 786     case 0xB6: // movzbl r, a (movzxb)
 787     case 0xB7: // movzwl r, a (movzxw)
 788     case REP16(0x40): // cmovl cc, r, a
 789     case 0xB0: // cmpxchgb
 790     case 0xB1: // cmpxchg
 791     case 0xC1: // xaddl
 792     case 0xC7: // cmpxchg8
 793     case REP16(0x90): // setcc a
 794       debug_only(has_disp32 = true);
 795       // fall out of the switch to decode the address


1219   emit_int8(0x0F);
1220   emit_int8(0x1F);
1221   emit_int8((unsigned char)0x80);
1222                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1223   emit_int32(0);   // 32-bits offset (4 bytes)
1224 }
1225 
1226 void Assembler::addr_nop_8() {
1227   assert(UseAddressNop, "no CPU support");
1228   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1229   emit_int8(0x0F);
1230   emit_int8(0x1F);
1231   emit_int8((unsigned char)0x84);
1232                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1233   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1234   emit_int32(0);   // 32-bits offset (4 bytes)
1235 }
1236 
1237 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1238   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1239   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1240   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1241   emit_int8(0x58);
1242   emit_int8((unsigned char)(0xC0 | encode));

1243 }
1244 
1245 void Assembler::addsd(XMMRegister dst, Address src) {
1246   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1247   InstructionMark im(this);
1248   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1249   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1250   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1251   emit_int8(0x58);
1252   emit_operand(dst, src);

1253 }
1254 
1255 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1256   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1257   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1258   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1259   emit_int8(0x58);
1260   emit_int8((unsigned char)(0xC0 | encode));
1261 }
1262 
1263 void Assembler::addss(XMMRegister dst, Address src) {
1264   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1265   InstructionMark im(this);
1266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1267   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1268   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1269   emit_int8(0x58);
1270   emit_operand(dst, src);
1271 }
1272 
1273 void Assembler::aesdec(XMMRegister dst, Address src) {
1274   assert(VM_Version::supports_aes(), "");
1275   InstructionMark im(this);
1276   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1277   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1278   emit_int8((unsigned char)0xDE);
1279   emit_operand(dst, src);
1280 }
1281 
1282 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1283   assert(VM_Version::supports_aes(), "");
1284   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1285   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1286   emit_int8((unsigned char)0xDE);
1287   emit_int8(0xC0 | encode);
1288 }
1289 
1290 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1291   assert(VM_Version::supports_aes(), "");
1292   InstructionMark im(this);
1293   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1294   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1295   emit_int8((unsigned char)0xDF);
1296   emit_operand(dst, src);
1297 }
1298 
1299 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1300   assert(VM_Version::supports_aes(), "");
1301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1302   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1303   emit_int8((unsigned char)0xDF);
1304   emit_int8((unsigned char)(0xC0 | encode));
1305 }
1306 
1307 void Assembler::aesenc(XMMRegister dst, Address src) {
1308   assert(VM_Version::supports_aes(), "");
1309   InstructionMark im(this);
1310   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1311   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1312   emit_int8((unsigned char)0xDC);
1313   emit_operand(dst, src);
1314 }
1315 
1316 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1317   assert(VM_Version::supports_aes(), "");
1318   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1319   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1320   emit_int8((unsigned char)0xDC);
1321   emit_int8(0xC0 | encode);
1322 }
1323 
1324 void Assembler::aesenclast(XMMRegister dst, Address src) {
1325   assert(VM_Version::supports_aes(), "");
1326   InstructionMark im(this);
1327   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1328   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1329   emit_int8((unsigned char)0xDD);
1330   emit_operand(dst, src);
1331 }
1332 
1333 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1334   assert(VM_Version::supports_aes(), "");
1335   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1336   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1337   emit_int8((unsigned char)0xDD);
1338   emit_int8((unsigned char)(0xC0 | encode));
1339 }
1340 
1341 void Assembler::andl(Address dst, int32_t imm32) {
1342   InstructionMark im(this);
1343   prefix(dst);
1344   emit_int8((unsigned char)0x81);
1345   emit_operand(rsp, dst, 4);
1346   emit_int32(imm32);
1347 }
1348 
1349 void Assembler::andl(Register dst, int32_t imm32) {
1350   prefix(dst);
1351   emit_arith(0x81, 0xE0, dst, imm32);
1352 }
1353 
1354 void Assembler::andl(Register dst, Address src) {
1355   InstructionMark im(this);
1356   prefix(src, dst);
1357   emit_int8(0x23);
1358   emit_operand(dst, src);
1359 }
1360 
1361 void Assembler::andl(Register dst, Register src) {
1362   (void) prefix_and_encode(dst->encoding(), src->encoding());
1363   emit_arith(0x23, 0xC0, dst, src);
1364 }
1365 
1366 void Assembler::andnl(Register dst, Register src1, Register src2) {
1367   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1368   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1369   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1370   emit_int8((unsigned char)0xF2);
1371   emit_int8((unsigned char)(0xC0 | encode));
1372 }
1373 
1374 void Assembler::andnl(Register dst, Register src1, Address src2) {

1375   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1376   InstructionMark im(this);
1377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1378   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1379   emit_int8((unsigned char)0xF2);
1380   emit_operand(dst, src2);
1381 }
1382 
1383 void Assembler::bsfl(Register dst, Register src) {
1384   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1385   emit_int8(0x0F);
1386   emit_int8((unsigned char)0xBC);
1387   emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389 
1390 void Assembler::bsrl(Register dst, Register src) {
1391   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1392   emit_int8(0x0F);
1393   emit_int8((unsigned char)0xBD);
1394   emit_int8((unsigned char)(0xC0 | encode));
1395 }
1396 
1397 void Assembler::bswapl(Register reg) { // bswap
1398   int encode = prefix_and_encode(reg->encoding());
1399   emit_int8(0x0F);
1400   emit_int8((unsigned char)(0xC8 | encode));
1401 }
1402 
1403 void Assembler::blsil(Register dst, Register src) {
1404   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1405   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1406   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1407   emit_int8((unsigned char)0xF3);
1408   emit_int8((unsigned char)(0xC0 | encode));
1409 }
1410 
1411 void Assembler::blsil(Register dst, Address src) {

1412   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1413   InstructionMark im(this);
1414   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1415   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1416   emit_int8((unsigned char)0xF3);
1417   emit_operand(rbx, src);
1418 }
1419 
1420 void Assembler::blsmskl(Register dst, Register src) {
1421   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1422   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1423   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1424   emit_int8((unsigned char)0xF3);
1425   emit_int8((unsigned char)(0xC0 | encode));
1426 }
1427 
1428 void Assembler::blsmskl(Register dst, Address src) {

1429   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1430   InstructionMark im(this);
1431   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1432   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1433   emit_int8((unsigned char)0xF3);
1434   emit_operand(rdx, src);
1435 }
1436 
1437 void Assembler::blsrl(Register dst, Register src) {
1438   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1439   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1440   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1441   emit_int8((unsigned char)0xF3);
1442   emit_int8((unsigned char)(0xC0 | encode));
1443 }
1444 
1445 void Assembler::blsrl(Register dst, Address src) {

1446   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1447   InstructionMark im(this);
1448   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1449   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1450   emit_int8((unsigned char)0xF3);
1451   emit_operand(rcx, src);
1452 }
1453 
1454 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1455   // suspect disp32 is always good
1456   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1457 
1458   if (L.is_bound()) {
1459     const int long_size = 5;
1460     int offs = (int)( target(L) - pc() );
1461     assert(offs <= 0, "assembler error");
1462     InstructionMark im(this);
1463     // 1110 1000 #32-bit disp
1464     emit_int8((unsigned char)0xE8);
1465     emit_data(offs - long_size, rtype, operand);
1466   } else {
1467     InstructionMark im(this);
1468     // 1110 1000 #32-bit disp
1469     L.add_patch_at(code(), locator());


1576   emit_int8(0x0F);
1577   emit_int8((unsigned char)0xB1);
1578   emit_operand(reg, adr);
1579 }
1580 
1581 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1582 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1583 // The ZF is set if the compared values were equal, and cleared otherwise.
1584 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1585   InstructionMark im(this);
1586   prefix(adr, reg, true);
1587   emit_int8(0x0F);
1588   emit_int8((unsigned char)0xB0);
1589   emit_operand(reg, adr);
1590 }
1591 
1592 void Assembler::comisd(XMMRegister dst, Address src) {
1593   // NOTE: dbx seems to decode this as comiss even though the
1594   // 0x66 is there. Strangly ucomisd comes out correct
1595   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1596   InstructionMark im(this);
1597   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1598   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1599   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1600   emit_int8(0x2F);
1601   emit_operand(dst, src);

1602 }
1603 
1604 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1605   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1606   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1607   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1608   emit_int8(0x2F);
1609   emit_int8((unsigned char)(0xC0 | encode));

1610 }
1611 
1612 void Assembler::comiss(XMMRegister dst, Address src) {




1613   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1614   InstructionMark im(this);
1615   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1616   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1617   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1618   emit_int8(0x2F);
1619   emit_operand(dst, src);
1620 }
1621 
1622 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1623   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1624   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1625   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1626   emit_int8(0x2F);
1627   emit_int8((unsigned char)(0xC0 | encode));
1628 }
1629 
1630 void Assembler::cpuid() {
1631   emit_int8(0x0F);
1632   emit_int8((unsigned char)0xA2);
1633 }
1634 
1635 // Opcode / Instruction                      Op /  En  64 - Bit Mode     Compat / Leg Mode Description                  Implemented
1636 // F2 0F 38 F0 / r       CRC32 r32, r / m8   RM        Valid             Valid             Accumulate CRC32 on r / m8.  v
1637 // F2 REX 0F 38 F0 / r   CRC32 r32, r / m8*  RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1638 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8   RM        Valid             N.E.              Accumulate CRC32 on r / m8.  -
1639 //
1640 // F2 0F 38 F1 / r       CRC32 r32, r / m16  RM        Valid             Valid             Accumulate CRC32 on r / m16. v
1641 //
1642 // F2 0F 38 F1 / r       CRC32 r32, r / m32  RM        Valid             Valid             Accumulate CRC32 on r / m32. v
1643 //
1644 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64  RM        Valid             N.E.              Accumulate CRC32 on r / m64. v
1645 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1646   assert(VM_Version::supports_sse4_2(), "");
1647   int8_t w = 0x01;


1696   case 2:
1697   case 4:
1698     break;
1699   LP64_ONLY(case 8:)
1700     // This instruction is not valid in 32 bits
1701     p = REX_W;
1702     break;
1703   default:
1704     assert(0, "Unsupported value for a sizeInBytes argument");
1705     break;
1706   }
1707   LP64_ONLY(prefix(crc, adr, p);)
1708   emit_int8((int8_t)0x0F);
1709   emit_int8(0x38);
1710   emit_int8((int8_t)(0xF0 | w));
1711   emit_operand(crc, adr);
1712 }
1713 
1714 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1716   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1717   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1718   emit_int8((unsigned char)0xE6);
1719   emit_int8((unsigned char)(0xC0 | encode));
1720 }
1721 
1722 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1724   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1725   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1726   emit_int8(0x5B);
1727   emit_int8((unsigned char)(0xC0 | encode));
1728 }
1729 
1730 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1731   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1732   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1733   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1734   emit_int8(0x5A);
1735   emit_int8((unsigned char)(0xC0 | encode));

1736 }
1737 
1738 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1739   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740   InstructionMark im(this);
1741   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1742   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1743   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1744   emit_int8(0x5A);
1745   emit_operand(dst, src);

1746 }
1747 
1748 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1750   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1751   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1752   emit_int8(0x2A);
1753   emit_int8((unsigned char)(0xC0 | encode));
1754 }
1755 
1756 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1757   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1758   InstructionMark im(this);
1759   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1760   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1761   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1762   emit_int8(0x2A);
1763   emit_operand(dst, src);

1764 }
1765 
1766 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1767   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1768   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1769   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1770   emit_int8(0x2A);
1771   emit_int8((unsigned char)(0xC0 | encode));
1772 }
1773 
1774 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {




1775   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1776   InstructionMark im(this);
1777   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1778   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1779   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1780   emit_int8(0x2A);
1781   emit_operand(dst, src);
1782 }
1783 
1784 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1785   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1786   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1787   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1788   emit_int8(0x2A);
1789   emit_int8((unsigned char)(0xC0 | encode));
1790 }
1791 
1792 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1793   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1794   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1795   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796   emit_int8(0x5A);
1797   emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799 
1800 void Assembler::cvtss2sd(XMMRegister dst, Address src) {




1801   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1802   InstructionMark im(this);
1803   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1804   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806   emit_int8(0x5A);
1807   emit_operand(dst, src);
1808 }
1809 
1810 
1811 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1813   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1814   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1815   emit_int8(0x2C);
1816   emit_int8((unsigned char)(0xC0 | encode));
1817 }
1818 
1819 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1820   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1821   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1822   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1823   emit_int8(0x2C);
1824   emit_int8((unsigned char)(0xC0 | encode));
1825 }
1826 
1827 void Assembler::decl(Address dst) {
1828   // Don't use it directly. Use MacroAssembler::decrement() instead.
1829   InstructionMark im(this);
1830   prefix(dst);
1831   emit_int8((unsigned char)0xFF);
1832   emit_operand(rcx, dst);
1833 }
1834 
1835 void Assembler::divsd(XMMRegister dst, Address src) {
1836   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1837   InstructionMark im(this);
1838   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1839   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1840   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841   emit_int8(0x5E);
1842   emit_operand(dst, src);

1843 }
1844 
1845 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1846   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1847   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1848   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1849   emit_int8(0x5E);
1850   emit_int8((unsigned char)(0xC0 | encode));

1851 }
1852 
1853 void Assembler::divss(XMMRegister dst, Address src) {




1854   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1855   InstructionMark im(this);
1856   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1857   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1858   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1859   emit_int8(0x5E);
1860   emit_operand(dst, src);
1861 }
1862 
1863 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1864   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1865   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1866   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1867   emit_int8(0x5E);
1868   emit_int8((unsigned char)(0xC0 | encode));
1869 }
1870 
1871 void Assembler::emms() {
1872   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1873   emit_int8(0x0F);
1874   emit_int8(0x77);
1875 }
1876 
1877 void Assembler::hlt() {
1878   emit_int8((unsigned char)0xF4);
1879 }
1880 
1881 void Assembler::idivl(Register src) {
1882   int encode = prefix_and_encode(src->encoding());
1883   emit_int8((unsigned char)0xF7);
1884   emit_int8((unsigned char)(0xF8 | encode));
1885 }
1886 
1887 void Assembler::divl(Register src) { // Unsigned
1888   int encode = prefix_and_encode(src->encoding());


2094   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2095   emit_int8(0x0F);
2096   emit_int8((unsigned char)0xBD);
2097   emit_int8((unsigned char)(0xC0 | encode));
2098 }
2099 
2100 // Emit mfence instruction
2101 void Assembler::mfence() {
2102   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2103   emit_int8(0x0F);
2104   emit_int8((unsigned char)0xAE);
2105   emit_int8((unsigned char)0xF0);
2106 }
2107 
2108 void Assembler::mov(Register dst, Register src) {
2109   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2110 }
2111 
2112 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2113   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2114   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2115   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2116   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);



2117   emit_int8(0x28);
2118   emit_int8((unsigned char)(0xC0 | encode));





2119 }
2120 
2121 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2122   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2123   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2124   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2125   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2126   emit_int8(0x28);
2127   emit_int8((unsigned char)(0xC0 | encode));



2128 }
2129 
2130 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2131   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2132   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2133   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2134   emit_int8(0x16);
2135   emit_int8((unsigned char)(0xC0 | encode));
2136 }
2137 
2138 void Assembler::movb(Register dst, Address src) {
2139   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2140   InstructionMark im(this);
2141   prefix(src, dst, true);
2142   emit_int8((unsigned char)0x8A);
2143   emit_operand(dst, src);
2144 }
2145 
2146 void Assembler::movddup(XMMRegister dst, XMMRegister src) {

2147   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2148   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
2149   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2150   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2151   emit_int8(0x12);
2152   emit_int8(0xC0 | encode);

2153 }
2154 
2155 void Assembler::kmovql(KRegister dst, KRegister src) {
2156   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2157   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2158   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2159   emit_int8((unsigned char)0x90);
2160   emit_int8((unsigned char)(0xC0 | encode));
2161 }
2162 
2163 void Assembler::kmovql(KRegister dst, Address src) {
2164   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2165   InstructionMark im(this);
2166   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2167   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);

2168   emit_int8((unsigned char)0x90);
2169   emit_operand((Register)dst, src);
2170 }
2171 
2172 void Assembler::kmovql(Address dst, KRegister src) {
2173   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2174   InstructionMark im(this);
2175   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2176   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);

2177   emit_int8((unsigned char)0x90);
2178   emit_operand((Register)src, dst);
2179 }
2180 
2181 void Assembler::kmovql(KRegister dst, Register src) {
2182   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2183   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2184   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_bw, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2185   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
2186   emit_int8((unsigned char)0x92);
2187   emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189 
2190 void Assembler::kmovdl(KRegister dst, Register src) {
2191   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2192   VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2193   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2194   int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
2195   emit_int8((unsigned char)0x92);
2196   emit_int8((unsigned char)(0xC0 | encode));
2197 }
2198 
2199 void Assembler::kmovwl(KRegister dst, Register src) {
2200   NOT_LP64(assert(VM_Version::supports_evex(), ""));
2201   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2202   int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2203   emit_int8((unsigned char)0x92);
2204   emit_int8((unsigned char)(0xC0 | encode));
2205 }
2206 
2207 void Assembler::movb(Address dst, int imm8) {
2208   InstructionMark im(this);
2209    prefix(dst);
2210   emit_int8((unsigned char)0xC6);
2211   emit_operand(rax, dst, 1);
2212   emit_int8(imm8);
2213 }
2214 
2215 
2216 void Assembler::movb(Address dst, Register src) {
2217   assert(src->has_byte_register(), "must have byte register");
2218   InstructionMark im(this);
2219   prefix(dst, src, true);
2220   emit_int8((unsigned char)0x88);
2221   emit_operand(src, dst);
2222 }
2223 
2224 void Assembler::movdl(XMMRegister dst, Register src) {
2225   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2226   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2227   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2228   emit_int8(0x6E);
2229   emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231 
2232 void Assembler::movdl(Register dst, XMMRegister src) {
2233   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2234   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2235   // swap src/dst to get correct prefix
2236   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2237   emit_int8(0x7E);
2238   emit_int8((unsigned char)(0xC0 | encode));
2239 }
2240 
2241 void Assembler::movdl(XMMRegister dst, Address src) {
2242   NOT_LP64(assert(VM_Version::supports_sse2(), ""));




2243   InstructionMark im(this);
2244   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2245   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2246   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2247   emit_int8(0x6E);
2248   emit_operand(dst, src);
2249 }
2250 
2251 void Assembler::movdl(Address dst, XMMRegister src) {
2252   NOT_LP64(assert(VM_Version::supports_sse2(), ""));




2253   InstructionMark im(this);
2254   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2255   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2256   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2257   emit_int8(0x7E);
2258   emit_operand(src, dst);
2259 }
2260 
2261 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {

2262   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2263   int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2264   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2265   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2266   emit_int8(0x6F);
2267   emit_int8((unsigned char)(0xC0 | encode));
2268 }
2269 
2270 void Assembler::movdqa(XMMRegister dst, Address src) {

2271   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2272   InstructionMark im(this);
2273   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2274   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2275   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2276   emit_int8(0x6F);
2277   emit_operand(dst, src);
2278 }
2279 
2280 void Assembler::movdqu(XMMRegister dst, Address src) {

2281   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2282   InstructionMark im(this);
2283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2284   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2285   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2286   emit_int8(0x6F);
2287   emit_operand(dst, src);
2288 }
2289 
2290 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {

2291   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2292   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2293   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2294   emit_int8(0x6F);
2295   emit_int8((unsigned char)(0xC0 | encode));
2296 }
2297 
2298 void Assembler::movdqu(Address dst, XMMRegister src) {

2299   NOT_LP64(assert(VM_Version::supports_sse2(), ""));



2300   InstructionMark im(this);
2301   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2302   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2303   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2304   emit_int8(0x7F);
2305   emit_operand(src, dst);
2306 }
2307 
2308 // Move Unaligned 256bit Vector
2309 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {

2310   assert(UseAVX > 0, "");
2311   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2312   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2313   emit_int8(0x6F);
2314   emit_int8((unsigned char)(0xC0 | encode));
2315 }
2316 
2317 void Assembler::vmovdqu(XMMRegister dst, Address src) {

2318   assert(UseAVX > 0, "");



2319   InstructionMark im(this);
2320   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2321   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2322   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2323   emit_int8(0x6F);
2324   emit_operand(dst, src);
2325 }
2326 
2327 void Assembler::vmovdqu(Address dst, XMMRegister src) {

2328   assert(UseAVX > 0, "");



2329   InstructionMark im(this);
2330   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2331   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2332   // swap src<->dst for encoding
2333   assert(src != xnoreg, "sanity");
2334   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2335   emit_int8(0x7F);
2336   emit_operand(src, dst);
2337 }
2338 
2339 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2340 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2341   assert(VM_Version::supports_evex(), "");
2342   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2343   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);



2344   emit_int8(0x6F);
2345   emit_int8((unsigned char)(0xC0 | encode));
2346 }
2347 
2348 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2349   assert(VM_Version::supports_evex(), "");

2350   InstructionMark im(this);
2351   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2352   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2353   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);

2354   emit_int8(0x6F);
2355   emit_operand(dst, src);
2356 }
2357 
2358 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2359   assert(VM_Version::supports_evex(), "");


2360   assert(src != xnoreg, "sanity");
2361   InstructionMark im(this);
2362   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2363   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2364   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);

2365   emit_int8(0x7F);
2366   emit_operand(src, dst);
2367 }
2368 
2369 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2370   assert(VM_Version::supports_evex(), "");
2371   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2372   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);



2373   emit_int8(0x6F);
2374   emit_int8((unsigned char)(0xC0 | encode));
2375 }
2376 
2377 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2378   assert(VM_Version::supports_evex(), "");

2379   InstructionMark im(this);
2380   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2381   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2382   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2383   emit_int8(0x6F);
2384   emit_operand(dst, src);
2385 }
2386 
2387 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2388   assert(VM_Version::supports_evex(), "");


2389   assert(src != xnoreg, "sanity");
2390   InstructionMark im(this);
2391   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2392   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2393   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2394   emit_int8(0x7F);
2395   emit_operand(src, dst);
2396 }
2397 
2398 // Uses zero extension on 64bit
2399 
2400 void Assembler::movl(Register dst, int32_t imm32) {
2401   int encode = prefix_and_encode(dst->encoding());
2402   emit_int8((unsigned char)(0xB8 | encode));
2403   emit_int32(imm32);
2404 }
2405 
2406 void Assembler::movl(Register dst, Register src) {
2407   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2408   emit_int8((unsigned char)0x8B);
2409   emit_int8((unsigned char)(0xC0 | encode));
2410 }
2411 
2412 void Assembler::movl(Register dst, Address src) {
2413   InstructionMark im(this);


2419 void Assembler::movl(Address dst, int32_t imm32) {
2420   InstructionMark im(this);
2421   prefix(dst);
2422   emit_int8((unsigned char)0xC7);
2423   emit_operand(rax, dst, 4);
2424   emit_int32(imm32);
2425 }
2426 
2427 void Assembler::movl(Address dst, Register src) {
2428   InstructionMark im(this);
2429   prefix(dst, src);
2430   emit_int8((unsigned char)0x89);
2431   emit_operand(src, dst);
2432 }
2433 
2434 // New cpus require to use movsd and movss to avoid partial register stall
2435 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2436 // The selection is done in MacroAssembler::movdbl() and movflt().
2437 void Assembler::movlpd(XMMRegister dst, Address src) {
2438   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2439   InstructionMark im(this);
2440   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2441   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2442   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2443   emit_int8(0x12);
2444   emit_operand(dst, src);

2445 }
2446 
2447 void Assembler::movq( MMXRegister dst, Address src ) {
2448   assert( VM_Version::supports_mmx(), "" );
2449   emit_int8(0x0F);
2450   emit_int8(0x6F);
2451   emit_operand(dst, src);
2452 }
2453 
2454 void Assembler::movq( Address dst, MMXRegister src ) {
2455   assert( VM_Version::supports_mmx(), "" );
2456   emit_int8(0x0F);
2457   emit_int8(0x7F);
2458   // workaround gcc (3.2.1-7a) bug
2459   // In that version of gcc with only an emit_operand(MMX, Address)
2460   // gcc will tail jump and try and reverse the parameters completely
2461   // obliterating dst in the process. By having a version available
2462   // that doesn't need to swap the args at the tail jump the bug is
2463   // avoided.
2464   emit_operand(dst, src);
2465 }
2466 
2467 void Assembler::movq(XMMRegister dst, Address src) {
2468   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2469   InstructionMark im(this);
2470   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2471   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2472   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);




2473   emit_int8(0x7E);
2474   emit_operand(dst, src);
2475 }
2476 
2477 void Assembler::movq(Address dst, XMMRegister src) {
2478   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2479   InstructionMark im(this);
2480   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2481   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2482   simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);





2483   emit_int8((unsigned char)0xD6);
2484   emit_operand(src, dst);
2485 }
2486 
2487 void Assembler::movsbl(Register dst, Address src) { // movsxb
2488   InstructionMark im(this);
2489   prefix(src, dst);
2490   emit_int8(0x0F);
2491   emit_int8((unsigned char)0xBE);
2492   emit_operand(dst, src);
2493 }
2494 
2495 void Assembler::movsbl(Register dst, Register src) { // movsxb
2496   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2497   int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2498   emit_int8(0x0F);
2499   emit_int8((unsigned char)0xBE);
2500   emit_int8((unsigned char)(0xC0 | encode));
2501 }
2502 
2503 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2504   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2505   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2506   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2507   emit_int8(0x10);
2508   emit_int8((unsigned char)(0xC0 | encode));

2509 }
2510 
2511 void Assembler::movsd(XMMRegister dst, Address src) {
2512   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2513   InstructionMark im(this);
2514   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2515   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2516   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2517   emit_int8(0x10);
2518   emit_operand(dst, src);

2519 }
2520 
2521 void Assembler::movsd(Address dst, XMMRegister src) {
2522   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2523   InstructionMark im(this);
2524   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2526   simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);




2527   emit_int8(0x11);
2528   emit_operand(src, dst);
2529 }
2530 
2531 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2532   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2533   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2534   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2535   emit_int8(0x10);
2536   emit_int8((unsigned char)(0xC0 | encode));
2537 }
2538 
2539 void Assembler::movss(XMMRegister dst, Address src) {
2540   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2541   InstructionMark im(this);
2542   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2543   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2544   simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2545   emit_int8(0x10);
2546   emit_operand(dst, src);
2547 }
2548 
2549 void Assembler::movss(Address dst, XMMRegister src) {
2550   NOT_LP64(assert(VM_Version::supports_sse(), ""));




2551   InstructionMark im(this);
2552   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2553   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2554   simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2555   emit_int8(0x11);
2556   emit_operand(src, dst);
2557 }
2558 
2559 void Assembler::movswl(Register dst, Address src) { // movsxw
2560   InstructionMark im(this);
2561   prefix(src, dst);
2562   emit_int8(0x0F);
2563   emit_int8((unsigned char)0xBF);
2564   emit_operand(dst, src);
2565 }
2566 
2567 void Assembler::movswl(Register dst, Register src) { // movsxw
2568   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2569   emit_int8(0x0F);
2570   emit_int8((unsigned char)0xBF);
2571   emit_int8((unsigned char)(0xC0 | encode));
2572 }
2573 
2574 void Assembler::movw(Address dst, int imm16) {


2626   emit_int8(0x0F);
2627   emit_int8((unsigned char)0xB7);
2628   emit_int8(0xC0 | encode);
2629 }
2630 
2631 void Assembler::mull(Address src) {
2632   InstructionMark im(this);
2633   prefix(src);
2634   emit_int8((unsigned char)0xF7);
2635   emit_operand(rsp, src);
2636 }
2637 
2638 void Assembler::mull(Register src) {
2639   int encode = prefix_and_encode(src->encoding());
2640   emit_int8((unsigned char)0xF7);
2641   emit_int8((unsigned char)(0xE0 | encode));
2642 }
2643 
2644 void Assembler::mulsd(XMMRegister dst, Address src) {
2645   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2646   InstructionMark im(this);
2647   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2648   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2649   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2650   emit_int8(0x59);
2651   emit_operand(dst, src);

2652 }
2653 
2654 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2655   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2656   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2657   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2658   emit_int8(0x59);
2659   emit_int8((unsigned char)(0xC0 | encode));

2660 }
2661 
2662 void Assembler::mulss(XMMRegister dst, Address src) {
2663   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2664   InstructionMark im(this);
2665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2666   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2667   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2668   emit_int8(0x59);
2669   emit_operand(dst, src);
2670 }
2671 
2672 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2673   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2674   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2675   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2676   emit_int8(0x59);
2677   emit_int8((unsigned char)(0xC0 | encode));
2678 }
2679 
2680 void Assembler::negl(Register dst) {
2681   int encode = prefix_and_encode(dst->encoding());
2682   emit_int8((unsigned char)0xF7);
2683   emit_int8((unsigned char)(0xD8 | encode));
2684 }
2685 
2686 void Assembler::nop(int i) {
2687 #ifdef ASSERT
2688   assert(i > 0, " ");
2689   // The fancy nops aren't currently recognized by debuggers making it a
2690   // pain to disassemble code while debugging. If asserts are on clearly
2691   // speed is not an issue so simply use the single byte traditional nop
2692   // to do alignment.
2693 
2694   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2695   return;
2696 
2697 #endif // ASSERT


2958   prefix(src, dst);
2959   emit_int8(0x0B);
2960   emit_operand(dst, src);
2961 }
2962 
2963 void Assembler::orl(Register dst, Register src) {
2964   (void) prefix_and_encode(dst->encoding(), src->encoding());
2965   emit_arith(0x0B, 0xC0, dst, src);
2966 }
2967 
2968 void Assembler::orl(Address dst, Register src) {
2969   InstructionMark im(this);
2970   prefix(dst, src);
2971   emit_int8(0x09);
2972   emit_operand(src, dst);
2973 }
2974 
2975 void Assembler::packuswb(XMMRegister dst, Address src) {
2976   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2977   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2978   InstructionMark im(this);
2979   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2980   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
2981   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2982   emit_int8(0x67);
2983   emit_operand(dst, src);
2984 }
2985 
2986 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2989   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2990   emit_int8(0x67);
2991   emit_int8((unsigned char)(0xC0 | encode));
2992 }
2993 
2994 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2995   assert(UseAVX > 0, "some form of AVX must be enabled");
2996   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2997   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
2998   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2999   emit_int8(0x67);
3000   emit_int8((unsigned char)(0xC0 | encode));
3001 }
3002 
3003 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {

3004   assert(VM_Version::supports_avx2(), "");
3005   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3006   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3007   emit_int8(0x00);
3008   emit_int8(0xC0 | encode);
3009   emit_int8(imm8);
3010 }
3011 
3012 void Assembler::pause() {
3013   emit_int8((unsigned char)0xF3);
3014   emit_int8((unsigned char)0x90);
3015 }
3016 
3017 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3018   assert(VM_Version::supports_sse4_2(), "");
3019   InstructionMark im(this);
3020   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3021   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3022   emit_int8(0x61);
3023   emit_operand(dst, src);
3024   emit_int8(imm8);
3025 }
3026 
3027 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3028   assert(VM_Version::supports_sse4_2(), "");
3029   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3030   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3031   emit_int8(0x61);
3032   emit_int8((unsigned char)(0xC0 | encode));
3033   emit_int8(imm8);
3034 }
3035 
3036 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3037   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3038   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3039   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3040   emit_int8(0x75);
3041   emit_int8((unsigned char)(0xC0 | encode));
3042 }
3043 
3044 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3045   assert(VM_Version::supports_avx(), "");
3046   assert(!VM_Version::supports_evex(), "");
3047   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3048   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3049   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3050   emit_int8(0x75);
3051   emit_int8((unsigned char)(0xC0 | encode));
3052 }
3053 
3054 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3055   assert(VM_Version::supports_sse2(), "");
3056   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3057   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3058   emit_int8((unsigned char)0xD7);
3059   emit_int8((unsigned char)(0xC0 | encode));
3060 }
3061 
3062 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3063   assert(VM_Version::supports_avx2(), "");
3064   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3065   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

3066   emit_int8((unsigned char)0xD7);
3067   emit_int8((unsigned char)(0xC0 | encode));
3068 }
3069 
3070 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3071   assert(VM_Version::supports_sse4_1(), "");
3072   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3073   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3074   emit_int8(0x16);
3075   emit_int8((unsigned char)(0xC0 | encode));
3076   emit_int8(imm8);
3077 }
3078 
3079 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3080   assert(VM_Version::supports_sse4_1(), "");
3081   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3082   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3083   emit_int8(0x16);
3084   emit_int8((unsigned char)(0xC0 | encode));
3085   emit_int8(imm8);
3086 }
3087 
3088 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3089   assert(VM_Version::supports_sse2(), "");
3090   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3091   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3092   emit_int8((unsigned char)0xC5);
3093   emit_int8((unsigned char)(0xC0 | encode));
3094   emit_int8(imm8);
3095 }
3096 
3097 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3098   assert(VM_Version::supports_sse4_1(), "");
3099   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3100   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3101   emit_int8(0x22);
3102   emit_int8((unsigned char)(0xC0 | encode));
3103   emit_int8(imm8);
3104 }
3105 
3106 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3107   assert(VM_Version::supports_sse4_1(), "");
3108   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3109   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3110   emit_int8(0x22);
3111   emit_int8((unsigned char)(0xC0 | encode));
3112   emit_int8(imm8);
3113 }
3114 
3115 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3116   assert(VM_Version::supports_sse2(), "");
3117   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3118   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3119   emit_int8((unsigned char)0xC4);
3120   emit_int8((unsigned char)(0xC0 | encode));
3121   emit_int8(imm8);
3122 }
3123 
3124 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3125   assert(VM_Version::supports_sse4_1(), "");



3126   InstructionMark im(this);
3127   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3128   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3129   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3130   emit_int8(0x30);
3131   emit_operand(dst, src);
3132 }
3133 
3134 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3135   assert(VM_Version::supports_sse4_1(), "");
3136   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3137   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3138   emit_int8(0x30);
3139   emit_int8((unsigned char)(0xC0 | encode));
3140 }
3141 
3142 void Assembler::vpmovzxbw(XMMRegister dst, Address src) {
3143   assert(VM_Version::supports_avx(), "");
3144   InstructionMark im(this);

3145   assert(dst != xnoreg, "sanity");
3146   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3147   attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3148   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3149   emit_int8(0x30);
3150   emit_operand(dst, src);
3151 }
3152 
3153 // generic
3154 void Assembler::pop(Register dst) {
3155   int encode = prefix_and_encode(dst->encoding());
3156   emit_int8(0x58 | encode);
3157 }
3158 
3159 void Assembler::popcntl(Register dst, Address src) {
3160   assert(VM_Version::supports_popcnt(), "must support");
3161   InstructionMark im(this);
3162   emit_int8((unsigned char)0xF3);
3163   prefix(src, dst);
3164   emit_int8(0x0F);
3165   emit_int8((unsigned char)0xB8);
3166   emit_operand(dst, src);
3167 }
3168 


3231   InstructionMark im(this);
3232   prefetch_prefix(src);
3233   emit_int8(0x18);
3234   emit_operand(rbx, src); // 3, src
3235 }
3236 
3237 void Assembler::prefetchw(Address src) {
3238   assert(VM_Version::supports_3dnow_prefetch(), "must support");
3239   InstructionMark im(this);
3240   prefetch_prefix(src);
3241   emit_int8(0x0D);
3242   emit_operand(rcx, src); // 1, src
3243 }
3244 
3245 void Assembler::prefix(Prefix p) {
3246   emit_int8(p);
3247 }
3248 
3249 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3250   assert(VM_Version::supports_ssse3(), "");
3251   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3252   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3253   emit_int8(0x00);
3254   emit_int8((unsigned char)(0xC0 | encode));
3255 }
3256 
3257 void Assembler::pshufb(XMMRegister dst, Address src) {
3258   assert(VM_Version::supports_ssse3(), "");



3259   InstructionMark im(this);
3260   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3261   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3262   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3263   emit_int8(0x00);
3264   emit_operand(dst, src);
3265 }
3266 
3267 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {

3268   assert(isByte(mode), "invalid value");
3269   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3270   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
3271   InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3272   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3273   emit_int8(0x70);
3274   emit_int8((unsigned char)(0xC0 | encode));
3275   emit_int8(mode & 0xFF);
3276 }
3277 
3278 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {

3279   assert(isByte(mode), "invalid value");
3280   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3281   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");




3282   InstructionMark im(this);
3283   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3284   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3285   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3286   emit_int8(0x70);
3287   emit_operand(dst, src);
3288   emit_int8(mode & 0xFF);
3289 }
3290 
3291 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3292   assert(isByte(mode), "invalid value");
3293   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3294   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3295   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3296   emit_int8(0x70);
3297   emit_int8((unsigned char)(0xC0 | encode));
3298   emit_int8(mode & 0xFF);
3299 }
3300 
3301 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3302   assert(isByte(mode), "invalid value");
3303   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3304   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");



3305   InstructionMark im(this);
3306   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3307   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3308   simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3309   emit_int8(0x70);
3310   emit_operand(dst, src);
3311   emit_int8(mode & 0xFF);
3312 }
3313 
3314 void Assembler::psrldq(XMMRegister dst, int shift) {
3315   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3316   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3317   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3318   // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3319   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

3320   emit_int8(0x73);
3321   emit_int8((unsigned char)(0xC0 | encode));
3322   emit_int8(shift);
3323 }
3324 
3325 void Assembler::pslldq(XMMRegister dst, int shift) {
3326   // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3327   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3328   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3329   // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3330   int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

3331   emit_int8(0x73);
3332   emit_int8((unsigned char)(0xC0 | encode));
3333   emit_int8(shift);
3334 }
3335 
3336 void Assembler::ptest(XMMRegister dst, Address src) {
3337   assert(VM_Version::supports_sse4_1(), "");
3338   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3339   InstructionMark im(this);
3340   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3341   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3342   emit_int8(0x17);
3343   emit_operand(dst, src);
3344 }
3345 
3346 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3347   assert(VM_Version::supports_sse4_1(), "");
3348   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3349   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3350   emit_int8(0x17);
3351   emit_int8((unsigned char)(0xC0 | encode));
3352 }
3353 
3354 void Assembler::vptest(XMMRegister dst, Address src) {
3355   assert(VM_Version::supports_avx(), "");
3356   InstructionMark im(this);
3357   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3358   assert(dst != xnoreg, "sanity");

3359   // swap src<->dst for encoding
3360   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

3361   emit_int8(0x17);
3362   emit_operand(dst, src);
3363 }
3364 
3365 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3366   assert(VM_Version::supports_avx(), "");
3367   InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3368   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3369   emit_int8(0x17);
3370   emit_int8((unsigned char)(0xC0 | encode));
3371 }
3372 
3373 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3374   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3375   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3376   InstructionMark im(this);
3377   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3378   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3379   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3380   emit_int8(0x60);
3381   emit_operand(dst, src);
3382 }
3383 
3384 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3385   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3386   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3387   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3388   emit_int8(0x60);
3389   emit_int8((unsigned char)(0xC0 | encode));
3390 }
3391 
3392 void Assembler::punpckldq(XMMRegister dst, Address src) {

3393   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3394   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3395   InstructionMark im(this);
3396   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3397   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3398   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3399   emit_int8(0x62);
3400   emit_operand(dst, src);
3401 }
3402 
3403 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {

3404   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3405   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3406   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3407   emit_int8(0x62);
3408   emit_int8((unsigned char)(0xC0 | encode));
3409 }
3410 
3411 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {

3412   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3413   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3414   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3415   emit_int8(0x6C);
3416   emit_int8((unsigned char)(0xC0 | encode));

3417 }
3418 
3419 void Assembler::push(int32_t imm32) {
3420   // in 64bits we push 64bits onto the stack but only
3421   // take a 32bit immediate
3422   emit_int8(0x68);
3423   emit_int32(imm32);
3424 }
3425 
3426 void Assembler::push(Register src) {
3427   int encode = prefix_and_encode(src->encoding());
3428 
3429   emit_int8(0x50 | encode);
3430 }
3431 
3432 void Assembler::pushf() {
3433   emit_int8((unsigned char)0x9C);
3434 }
3435 
3436 #ifndef _LP64 // no 32bit push/pop on amd64


3441   emit_int8((unsigned char)0xFF);
3442   emit_operand(rsi, src);
3443 }
3444 #endif
3445 
3446 void Assembler::rcll(Register dst, int imm8) {
3447   assert(isShiftCount(imm8), "illegal shift count");
3448   int encode = prefix_and_encode(dst->encoding());
3449   if (imm8 == 1) {
3450     emit_int8((unsigned char)0xD1);
3451     emit_int8((unsigned char)(0xD0 | encode));
3452   } else {
3453     emit_int8((unsigned char)0xC1);
3454     emit_int8((unsigned char)0xD0 | encode);
3455     emit_int8(imm8);
3456   }
3457 }
3458 
3459 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3460   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3461   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3462   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3463   emit_int8(0x53);
3464   emit_int8((unsigned char)(0xC0 | encode));
3465 }
3466 
3467 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3468   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3469   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3470   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3471   emit_int8(0x53);
3472   emit_int8((unsigned char)(0xC0 | encode));
3473 }
3474 
3475 void Assembler::rdtsc() {
3476   emit_int8((unsigned char)0x0F);
3477   emit_int8((unsigned char)0x31);
3478 }
3479 
3480 // copies data from [esi] to [edi] using rcx pointer sized words
3481 // generic
3482 void Assembler::rep_mov() {
3483   emit_int8((unsigned char)0xF3);
3484   // MOVSQ
3485   LP64_ONLY(prefix(REX_W));
3486   emit_int8((unsigned char)0xA5);
3487 }
3488 
3489 // sets rcx bytes with rax, value at [edi]
3490 void Assembler::rep_stosb() {
3491   emit_int8((unsigned char)0xF3); // REP
3492   LP64_ONLY(prefix(REX_W));


3611   assert(isShiftCount(imm8), "illegal shift count");
3612   int encode = prefix_and_encode(dst->encoding());
3613   emit_int8((unsigned char)0xC1);
3614   emit_int8((unsigned char)(0xE8 | encode));
3615   emit_int8(imm8);
3616 }
3617 
3618 void Assembler::shrl(Register dst) {
3619   int encode = prefix_and_encode(dst->encoding());
3620   emit_int8((unsigned char)0xD3);
3621   emit_int8((unsigned char)(0xE8 | encode));
3622 }
3623 
3624 // copies a single word from [esi] to [edi]
3625 void Assembler::smovl() {
3626   emit_int8((unsigned char)0xA5);
3627 }
3628 
3629 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3630   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3631   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3632   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3633   emit_int8(0x51);
3634   emit_int8((unsigned char)(0xC0 | encode));

3635 }
3636 
3637 void Assembler::sqrtsd(XMMRegister dst, Address src) {
3638   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3639   InstructionMark im(this);
3640   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3641   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3642   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3643   emit_int8(0x51);
3644   emit_operand(dst, src);

3645 }
3646 
3647 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3648   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3649   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3650   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3651   emit_int8(0x51);
3652   emit_int8((unsigned char)(0xC0 | encode));
3653 }
3654 
3655 void Assembler::std() {
3656   emit_int8((unsigned char)0xFD);
3657 }
3658 
3659 void Assembler::sqrtss(XMMRegister dst, Address src) {
3660   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3661   InstructionMark im(this);
3662   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3663   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3664   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3665   emit_int8(0x51);
3666   emit_operand(dst, src);
3667 }
3668 
3669 void Assembler::stmxcsr( Address dst) {
3670   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3671   InstructionMark im(this);
3672   prefix(dst);
3673   emit_int8(0x0F);
3674   emit_int8((unsigned char)0xAE);
3675   emit_operand(as_Register(3), dst);
3676 }
3677 
3678 void Assembler::subl(Address dst, int32_t imm32) {
3679   InstructionMark im(this);
3680   prefix(dst);
3681   emit_arith_operand(0x81, rbp, dst, imm32);
3682 }
3683 
3684 void Assembler::subl(Address dst, Register src) {
3685   InstructionMark im(this);
3686   prefix(dst, src);


3696 // Force generation of a 4 byte immediate value even if it fits into 8bit
3697 void Assembler::subl_imm32(Register dst, int32_t imm32) {
3698   prefix(dst);
3699   emit_arith_imm32(0x81, 0xE8, dst, imm32);
3700 }
3701 
3702 void Assembler::subl(Register dst, Address src) {
3703   InstructionMark im(this);
3704   prefix(src, dst);
3705   emit_int8(0x2B);
3706   emit_operand(dst, src);
3707 }
3708 
3709 void Assembler::subl(Register dst, Register src) {
3710   (void) prefix_and_encode(dst->encoding(), src->encoding());
3711   emit_arith(0x2B, 0xC0, dst, src);
3712 }
3713 
3714 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3715   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3716   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3717   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3718   emit_int8(0x5C);
3719   emit_int8((unsigned char)(0xC0 | encode));

3720 }
3721 
3722 void Assembler::subsd(XMMRegister dst, Address src) {
3723   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3724   InstructionMark im(this);
3725   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3726   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3727   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3728   emit_int8(0x5C);
3729   emit_operand(dst, src);



3730 }
3731 
3732 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3733   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3734   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3735   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3736   emit_int8(0x5C);
3737   emit_int8((unsigned char)(0xC0 | encode));
3738 }
3739 
3740 void Assembler::subss(XMMRegister dst, Address src) {
3741   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3742   InstructionMark im(this);
3743   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3744   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3745   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3746   emit_int8(0x5C);
3747   emit_operand(dst, src);
3748 }
3749 
3750 void Assembler::testb(Register dst, int imm8) {
3751   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
3752   (void) prefix_and_encode(dst->encoding(), true);
3753   emit_arith_b(0xF6, 0xC0, dst, imm8);
3754 }
3755 
3756 void Assembler::testl(Register dst, int32_t imm32) {
3757   // not using emit_arith because test
3758   // doesn't support sign-extension of
3759   // 8bit operands
3760   int encode = dst->encoding();
3761   if (encode == 0) {
3762     emit_int8((unsigned char)0xA9);
3763   } else {
3764     encode = prefix_and_encode(encode);
3765     emit_int8((unsigned char)0xF7);
3766     emit_int8((unsigned char)(0xC0 | encode));
3767   }


3783 void Assembler::tzcntl(Register dst, Register src) {
3784   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3785   emit_int8((unsigned char)0xF3);
3786   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3787   emit_int8(0x0F);
3788   emit_int8((unsigned char)0xBC);
3789   emit_int8((unsigned char)0xC0 | encode);
3790 }
3791 
3792 void Assembler::tzcntq(Register dst, Register src) {
3793   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3794   emit_int8((unsigned char)0xF3);
3795   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3796   emit_int8(0x0F);
3797   emit_int8((unsigned char)0xBC);
3798   emit_int8((unsigned char)(0xC0 | encode));
3799 }
3800 
3801 void Assembler::ucomisd(XMMRegister dst, Address src) {
3802   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3803   InstructionMark im(this);
3804   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3805   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3806   simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3807   emit_int8(0x2E);
3808   emit_operand(dst, src);

3809 }
3810 
3811 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3812   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3813   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3814   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3815   emit_int8(0x2E);
3816   emit_int8((unsigned char)(0xC0 | encode));

3817 }
3818 
3819 void Assembler::ucomiss(XMMRegister dst, Address src) {
3820   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3821   InstructionMark im(this);
3822   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3823   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3824   simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3825   emit_int8(0x2E);
3826   emit_operand(dst, src);
3827 }
3828 
3829 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3830   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3831   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3832   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3833   emit_int8(0x2E);
3834   emit_int8((unsigned char)(0xC0 | encode));
3835 }
3836 
3837 void Assembler::xabort(int8_t imm8) {
3838   emit_int8((unsigned char)0xC6);
3839   emit_int8((unsigned char)0xF8);
3840   emit_int8((unsigned char)(imm8 & 0xFF));
3841 }
3842 
3843 void Assembler::xaddl(Address dst, Register src) {
3844   InstructionMark im(this);
3845   prefix(dst, src);
3846   emit_int8(0x0F);
3847   emit_int8((unsigned char)0xC1);
3848   emit_operand(src, dst);
3849 }
3850 
3851 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3852   InstructionMark im(this);
3853   relocate(rtype);
3854   if (abort.is_bound()) {


3896   emit_arith(0x81, 0xF0, dst, imm32);
3897 }
3898 
3899 void Assembler::xorl(Register dst, Address src) {
3900   InstructionMark im(this);
3901   prefix(src, dst);
3902   emit_int8(0x33);
3903   emit_operand(dst, src);
3904 }
3905 
3906 void Assembler::xorl(Register dst, Register src) {
3907   (void) prefix_and_encode(dst->encoding(), src->encoding());
3908   emit_arith(0x33, 0xC0, dst, src);
3909 }
3910 
3911 
3912 // AVX 3-operands scalar float-point arithmetic instructions
3913 
3914 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3915   assert(VM_Version::supports_avx(), "");
3916   InstructionMark im(this);
3917   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3918   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3919   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3920   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3921   emit_int8(0x58);
3922   emit_operand(dst, src);
3923 }
3924 
3925 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3926   assert(VM_Version::supports_avx(), "");
3927   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3928   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3929   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3930   emit_int8(0x58);
3931   emit_int8((unsigned char)(0xC0 | encode));
3932 }
3933 
3934 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3935   assert(VM_Version::supports_avx(), "");
3936   InstructionMark im(this);
3937   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3938   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3939   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3940   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3941   emit_int8(0x58);
3942   emit_operand(dst, src);
3943 }
3944 
3945 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3946   assert(VM_Version::supports_avx(), "");
3947   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3948   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3949   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3950   emit_int8(0x58);
3951   emit_int8((unsigned char)(0xC0 | encode));
3952 }
3953 
3954 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3955   assert(VM_Version::supports_avx(), "");
3956   InstructionMark im(this);
3957   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3958   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3959   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3960   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3961   emit_int8(0x5E);
3962   emit_operand(dst, src);
3963 }
3964 
3965 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3966   assert(VM_Version::supports_avx(), "");
3967   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3968   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3969   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3970   emit_int8(0x5E);
3971   emit_int8((unsigned char)(0xC0 | encode));
3972 }
3973 
3974 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3975   assert(VM_Version::supports_avx(), "");
3976   InstructionMark im(this);
3977   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3978   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3979   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3980   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3981   emit_int8(0x5E);
3982   emit_operand(dst, src);
3983 }
3984 
3985 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3986   assert(VM_Version::supports_avx(), "");
3987   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3988   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3989   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3990   emit_int8(0x5E);
3991   emit_int8((unsigned char)(0xC0 | encode));
3992 }
3993 
3994 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3995   assert(VM_Version::supports_avx(), "");
3996   InstructionMark im(this);
3997   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3998   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3999   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4000   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4001   emit_int8(0x59);
4002   emit_operand(dst, src);
4003 }
4004 
4005 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4006   assert(VM_Version::supports_avx(), "");
4007   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4008   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4009   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4010   emit_int8(0x59);
4011   emit_int8((unsigned char)(0xC0 | encode));
4012 }
4013 
4014 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4015   assert(VM_Version::supports_avx(), "");
4016   InstructionMark im(this);
4017   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4018   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4019   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4020   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4021   emit_int8(0x59);
4022   emit_operand(dst, src);
4023 }
4024 
4025 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4026   assert(VM_Version::supports_avx(), "");
4027   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4028   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4029   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4030   emit_int8(0x59);
4031   emit_int8((unsigned char)(0xC0 | encode));
4032 }
4033 
4034 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4035   assert(VM_Version::supports_avx(), "");
4036   InstructionMark im(this);
4037   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4038   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4039   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4040   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4041   emit_int8(0x5C);
4042   emit_operand(dst, src);
4043 }
4044 
4045 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4046   assert(VM_Version::supports_avx(), "");
4047   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4048   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4049   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4050   emit_int8(0x5C);
4051   emit_int8((unsigned char)(0xC0 | encode));
4052 }
4053 
4054 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4055   assert(VM_Version::supports_avx(), "");
4056   InstructionMark im(this);
4057   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4058   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4059   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4060   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4061   emit_int8(0x5C);
4062   emit_operand(dst, src);
4063 }
4064 
4065 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4066   assert(VM_Version::supports_avx(), "");
4067   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4068   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4069   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4070   emit_int8(0x5C);
4071   emit_int8((unsigned char)(0xC0 | encode));
4072 }
4073 
4074 //====================VECTOR ARITHMETIC=====================================
4075 
4076 // Float-point vector arithmetic
4077 
4078 void Assembler::addpd(XMMRegister dst, XMMRegister src) {

4079   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4080   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4081   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4082   emit_int8(0x58);
4083   emit_int8((unsigned char)(0xC0 | encode));

4084 }
4085 
4086 void Assembler::addps(XMMRegister dst, XMMRegister src) {

4087   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4088   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4089   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4090   emit_int8(0x58);
4091   emit_int8((unsigned char)(0xC0 | encode));
4092 }
4093 
4094 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4095   assert(VM_Version::supports_avx(), "");
4096   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4097   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4098   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4099   emit_int8(0x58);
4100   emit_int8((unsigned char)(0xC0 | encode));
4101 }
4102 
4103 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4104   assert(VM_Version::supports_avx(), "");
4105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4106   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4107   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4108   emit_int8(0x58);
4109   emit_int8((unsigned char)(0xC0 | encode));
4110 }
4111 
4112 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4113   assert(VM_Version::supports_avx(), "");
4114   InstructionMark im(this);
4115   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4116   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4117   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4118   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4119   emit_int8(0x58);
4120   emit_operand(dst, src);
4121 }
4122 
4123 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4124   assert(VM_Version::supports_avx(), "");
4125   InstructionMark im(this);
4126   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4127   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4128   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4129   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4130   emit_int8(0x58);
4131   emit_operand(dst, src);
4132 }
4133 
4134 void Assembler::subpd(XMMRegister dst, XMMRegister src) {

4135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4136   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4137   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4138   emit_int8(0x5C);
4139   emit_int8((unsigned char)(0xC0 | encode));

4140 }
4141 
4142 void Assembler::subps(XMMRegister dst, XMMRegister src) {

4143   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4144   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4145   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4146   emit_int8(0x5C);
4147   emit_int8((unsigned char)(0xC0 | encode));
4148 }
4149 
4150 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4151   assert(VM_Version::supports_avx(), "");
4152   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4153   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4154   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4155   emit_int8(0x5C);
4156   emit_int8((unsigned char)(0xC0 | encode));
4157 }
4158 
4159 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4160   assert(VM_Version::supports_avx(), "");
4161   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4162   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4163   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4164   emit_int8(0x5C);
4165   emit_int8((unsigned char)(0xC0 | encode));
4166 }
4167 
4168 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4169   assert(VM_Version::supports_avx(), "");
4170   InstructionMark im(this);
4171   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4172   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4173   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4174   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4175   emit_int8(0x5C);
4176   emit_operand(dst, src);
4177 }
4178 
4179 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4180   assert(VM_Version::supports_avx(), "");
4181   InstructionMark im(this);
4182   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4183   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4184   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4185   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4186   emit_int8(0x5C);
4187   emit_operand(dst, src);
4188 }
4189 
4190 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {

4191   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4192   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4193   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4194   emit_int8(0x59);
4195   emit_int8((unsigned char)(0xC0 | encode));

4196 }
4197 
4198 void Assembler::mulpd(XMMRegister dst, Address src) {

4199   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4200   InstructionMark im(this);
4201   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4202   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4203   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4204   emit_int8(0x59);
4205   emit_operand(dst, src);
4206 }
4207 
4208 void Assembler::mulps(XMMRegister dst, XMMRegister src) {

4209   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4210   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4211   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4212   emit_int8(0x59);
4213   emit_int8((unsigned char)(0xC0 | encode));
4214 }
4215 
4216 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4217   assert(VM_Version::supports_avx(), "");
4218   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4219   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4220   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4221   emit_int8(0x59);
4222   emit_int8((unsigned char)(0xC0 | encode));
4223 }
4224 
4225 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4226   assert(VM_Version::supports_avx(), "");
4227   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4228   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4229   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4230   emit_int8(0x59);
4231   emit_int8((unsigned char)(0xC0 | encode));
4232 }
4233 
4234 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4235   assert(VM_Version::supports_avx(), "");
4236   InstructionMark im(this);
4237   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4238   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4239   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4240   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4241   emit_int8(0x59);
4242   emit_operand(dst, src);
4243 }
4244 
4245 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4246   assert(VM_Version::supports_avx(), "");
4247   InstructionMark im(this);
4248   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4249   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4250   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4251   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4252   emit_int8(0x59);
4253   emit_operand(dst, src);
4254 }
4255 
4256 void Assembler::divpd(XMMRegister dst, XMMRegister src) {

4257   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4258   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4259   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4260   emit_int8(0x5E);
4261   emit_int8((unsigned char)(0xC0 | encode));

4262 }
4263 
4264 void Assembler::divps(XMMRegister dst, XMMRegister src) {

4265   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4266   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4267   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4268   emit_int8(0x5E);
4269   emit_int8((unsigned char)(0xC0 | encode));
4270 }
4271 
4272 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4273   assert(VM_Version::supports_avx(), "");
4274   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4275   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4276   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4277   emit_int8(0x5E);
4278   emit_int8((unsigned char)(0xC0 | encode));
4279 }
4280 
4281 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4282   assert(VM_Version::supports_avx(), "");
4283   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4284   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4285   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4286   emit_int8(0x5E);
4287   emit_int8((unsigned char)(0xC0 | encode));
4288 }
4289 
4290 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4291   assert(VM_Version::supports_avx(), "");
4292   InstructionMark im(this);
4293   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4294   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4295   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4296   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4297   emit_int8(0x5E);
4298   emit_operand(dst, src);
4299 }
4300 
4301 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4302   assert(VM_Version::supports_avx(), "");
4303   InstructionMark im(this);
4304   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4305   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4306   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4307   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4308   emit_int8(0x5E);
4309   emit_operand(dst, src);
4310 }
4311 
4312 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {

4313   assert(VM_Version::supports_avx(), "");
4314   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4315   int nds_enc = 0;
4316   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4317   emit_int8(0x51);
4318   emit_int8((unsigned char)(0xC0 | encode));
4319 }
4320 
4321 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {

4322   assert(VM_Version::supports_avx(), "");
4323   InstructionMark im(this);
4324   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4325   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4326   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4327   emit_int8(0x51);
4328   emit_operand(dst, src);

4329 }
4330 
4331 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4332   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4333   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4334   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4335   emit_int8(0x54);
4336   emit_int8((unsigned char)(0xC0 | encode));

4337 }
4338 
4339 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4340   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4341   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4342   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4343   emit_int8(0x54);
4344   emit_int8((unsigned char)(0xC0 | encode));
4345 }
4346 
4347 void Assembler::andps(XMMRegister dst, Address src) {
4348   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4349   InstructionMark im(this);
4350   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4351   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4352   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4353   emit_int8(0x54);
4354   emit_operand(dst, src);
4355 }
4356 
4357 void Assembler::andpd(XMMRegister dst, Address src) {
4358   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4359   InstructionMark im(this);
4360   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4361   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4362   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4363   emit_int8(0x54);
4364   emit_operand(dst, src);

4365 }
4366 
4367 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4368   assert(VM_Version::supports_avx(), "");
4369   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4370   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4371   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4372   emit_int8(0x54);
4373   emit_int8((unsigned char)(0xC0 | encode));
4374 }
4375 
4376 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4377   assert(VM_Version::supports_avx(), "");
4378   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4379   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4380   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4381   emit_int8(0x54);
4382   emit_int8((unsigned char)(0xC0 | encode));
4383 }
4384 
4385 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4386   assert(VM_Version::supports_avx(), "");
4387   InstructionMark im(this);
4388   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4389   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4390   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4391   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4392   emit_int8(0x54);
4393   emit_operand(dst, src);
4394 }
4395 
4396 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4397   assert(VM_Version::supports_avx(), "");
4398   InstructionMark im(this);
4399   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4400   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4401   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4402   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4403   emit_int8(0x54);
4404   emit_operand(dst, src);
4405 }
4406 
4407 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {

4408   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4409   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4410   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4411   emit_int8(0x15);
4412   emit_int8((unsigned char)(0xC0 | encode));

4413 }
4414 
4415 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {

4416   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4417   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4418   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4419   emit_int8(0x14);
4420   emit_int8((unsigned char)(0xC0 | encode));

4421 }
4422 
4423 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4424   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4425   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4426   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4427   emit_int8(0x57);
4428   emit_int8((unsigned char)(0xC0 | encode));

4429 }
4430 
4431 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4432   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4433   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4434   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4435   emit_int8(0x57);
4436   emit_int8((unsigned char)(0xC0 | encode));
4437 }
4438 
4439 void Assembler::xorpd(XMMRegister dst, Address src) {
4440   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4441   InstructionMark im(this);
4442   InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4443   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4444   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4445   emit_int8(0x57);
4446   emit_operand(dst, src);

4447 }
4448 
4449 void Assembler::xorps(XMMRegister dst, Address src) {
4450   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4451   InstructionMark im(this);
4452   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4453   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4454   simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4455   emit_int8(0x57);
4456   emit_operand(dst, src);
4457 }
4458 
4459 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4460   assert(VM_Version::supports_avx(), "");
4461   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4462   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4463   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4464   emit_int8(0x57);
4465   emit_int8((unsigned char)(0xC0 | encode));
4466 }
4467 
4468 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4469   assert(VM_Version::supports_avx(), "");
4470   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4471   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4472   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4473   emit_int8(0x57);
4474   emit_int8((unsigned char)(0xC0 | encode));
4475 }
4476 
4477 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4478   assert(VM_Version::supports_avx(), "");
4479   InstructionMark im(this);
4480   InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4481   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4482   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4483   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4484   emit_int8(0x57);
4485   emit_operand(dst, src);
4486 }
4487 
4488 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4489   assert(VM_Version::supports_avx(), "");
4490   InstructionMark im(this);
4491   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4492   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4493   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4494   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4495   emit_int8(0x57);
4496   emit_operand(dst, src);
4497 }
4498 
4499 // Integer vector arithmetic
4500 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4501   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4502          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4503   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4504   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4505   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4506   emit_int8(0x01);
4507   emit_int8((unsigned char)(0xC0 | encode));
4508 }
4509 
4510 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4511   assert(VM_Version::supports_avx() && (vector_len == 0) ||
4512          VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4513   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4514   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4515   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4516   emit_int8(0x02);
4517   emit_int8((unsigned char)(0xC0 | encode));
4518 }
4519 
4520 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4521   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4522   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4523   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4524   emit_int8((unsigned char)0xFC);
4525   emit_int8((unsigned char)(0xC0 | encode));
4526 }
4527 
4528 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4529   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4530   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4531   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4532   emit_int8((unsigned char)0xFD);
4533   emit_int8((unsigned char)(0xC0 | encode));
4534 }
4535 
4536 void Assembler::paddd(XMMRegister dst, XMMRegister src) {

4537   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4538   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4539   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4540   emit_int8((unsigned char)0xFE);
4541   emit_int8((unsigned char)(0xC0 | encode));
4542 }
4543 
4544 void Assembler::paddq(XMMRegister dst, XMMRegister src) {

4545   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4546   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4547   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4548   emit_int8((unsigned char)0xD4);
4549   emit_int8((unsigned char)(0xC0 | encode));

4550 }
4551 
4552 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
4553   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4554   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4555   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4556   emit_int8(0x01);
4557   emit_int8((unsigned char)(0xC0 | encode));
4558 }
4559 
4560 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
4561   NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4562   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4563   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4564   emit_int8(0x02);
4565   emit_int8((unsigned char)(0xC0 | encode));
4566 }
4567 
4568 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4569   assert(UseAVX > 0, "requires some form of AVX");
4570   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4571   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4572   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4573   emit_int8((unsigned char)0xFC);
4574   emit_int8((unsigned char)(0xC0 | encode));
4575 }
4576 
4577 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4578   assert(UseAVX > 0, "requires some form of AVX");
4579   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4580   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4581   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4582   emit_int8((unsigned char)0xFD);
4583   emit_int8((unsigned char)(0xC0 | encode));
4584 }
4585 
4586 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4587   assert(UseAVX > 0, "requires some form of AVX");
4588   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4589   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4590   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4591   emit_int8((unsigned char)0xFE);
4592   emit_int8((unsigned char)(0xC0 | encode));
4593 }
4594 
4595 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4596   assert(UseAVX > 0, "requires some form of AVX");
4597   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4598   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4599   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4600   emit_int8((unsigned char)0xD4);
4601   emit_int8((unsigned char)(0xC0 | encode));
4602 }
4603 
4604 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4605   assert(UseAVX > 0, "requires some form of AVX");
4606   InstructionMark im(this);
4607   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4608   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4609   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4610   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4611   emit_int8((unsigned char)0xFC);
4612   emit_operand(dst, src);
4613 }
4614 
4615 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4616   assert(UseAVX > 0, "requires some form of AVX");
4617   InstructionMark im(this);
4618   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4619   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4620   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4621   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4622   emit_int8((unsigned char)0xFD);
4623   emit_operand(dst, src);
4624 }
4625 
4626 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4627   assert(UseAVX > 0, "requires some form of AVX");
4628   InstructionMark im(this);
4629   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4630   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4631   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4632   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4633   emit_int8((unsigned char)0xFE);
4634   emit_operand(dst, src);
4635 }
4636 
4637 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4638   assert(UseAVX > 0, "requires some form of AVX");
4639   InstructionMark im(this);
4640   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4641   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4642   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4643   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4644   emit_int8((unsigned char)0xD4);
4645   emit_operand(dst, src);
4646 }
4647 
4648 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
4649   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4650   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4651   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4652   emit_int8((unsigned char)0xF8);
4653   emit_int8((unsigned char)(0xC0 | encode));
4654 }
4655 
4656 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
4657   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4658   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4659   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4660   emit_int8((unsigned char)0xF9);
4661   emit_int8((unsigned char)(0xC0 | encode));
4662 }
4663 
4664 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
4665   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4666   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4667   emit_int8((unsigned char)0xFA);
4668   emit_int8((unsigned char)(0xC0 | encode));
4669 }
4670 
4671 void Assembler::psubq(XMMRegister dst, XMMRegister src) {

4672   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4673   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4674   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4675   emit_int8((unsigned char)0xFB);
4676   emit_int8((unsigned char)(0xC0 | encode));

4677 }
4678 
4679 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4680   assert(UseAVX > 0, "requires some form of AVX");
4681   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4682   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4683   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4684   emit_int8((unsigned char)0xF8);
4685   emit_int8((unsigned char)(0xC0 | encode));
4686 }
4687 
4688 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4689   assert(UseAVX > 0, "requires some form of AVX");
4690   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4691   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4692   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4693   emit_int8((unsigned char)0xF9);
4694   emit_int8((unsigned char)(0xC0 | encode));
4695 }
4696 
4697 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4698   assert(UseAVX > 0, "requires some form of AVX");
4699   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4700   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4701   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4702   emit_int8((unsigned char)0xFA);
4703   emit_int8((unsigned char)(0xC0 | encode));
4704 }
4705 
4706 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4707   assert(UseAVX > 0, "requires some form of AVX");
4708   InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4709   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4710   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4711   emit_int8((unsigned char)0xFB);
4712   emit_int8((unsigned char)(0xC0 | encode));
4713 }
4714 
4715 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4716   assert(UseAVX > 0, "requires some form of AVX");
4717   InstructionMark im(this);
4718   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4719   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4720   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4721   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4722   emit_int8((unsigned char)0xF8);
4723   emit_operand(dst, src);
4724 }
4725 
4726 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4727   assert(UseAVX > 0, "requires some form of AVX");
4728   InstructionMark im(this);
4729   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4730   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4731   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4732   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4733   emit_int8((unsigned char)0xF9);
4734   emit_operand(dst, src);
4735 }
4736 
4737 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4738   assert(UseAVX > 0, "requires some form of AVX");
4739   InstructionMark im(this);
4740   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4741   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4742   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4743   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4744   emit_int8((unsigned char)0xFA);
4745   emit_operand(dst, src);
4746 }
4747 
4748 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4749   assert(UseAVX > 0, "requires some form of AVX");
4750   InstructionMark im(this);
4751   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4752   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4753   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4754   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4755   emit_int8((unsigned char)0xFB);
4756   emit_operand(dst, src);
4757 }
4758 
4759 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
4760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4761   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4762   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4763   emit_int8((unsigned char)0xD5);
4764   emit_int8((unsigned char)(0xC0 | encode));
4765 }
4766 
4767 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {

4768   assert(VM_Version::supports_sse4_1(), "");
4769   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4770   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4771   emit_int8(0x40);
4772   emit_int8((unsigned char)(0xC0 | encode));
4773 }
4774 
4775 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4776   assert(UseAVX > 0, "requires some form of AVX");
4777   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4778   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4779   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4780   emit_int8((unsigned char)0xD5);
4781   emit_int8((unsigned char)(0xC0 | encode));
4782 }
4783 
4784 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

4785   assert(UseAVX > 0, "requires some form of AVX");
4786   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4787   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4788   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4789   emit_int8(0x40);
4790   emit_int8((unsigned char)(0xC0 | encode));
4791 }
4792 
4793 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4794   assert(UseAVX > 2, "requires some form of AVX");
4795   InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);

4796   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4797   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

4798   emit_int8(0x40);
4799   emit_int8((unsigned char)(0xC0 | encode));
4800 }
4801 
4802 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4803   assert(UseAVX > 0, "requires some form of AVX");
4804   InstructionMark im(this);
4805   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4806   attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4807   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4808   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4809   emit_int8((unsigned char)0xD5);
4810   emit_operand(dst, src);
4811 }
4812 
4813 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

4814   assert(UseAVX > 0, "requires some form of AVX");




4815   InstructionMark im(this);
4816   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4817   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4818   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4819   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

4820   emit_int8(0x40);
4821   emit_operand(dst, src);
4822 }
4823 
4824 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4825   assert(UseAVX > 0, "requires some form of AVX");




4826   InstructionMark im(this);
4827   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4828   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4829   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4830   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

4831   emit_int8(0x40);
4832   emit_operand(dst, src);
4833 }
4834 
4835 // Shift packed integers left by specified number of bits.
4836 void Assembler::psllw(XMMRegister dst, int shift) {
4837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4838   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4839   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4840   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

4841   emit_int8(0x71);
4842   emit_int8((unsigned char)(0xC0 | encode));
4843   emit_int8(shift & 0xFF);
4844 }
4845 
4846 void Assembler::pslld(XMMRegister dst, int shift) {

4847   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4848   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4849   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4850   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4851   emit_int8(0x72);
4852   emit_int8((unsigned char)(0xC0 | encode));
4853   emit_int8(shift & 0xFF);
4854 }
4855 
4856 void Assembler::psllq(XMMRegister dst, int shift) {

4857   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4858   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4859   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4860   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4861   emit_int8(0x73);
4862   emit_int8((unsigned char)(0xC0 | encode));
4863   emit_int8(shift & 0xFF);
4864 }
4865 
4866 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
4867   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4868   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4869   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4870   emit_int8((unsigned char)0xF1);
4871   emit_int8((unsigned char)(0xC0 | encode));
4872 }
4873 
4874 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {

4875   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4876   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4877   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4878   emit_int8((unsigned char)0xF2);
4879   emit_int8((unsigned char)(0xC0 | encode));
4880 }
4881 
4882 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {

4883   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4884   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4885   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4886   emit_int8((unsigned char)0xF3);
4887   emit_int8((unsigned char)(0xC0 | encode));

4888 }
4889 
4890 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4891   assert(UseAVX > 0, "requires some form of AVX");
4892   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4893   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4894   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4895   emit_int8(0x71);
4896   emit_int8((unsigned char)(0xC0 | encode));
4897   emit_int8(shift & 0xFF);
4898 }
4899 
4900 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {

4901   assert(UseAVX > 0, "requires some form of AVX");
4902   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4903   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4904   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4905   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4906   emit_int8(0x72);
4907   emit_int8((unsigned char)(0xC0 | encode));
4908   emit_int8(shift & 0xFF);
4909 }
4910 
4911 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {

4912   assert(UseAVX > 0, "requires some form of AVX");
4913   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4914   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4915   int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4916   emit_int8(0x73);
4917   emit_int8((unsigned char)(0xC0 | encode));


4918   emit_int8(shift & 0xFF);
4919 }
4920 
4921 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4922   assert(UseAVX > 0, "requires some form of AVX");
4923   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4924   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4925   emit_int8((unsigned char)0xF1);
4926   emit_int8((unsigned char)(0xC0 | encode));
4927 }
4928 
4929 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {

4930   assert(UseAVX > 0, "requires some form of AVX");
4931   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4932   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4933   emit_int8((unsigned char)0xF2);
4934   emit_int8((unsigned char)(0xC0 | encode));
4935 }
4936 
4937 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {

4938   assert(UseAVX > 0, "requires some form of AVX");
4939   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4940   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4941   emit_int8((unsigned char)0xF3);
4942   emit_int8((unsigned char)(0xC0 | encode));

4943 }
4944 
4945 // Shift packed integers logically right by specified number of bits.
4946 void Assembler::psrlw(XMMRegister dst, int shift) {
4947   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4948   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4949   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4950   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

4951   emit_int8(0x71);
4952   emit_int8((unsigned char)(0xC0 | encode));
4953   emit_int8(shift & 0xFF);
4954 }
4955 
4956 void Assembler::psrld(XMMRegister dst, int shift) {

4957   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4958   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4959   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4960   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4961   emit_int8(0x72);
4962   emit_int8((unsigned char)(0xC0 | encode));
4963   emit_int8(shift & 0xFF);
4964 }
4965 
4966 void Assembler::psrlq(XMMRegister dst, int shift) {

4967   // Do not confuse it with psrldq SSE2 instruction which
4968   // shifts 128 bit value in xmm register by number of bytes.
4969   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4970   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4971   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4972   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

4973   emit_int8(0x73);
4974   emit_int8((unsigned char)(0xC0 | encode));
4975   emit_int8(shift & 0xFF);
4976 }
4977 
4978 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
4979   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4980   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4981   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4982   emit_int8((unsigned char)0xD1);
4983   emit_int8((unsigned char)(0xC0 | encode));
4984 }
4985 
4986 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {

4987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4988   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4989   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4990   emit_int8((unsigned char)0xD2);
4991   emit_int8((unsigned char)(0xC0 | encode));
4992 }
4993 
4994 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {

4995   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4996   InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4997   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4998   emit_int8((unsigned char)0xD3);
4999   emit_int8((unsigned char)(0xC0 | encode));

5000 }
5001 
5002 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5003   assert(UseAVX > 0, "requires some form of AVX");
5004   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5005   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5006   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5007   emit_int8(0x71);
5008   emit_int8((unsigned char)(0xC0 | encode));
5009   emit_int8(shift & 0xFF);
5010 }
5011 
5012 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {

5013   assert(UseAVX > 0, "requires some form of AVX");
5014   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5015   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5016   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5017   emit_int8(0x72);
5018   emit_int8((unsigned char)(0xC0 | encode));
5019   emit_int8(shift & 0xFF);
5020 }
5021 
5022 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {

5023   assert(UseAVX > 0, "requires some form of AVX");
5024   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5025   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5026   int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5027   emit_int8(0x73);
5028   emit_int8((unsigned char)(0xC0 | encode));


5029   emit_int8(shift & 0xFF);
5030 }
5031 
5032 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5033   assert(UseAVX > 0, "requires some form of AVX");
5034   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5035   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5036   emit_int8((unsigned char)0xD1);
5037   emit_int8((unsigned char)(0xC0 | encode));
5038 }
5039 
5040 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {

5041   assert(UseAVX > 0, "requires some form of AVX");
5042   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5043   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5044   emit_int8((unsigned char)0xD2);
5045   emit_int8((unsigned char)(0xC0 | encode));
5046 }
5047 
5048 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {

5049   assert(UseAVX > 0, "requires some form of AVX");
5050   InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5051   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5052   emit_int8((unsigned char)0xD3);
5053   emit_int8((unsigned char)(0xC0 | encode));

5054 }
5055 
5056 // Shift packed integers arithmetically right by specified number of bits.
5057 void Assembler::psraw(XMMRegister dst, int shift) {
5058   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5059   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5060   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5061   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);

5062   emit_int8(0x71);
5063   emit_int8((unsigned char)(0xC0 | encode));
5064   emit_int8(shift & 0xFF);
5065 }
5066 
5067 void Assembler::psrad(XMMRegister dst, int shift) {

5068   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5069   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5070   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5071   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5072   emit_int8(0x72);
5073   emit_int8((unsigned char)(0xC0 | encode));
5074   emit_int8(shift & 0xFF);
5075 }
5076 
5077 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5078   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5079   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5080   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5081   emit_int8((unsigned char)0xE1);
5082   emit_int8((unsigned char)(0xC0 | encode));
5083 }
5084 
5085 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {

5086   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5087   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5088   int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5089   emit_int8((unsigned char)0xE2);
5090   emit_int8((unsigned char)(0xC0 | encode));
5091 }
5092 
5093 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5094   assert(UseAVX > 0, "requires some form of AVX");
5095   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5096   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5097   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5098   emit_int8(0x71);
5099   emit_int8((unsigned char)(0xC0 | encode));
5100   emit_int8(shift & 0xFF);
5101 }
5102 
5103 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {

5104   assert(UseAVX > 0, "requires some form of AVX");
5105   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5106   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5107   int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5108   emit_int8(0x72);
5109   emit_int8((unsigned char)(0xC0 | encode));
5110   emit_int8(shift & 0xFF);
5111 }
5112 
5113 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5114   assert(UseAVX > 0, "requires some form of AVX");
5115   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5116   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5117   emit_int8((unsigned char)0xE1);
5118   emit_int8((unsigned char)(0xC0 | encode));
5119 }
5120 
5121 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {

5122   assert(UseAVX > 0, "requires some form of AVX");
5123   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5124   int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5125   emit_int8((unsigned char)0xE2);
5126   emit_int8((unsigned char)(0xC0 | encode));
5127 }
5128 
5129 
5130 // logical operations packed integers
5131 void Assembler::pand(XMMRegister dst, XMMRegister src) {

5132   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5133   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5134   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5135   emit_int8((unsigned char)0xDB);
5136   emit_int8((unsigned char)(0xC0 | encode));
5137 }
5138 
5139 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

5140   assert(UseAVX > 0, "requires some form of AVX");
5141   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5142   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5143   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5144   emit_int8((unsigned char)0xDB);
5145   emit_int8((unsigned char)(0xC0 | encode));
5146 }
5147 
5148 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

5149   assert(UseAVX > 0, "requires some form of AVX");
5150   InstructionMark im(this);
5151   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5152   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5153   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5154   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5155   emit_int8((unsigned char)0xDB);
5156   emit_operand(dst, src);
5157 }
5158 
5159 void Assembler::pandn(XMMRegister dst, XMMRegister src) {

5160   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5161   InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5162   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5163   emit_int8((unsigned char)0xDF);
5164   emit_int8((unsigned char)(0xC0 | encode));


5165 }
5166 
5167 void Assembler::por(XMMRegister dst, XMMRegister src) {

5168   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5169   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5170   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5171   emit_int8((unsigned char)0xEB);
5172   emit_int8((unsigned char)(0xC0 | encode));
5173 }
5174 
5175 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

5176   assert(UseAVX > 0, "requires some form of AVX");
5177   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5178   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5179   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5180   emit_int8((unsigned char)0xEB);
5181   emit_int8((unsigned char)(0xC0 | encode));
5182 }
5183 
5184 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

5185   assert(UseAVX > 0, "requires some form of AVX");
5186   InstructionMark im(this);
5187   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5188   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5189   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5190   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5191   emit_int8((unsigned char)0xEB);
5192   emit_operand(dst, src);
5193 }
5194 
5195 void Assembler::pxor(XMMRegister dst, XMMRegister src) {

5196   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5197   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5198   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5199   emit_int8((unsigned char)0xEF);
5200   emit_int8((unsigned char)(0xC0 | encode));
5201 }
5202 
5203 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {

5204   assert(UseAVX > 0, "requires some form of AVX");
5205   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5206   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5207   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5208   emit_int8((unsigned char)0xEF);
5209   emit_int8((unsigned char)(0xC0 | encode));
5210 }
5211 
5212 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {

5213   assert(UseAVX > 0, "requires some form of AVX");
5214   InstructionMark im(this);
5215   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5216   attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5217   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5218   vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5219   emit_int8((unsigned char)0xEF);
5220   emit_operand(dst, src);
5221 }
5222 
5223 
5224 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5225   assert(VM_Version::supports_avx(), "");
5226   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5227   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5228   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5229   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);

5230   emit_int8(0x18);
5231   emit_int8((unsigned char)(0xC0 | encode));
5232   // 0x00 - insert into lower 128 bits
5233   // 0x01 - insert into upper 128 bits
5234   emit_int8(0x01);
5235 }
5236 
5237 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5238   assert(VM_Version::supports_evex(), "");
5239   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);


5240   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5241   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);

5242   emit_int8(0x1A);
5243   emit_int8((unsigned char)(0xC0 | encode));
5244   // 0x00 - insert into lower 256 bits
5245   // 0x01 - insert into upper 256 bits
5246   emit_int8(value & 0x01);
5247 }
5248 
5249 void Assembler::vinsertf64x4h(XMMRegister dst, Address src, int value) {
5250   assert(VM_Version::supports_evex(), "");




5251   assert(dst != xnoreg, "sanity");
5252   InstructionMark im(this);
5253   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5254   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5255   // swap src<->dst for encoding
5256   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5257   emit_int8(0x1A);
5258   emit_operand(dst, src);
5259   // 0x00 - insert into lower 256 bits
5260   // 0x01 - insert into upper 128 bits
5261   emit_int8(value & 0x01);
5262 }
5263 
5264 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5265   assert(VM_Version::supports_evex(), "");
5266   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);


5267   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5268   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);

5269   emit_int8(0x18);
5270   emit_int8((unsigned char)(0xC0 | encode));
5271   // 0x00 - insert into q0 128 bits (0..127)
5272   // 0x01 - insert into q1 128 bits (128..255)
5273   // 0x02 - insert into q2 128 bits (256..383)
5274   // 0x03 - insert into q3 128 bits (384..511)
5275   emit_int8(value & 0x3);
5276 }
5277 
5278 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
5279   assert(VM_Version::supports_avx(), "");




5280   assert(dst != xnoreg, "sanity");
5281   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5282   InstructionMark im(this);
5283   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5284   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5285   // swap src<->dst for encoding
5286   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5287   emit_int8(0x18);
5288   emit_operand(dst, src);
5289   // 0x00 - insert into q0 128 bits (0..127)
5290   // 0x01 - insert into q1 128 bits (128..255)
5291   // 0x02 - insert into q2 128 bits (256..383)
5292   // 0x03 - insert into q3 128 bits (384..511)
5293   emit_int8(value & 0x3);
5294 }
5295 
5296 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
5297   assert(VM_Version::supports_avx(), "");







5298   assert(dst != xnoreg, "sanity");
5299   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5300   InstructionMark im(this);
5301   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5302   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5303   // swap src<->dst for encoding
5304   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5305   emit_int8(0x18);
5306   emit_operand(dst, src);
5307   // 0x01 - insert into upper 128 bits
5308   emit_int8(0x01);
5309 }
5310 
5311 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
5312   assert(VM_Version::supports_avx(), "");
5313   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5314   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5315   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);


5316   emit_int8(0x19);
5317   emit_int8((unsigned char)(0xC0 | encode));
5318   // 0x00 - insert into lower 128 bits
5319   // 0x01 - insert into upper 128 bits
5320   emit_int8(0x01);
5321 }
5322 
5323 void Assembler::vextractf128h(Address dst, XMMRegister src) {
5324   assert(VM_Version::supports_avx(), "");







5325   assert(src != xnoreg, "sanity");
5326   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5327   InstructionMark im(this);
5328   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5329   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5330   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5331   emit_int8(0x19);
5332   emit_operand(src, dst);
5333   // 0x01 - extract from upper 128 bits
5334   emit_int8(0x01);
5335 }
5336 
5337 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5338   assert(VM_Version::supports_avx2(), "");
5339   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5340   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5341   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5342   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);

5343   emit_int8(0x38);
5344   emit_int8((unsigned char)(0xC0 | encode));
5345   // 0x00 - insert into lower 128 bits
5346   // 0x01 - insert into upper 128 bits
5347   emit_int8(0x01);
5348 }
5349 
5350 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5351   assert(VM_Version::supports_evex(), "");
5352   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);


5353   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5354   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);

5355   emit_int8(0x38);
5356   emit_int8((unsigned char)(0xC0 | encode));
5357   // 0x00 - insert into lower 256 bits
5358   // 0x01 - insert into upper 256 bits
5359   emit_int8(value & 0x01);
5360 }
5361 
5362 void Assembler::vinserti128h(XMMRegister dst, Address src) {
5363   assert(VM_Version::supports_avx2(), "");







5364   assert(dst != xnoreg, "sanity");
5365   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5366   InstructionMark im(this);
5367   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5368   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5369   // swap src<->dst for encoding
5370   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5371   emit_int8(0x38);
5372   emit_operand(dst, src);
5373   // 0x01 - insert into upper 128 bits
5374   emit_int8(0x01);
5375 }
5376 
5377 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
5378   assert(VM_Version::supports_avx(), "");
5379   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5380   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5381   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);


5382   emit_int8(0x39);
5383   emit_int8((unsigned char)(0xC0 | encode));
5384   // 0x00 - insert into lower 128 bits
5385   // 0x01 - insert into upper 128 bits
5386   emit_int8(0x01);
5387 }
5388 
5389 void Assembler::vextracti128h(Address dst, XMMRegister src) {
5390   assert(VM_Version::supports_avx2(), "");







5391   assert(src != xnoreg, "sanity");
5392   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5393   InstructionMark im(this);
5394   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5395   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5396   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5397   emit_int8(0x39);
5398   emit_operand(src, dst);
5399   // 0x01 - extract from upper 128 bits
5400   emit_int8(0x01);
5401 }
5402 
5403 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src, int value) {
5404   assert(VM_Version::supports_evex(), "");
5405   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5406   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);



5407   emit_int8(0x3B);
5408   emit_int8((unsigned char)(0xC0 | encode));
5409   // 0x00 - extract from lower 256 bits
5410   // 0x01 - extract from upper 256 bits
5411   emit_int8(value & 0x01);
5412 }
5413 
5414 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
5415   assert(VM_Version::supports_evex(), "");
5416   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5417   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);









5418   emit_int8(0x39);
5419   emit_int8((unsigned char)(0xC0 | encode));
5420   // 0x01 - extract from bits 255:128
5421   // 0x02 - extract from bits 383:256
5422   // 0x03 - extract from bits 511:384
5423   emit_int8(value & 0x3);
5424 }
5425 
5426 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src, int value) {
5427   assert(VM_Version::supports_evex(), "");
5428   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5429   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);



5430   emit_int8(0x1B);
5431   emit_int8((unsigned char)(0xC0 | encode));
5432   // 0x00 - extract from lower 256 bits
5433   // 0x01 - extract from upper 256 bits
5434   emit_int8(value & 0x1);
5435 }
5436 
5437 void Assembler::vextractf64x4h(Address dst, XMMRegister src, int value) {
5438   assert(VM_Version::supports_evex(), "");




5439   assert(src != xnoreg, "sanity");
5440   InstructionMark im(this);
5441   InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5442   attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */  EVEX_64bit);
5443   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5444   emit_int8(0x1B);
5445   emit_operand(src, dst);
5446   // 0x00 - extract from lower 256 bits
5447   // 0x01 - extract from upper 256 bits
5448   emit_int8(value & 0x01);
5449 }
5450 
5451 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
5452   assert(VM_Version::supports_avx(), "");
5453   int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5454   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5455   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);


5456   emit_int8(0x19);
5457   emit_int8((unsigned char)(0xC0 | encode));
5458   // 0x00 - extract from bits 127:0
5459   // 0x01 - extract from bits 255:128
5460   // 0x02 - extract from bits 383:256
5461   // 0x03 - extract from bits 511:384
5462   emit_int8(value & 0x3);
5463 }
5464 
5465 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
5466   assert(VM_Version::supports_evex(), "");




5467   assert(src != xnoreg, "sanity");
5468   InstructionMark im(this);
5469   InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5470   attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5471   vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5472   emit_int8(0x19);
5473   emit_operand(src, dst);
5474   // 0x00 - extract from bits 127:0
5475   // 0x01 - extract from bits 255:128
5476   // 0x02 - extract from bits 383:256
5477   // 0x03 - extract from bits 511:384
5478   emit_int8(value & 0x3);
5479 }
5480 
5481 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
5482   assert(VM_Version::supports_evex(), "");
5483   InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5484   int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);



5485   emit_int8(0x19);
5486   emit_int8((unsigned char)(0xC0 | encode));
5487   // 0x01 - extract from bits 255:128
5488   // 0x02 - extract from bits 383:256
5489   // 0x03 - extract from bits 511:384
5490   emit_int8(value & 0x3);
5491 }
5492 
5493 // duplicate 4-bytes integer data from src into 8 locations in dest
5494 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
5495   assert(VM_Version::supports_avx2(), "");
5496   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5497   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

5498   emit_int8(0x58);
5499   emit_int8((unsigned char)(0xC0 | encode));
5500 }
5501 
5502 // duplicate 2-bytes integer data from src into 16 locations in dest
5503 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
5504   assert(VM_Version::supports_avx2(), "");
5505   InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5506   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

5507   emit_int8(0x79);
5508   emit_int8((unsigned char)(0xC0 | encode));
5509 }
5510 
5511 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5512 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
5513   assert(VM_Version::supports_evex(), "");
5514   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5515   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5516   emit_int8(0x78);
5517   emit_int8((unsigned char)(0xC0 | encode));
5518 }
5519 
5520 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
5521   assert(VM_Version::supports_evex(), "");




5522   assert(dst != xnoreg, "sanity");
5523   InstructionMark im(this);
5524   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5525   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
5526   // swap src<->dst for encoding
5527   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5528   emit_int8(0x78);
5529   emit_operand(dst, src);
5530 }
5531 
5532 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5533 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5534   assert(VM_Version::supports_evex(), "");
5535   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5536   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5537   emit_int8(0x79);
5538   emit_int8((unsigned char)(0xC0 | encode));
5539 }
5540 
5541 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5542   assert(VM_Version::supports_evex(), "");




5543   assert(dst != xnoreg, "sanity");
5544   InstructionMark im(this);
5545   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5546   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
5547   // swap src<->dst for encoding
5548   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5549   emit_int8(0x79);
5550   emit_operand(dst, src);
5551 }
5552 
5553 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5554 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5555   assert(VM_Version::supports_evex(), "");
5556   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5557   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5558   emit_int8(0x58);
5559   emit_int8((unsigned char)(0xC0 | encode));
5560 }
5561 
5562 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5563   assert(VM_Version::supports_evex(), "");




5564   assert(dst != xnoreg, "sanity");
5565   InstructionMark im(this);
5566   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5567   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5568   // swap src<->dst for encoding
5569   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5570   emit_int8(0x58);
5571   emit_operand(dst, src);
5572 }
5573 
5574 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5575 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5576   assert(VM_Version::supports_evex(), "");
5577   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5578   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

5579   emit_int8(0x59);
5580   emit_int8((unsigned char)(0xC0 | encode));
5581 }
5582 
5583 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5584   assert(VM_Version::supports_evex(), "");




5585   assert(dst != xnoreg, "sanity");
5586   InstructionMark im(this);
5587   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5588   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5589   // swap src<->dst for encoding
5590   vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5591   emit_int8(0x59);
5592   emit_operand(dst, src);
5593 }
5594 
5595 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5596 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5597   assert(VM_Version::supports_evex(), "");
5598   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5599   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

5600   emit_int8(0x18);
5601   emit_int8((unsigned char)(0xC0 | encode));
5602 }
5603 
5604 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5605   assert(VM_Version::supports_evex(), "");



5606   assert(dst != xnoreg, "sanity");
5607   InstructionMark im(this);
5608   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5609   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5610   // swap src<->dst for encoding
5611   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5612   emit_int8(0x18);
5613   emit_operand(dst, src);
5614 }
5615 
5616 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5617 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5618   assert(VM_Version::supports_evex(), "");
5619   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5620   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);

5621   emit_int8(0x19);
5622   emit_int8((unsigned char)(0xC0 | encode));
5623 }
5624 
5625 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5626   assert(VM_Version::supports_evex(), "");




5627   assert(dst != xnoreg, "sanity");
5628   InstructionMark im(this);
5629   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5630   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5631   // swap src<->dst for encoding
5632   vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5633   emit_int8(0x19);
5634   emit_operand(dst, src);
5635 }
5636 
5637 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5638 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {

5639   assert(VM_Version::supports_evex(), "");
5640   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5641   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5642   emit_int8(0x7A);
5643   emit_int8((unsigned char)(0xC0 | encode));
5644 }
5645 
5646 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5647 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {

5648   assert(VM_Version::supports_evex(), "");
5649   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5650   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5651   emit_int8(0x7B);
5652   emit_int8((unsigned char)(0xC0 | encode));
5653 }
5654 
5655 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5656 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {

5657   assert(VM_Version::supports_evex(), "");
5658   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5659   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5660   emit_int8(0x7C);
5661   emit_int8((unsigned char)(0xC0 | encode));
5662 }
5663 
5664 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5665 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {

5666   assert(VM_Version::supports_evex(), "");
5667   InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5668   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5669   emit_int8(0x7C);
5670   emit_int8((unsigned char)(0xC0 | encode));
5671 }
5672 
5673 // Carry-Less Multiplication Quadword
5674 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5675   assert(VM_Version::supports_clmul(), "");
5676   InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5677   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5678   emit_int8(0x44);
5679   emit_int8((unsigned char)(0xC0 | encode));
5680   emit_int8((unsigned char)mask);
5681 }
5682 
5683 // Carry-Less Multiplication Quadword
5684 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5685   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5686   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5687   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5688   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5689   emit_int8(0x44);
5690   emit_int8((unsigned char)(0xC0 | encode));
5691   emit_int8((unsigned char)mask);
5692 }
5693 
5694 void Assembler::vzeroupper() {
5695   assert(VM_Version::supports_avx(), "");
5696   InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5697   (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);

5698   emit_int8(0x77);

5699 }
5700 
5701 
5702 #ifndef _LP64
5703 // 32bit only pieces of the assembler
5704 
5705 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5706   // NO PREFIX AS NEVER 64BIT
5707   InstructionMark im(this);
5708   emit_int8((unsigned char)0x81);
5709   emit_int8((unsigned char)(0xF8 | src1->encoding()));
5710   emit_data(imm32, rspec, 0);
5711 }
5712 
5713 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5714   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
5715   InstructionMark im(this);
5716   emit_int8((unsigned char)0x81);
5717   emit_operand(rdi, src1);
5718   emit_data(imm32, rspec, 0);


6182     emit_int8(simd_pre[pre]);
6183   }
6184   if (rex_w) {
6185     prefixq(adr, xreg);
6186   } else {
6187     prefix(adr, xreg);
6188   }
6189   if (opc > 0) {
6190     emit_int8(0x0F);
6191     int opc2 = simd_opc[opc];
6192     if (opc2 > 0) {
6193       emit_int8(opc2);
6194     }
6195   }
6196 }
6197 
6198 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6199   if (pre > 0) {
6200     emit_int8(simd_pre[pre]);
6201   }
6202   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);

6203   if (opc > 0) {
6204     emit_int8(0x0F);
6205     int opc2 = simd_opc[opc];
6206     if (opc2 > 0) {
6207       emit_int8(opc2);
6208     }
6209   }
6210   return encode;
6211 }
6212 
6213 
6214 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6215   int vector_len = _attributes->get_vector_len();
6216   bool vex_w = _attributes->is_rex_vex_w();
6217   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6218     prefix(VEX_3bytes);
6219 
6220     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6221     byte1 = (~byte1) & 0xE0;
6222     byte1 |= opc;
6223     emit_int8(byte1);
6224 
6225     int byte2 = ((~nds_enc) & 0xf) << 3;
6226     byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6227     emit_int8(byte2);
6228   } else {
6229     prefix(VEX_2bytes);
6230 
6231     int byte1 = vex_r ? VEX_R : 0;
6232     byte1 = (~byte1) & 0x80;
6233     byte1 |= ((~nds_enc) & 0xf) << 3;
6234     byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6235     emit_int8(byte1);
6236   }
6237 }
6238 
6239 // This is a 4 byte encoding
6240 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){



6241   // EVEX 0x62 prefix
6242   prefix(EVEX_4bytes);
6243   bool vex_w = _attributes->is_rex_vex_w();
6244   int evex_encoding = (vex_w ? VEX_W : 0);
6245   // EVEX.b is not currently used for broadcast of single element or data rounding modes
6246   _attributes->set_evex_encoding(evex_encoding);
6247 
6248   // P0: byte 2, initialized to RXBR`00mm
6249   // instead of not'd
6250   int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6251   byte2 = (~byte2) & 0xF0;
6252   // confine opc opcode extensions in mm bits to lower two bits
6253   // of form {0F, 0F_38, 0F_3A}
6254   byte2 |= opc;
6255   emit_int8(byte2);
6256 
6257   // P1: byte 3 as Wvvvv1pp
6258   int byte3 = ((~nds_enc) & 0xf) << 3;
6259   // p[10] is always 1
6260   byte3 |= EVEX_F;
6261   byte3 |= (vex_w & 1) << 7;
6262   // confine pre opcode extensions in pp bits to lower two bits
6263   // of form {66, F3, F2}
6264   byte3 |= pre;
6265   emit_int8(byte3);
6266 
6267   // P2: byte 4 as zL'Lbv'aaa
6268   int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6269   // EVEX.v` for extending EVEX.vvvv or VIDX
6270   byte4 |= (evex_v ? 0: EVEX_V);
6271   // third EXEC.b for broadcast actions
6272   byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6273   // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6274   byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6275   // last is EVEX.z for zero/merge actions
6276   byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6277   emit_int8(byte4);
6278 }
6279 
6280 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {

6281   bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6282   bool vex_b = adr.base_needs_rex();
6283   bool vex_x = adr.index_needs_rex();
6284   set_attributes(attributes);
6285   attributes->set_current_assembler(this);
6286 
6287   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6288   if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
6289     switch (attributes->get_vector_len()) {
6290     case AVX_128bit:
6291     case AVX_256bit:
6292       attributes->set_is_legacy_mode();
6293       break;
6294     }
6295   }
6296 
6297   if ((UseAVX > 2) && !attributes->is_legacy_mode())
6298   {
6299     bool evex_r = (xreg_enc >= 16);
6300     bool evex_v = (nds_enc >= 16);
6301     attributes->set_is_evex_instruction();
6302     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6303   } else {
6304     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6305   }

6306 }
6307 
6308 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {

6309   bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6310   bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6311   bool vex_x = false;
6312   set_attributes(attributes);
6313   attributes->set_current_assembler(this);
6314 
6315   // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6316   if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
6317     switch (attributes->get_vector_len()) {
6318     case AVX_128bit:
6319     case AVX_256bit:
6320       if ((dst_enc >= 16) | (nds_enc >= 16) | (src_enc >= 16)) {
6321         // up propagate arithmetic instructions to meet RA requirements
6322         attributes->set_vector_len(AVX_512bit);
6323       } else {
6324         attributes->set_is_legacy_mode();
6325       }
6326       break;
6327     }
6328   }
6329 
6330   if ((UseAVX > 2) && !attributes->is_legacy_mode())
6331   {
6332     bool evex_r = (dst_enc >= 16);
6333     bool evex_v = (nds_enc >= 16);
6334     // can use vex_x as bank extender on rm encoding
6335     vex_x = (src_enc >= 16);
6336     attributes->set_is_evex_instruction();
6337     evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6338   } else {
6339     vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6340   }
6341 


6342   // return modrm byte components for operands
6343   return (((dst_enc & 7) << 3) | (src_enc & 7));
6344 }
6345 
6346 
6347 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6348                             VexOpcode opc, InstructionAttr *attributes) {
6349   if (UseAVX > 0) {
6350     int xreg_enc = xreg->encoding();
6351     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6352     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
6353   } else {
6354     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6355     rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
6356   }
6357 }
6358 
6359 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6360                                       VexOpcode opc, InstructionAttr *attributes) {
6361   int dst_enc = dst->encoding();
6362   int src_enc = src->encoding();
6363   if (UseAVX > 0) {
6364     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6365     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
6366   } else {
6367     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6368     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
6369   }
6370 }
6371 
6372 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
6373                                       VexOpcode opc, InstructionAttr *attributes) {


6374   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6375   return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
6376 }
6377 
6378 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
6379                                       VexOpcode opc, InstructionAttr *attributes) {























































































6380   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6381   return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);


6382 }
6383 
6384 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
6385   assert(VM_Version::supports_avx(), "");
6386   assert(!VM_Version::supports_evex(), "");
6387   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6388   int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6389   emit_int8((unsigned char)0xC2);
6390   emit_int8((unsigned char)(0xC0 | encode));
6391   emit_int8((unsigned char)(0xF & cop));
6392 }
6393 
6394 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
6395   assert(VM_Version::supports_avx(), "");
6396   assert(!VM_Version::supports_evex(), "");
6397   InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6398   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6399   int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6400   emit_int8((unsigned char)0x4B);
6401   emit_int8((unsigned char)(0xC0 | encode));
6402   int src2_enc = src2->encoding();
6403   emit_int8((unsigned char)(0xF0 & src2_enc<<4));
6404 }
6405 
6406 
6407 #ifndef _LP64
6408 
6409 void Assembler::incl(Register dst) {
6410   // Don't use it directly. Use MacroAssembler::incrementl() instead.
6411   emit_int8(0x40 | dst->encoding());
6412 }
6413 
6414 void Assembler::lea(Register dst, Address src) {
6415   leal(dst, src);
6416 }
6417 
6418 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
6419   InstructionMark im(this);


6916 
6917 void Assembler::andq(Register dst, int32_t imm32) {
6918   (void) prefixq_and_encode(dst->encoding());
6919   emit_arith(0x81, 0xE0, dst, imm32);
6920 }
6921 
6922 void Assembler::andq(Register dst, Address src) {
6923   InstructionMark im(this);
6924   prefixq(src, dst);
6925   emit_int8(0x23);
6926   emit_operand(dst, src);
6927 }
6928 
6929 void Assembler::andq(Register dst, Register src) {
6930   (void) prefixq_and_encode(dst->encoding(), src->encoding());
6931   emit_arith(0x23, 0xC0, dst, src);
6932 }
6933 
6934 void Assembler::andnq(Register dst, Register src1, Register src2) {
6935   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6936   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6937   int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6938   emit_int8((unsigned char)0xF2);
6939   emit_int8((unsigned char)(0xC0 | encode));
6940 }
6941 
6942 void Assembler::andnq(Register dst, Register src1, Address src2) {

6943   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6944   InstructionMark im(this);
6945   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6946   vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6947   emit_int8((unsigned char)0xF2);
6948   emit_operand(dst, src2);
6949 }
6950 
6951 void Assembler::bsfq(Register dst, Register src) {
6952   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6953   emit_int8(0x0F);
6954   emit_int8((unsigned char)0xBC);
6955   emit_int8((unsigned char)(0xC0 | encode));
6956 }
6957 
6958 void Assembler::bsrq(Register dst, Register src) {
6959   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6960   emit_int8(0x0F);
6961   emit_int8((unsigned char)0xBD);
6962   emit_int8((unsigned char)(0xC0 | encode));
6963 }
6964 
6965 void Assembler::bswapq(Register reg) {
6966   int encode = prefixq_and_encode(reg->encoding());
6967   emit_int8(0x0F);
6968   emit_int8((unsigned char)(0xC8 | encode));
6969 }
6970 
6971 void Assembler::blsiq(Register dst, Register src) {
6972   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6973   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6974   int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6975   emit_int8((unsigned char)0xF3);
6976   emit_int8((unsigned char)(0xC0 | encode));
6977 }
6978 
6979 void Assembler::blsiq(Register dst, Address src) {

6980   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6981   InstructionMark im(this);
6982   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6983   vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6984   emit_int8((unsigned char)0xF3);
6985   emit_operand(rbx, src);
6986 }
6987 
6988 void Assembler::blsmskq(Register dst, Register src) {
6989   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6990   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6991   int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6992   emit_int8((unsigned char)0xF3);
6993   emit_int8((unsigned char)(0xC0 | encode));
6994 }
6995 
6996 void Assembler::blsmskq(Register dst, Address src) {

6997   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6998   InstructionMark im(this);
6999   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7000   vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7001   emit_int8((unsigned char)0xF3);
7002   emit_operand(rdx, src);
7003 }
7004 
7005 void Assembler::blsrq(Register dst, Register src) {
7006   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7007   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7008   int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7009   emit_int8((unsigned char)0xF3);
7010   emit_int8((unsigned char)(0xC0 | encode));
7011 }
7012 
7013 void Assembler::blsrq(Register dst, Address src) {

7014   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7015   InstructionMark im(this);
7016   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7017   vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7018   emit_int8((unsigned char)0xF3);
7019   emit_operand(rcx, src);
7020 }
7021 
7022 void Assembler::cdqq() {
7023   prefix(REX_W);
7024   emit_int8((unsigned char)0x99);
7025 }
7026 
7027 void Assembler::clflush(Address adr) {
7028   prefix(adr);
7029   emit_int8(0x0F);
7030   emit_int8((unsigned char)0xAE);
7031   emit_operand(rdi, adr);
7032 }
7033 
7034 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7035   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7036   emit_int8(0x0F);
7037   emit_int8(0x40 | cc);


7071   emit_arith(0x3B, 0xC0, dst, src);
7072 }
7073 
7074 void Assembler::cmpq(Register dst, Address  src) {
7075   InstructionMark im(this);
7076   prefixq(src, dst);
7077   emit_int8(0x3B);
7078   emit_operand(dst, src);
7079 }
7080 
7081 void Assembler::cmpxchgq(Register reg, Address adr) {
7082   InstructionMark im(this);
7083   prefixq(adr, reg);
7084   emit_int8(0x0F);
7085   emit_int8((unsigned char)0xB1);
7086   emit_operand(reg, adr);
7087 }
7088 
7089 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7090   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7091   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7092   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7093   emit_int8(0x2A);
7094   emit_int8((unsigned char)(0xC0 | encode));
7095 }
7096 
7097 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7098   NOT_LP64(assert(VM_Version::supports_sse2(), ""));




7099   InstructionMark im(this);
7100   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7101   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7102   simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7103   emit_int8(0x2A);
7104   emit_operand(dst, src);
7105 }
7106 
7107 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7108   NOT_LP64(assert(VM_Version::supports_sse(), ""));




7109   InstructionMark im(this);
7110   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7111   attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7112   simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7113   emit_int8(0x2A);
7114   emit_operand(dst, src);
7115 }
7116 
7117 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7118   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7119   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7120   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7121   emit_int8(0x2C);
7122   emit_int8((unsigned char)(0xC0 | encode));
7123 }
7124 
7125 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7126   NOT_LP64(assert(VM_Version::supports_sse(), ""));
7127   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7128   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7129   emit_int8(0x2C);
7130   emit_int8((unsigned char)(0xC0 | encode));
7131 }
7132 
7133 void Assembler::decl(Register dst) {
7134   // Don't use it directly. Use MacroAssembler::decrementl() instead.
7135   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7136   int encode = prefix_and_encode(dst->encoding());
7137   emit_int8((unsigned char)0xFF);
7138   emit_int8((unsigned char)(0xC8 | encode));
7139 }
7140 
7141 void Assembler::decq(Register dst) {
7142   // Don't use it directly. Use MacroAssembler::decrementq() instead.
7143   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7144   int encode = prefixq_and_encode(dst->encoding());
7145   emit_int8((unsigned char)0xFF);
7146   emit_int8(0xC8 | encode);
7147 }
7148 


7291 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7292   InstructionMark im(this);
7293   prefix(src1);
7294   emit_int8((unsigned char)0x81);
7295   emit_operand(rax, src1, 4);
7296   emit_data((int)imm32, rspec, narrow_oop_operand);
7297 }
7298 
7299 void Assembler::lzcntq(Register dst, Register src) {
7300   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7301   emit_int8((unsigned char)0xF3);
7302   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7303   emit_int8(0x0F);
7304   emit_int8((unsigned char)0xBD);
7305   emit_int8((unsigned char)(0xC0 | encode));
7306 }
7307 
7308 void Assembler::movdq(XMMRegister dst, Register src) {
7309   // table D-1 says MMX/SSE2
7310   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7311   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7312   int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7313   emit_int8(0x6E);
7314   emit_int8((unsigned char)(0xC0 | encode));
7315 }
7316 
7317 void Assembler::movdq(Register dst, XMMRegister src) {
7318   // table D-1 says MMX/SSE2
7319   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7320   InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7321   // swap src/dst to get correct prefix
7322   int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7323   emit_int8(0x7E);
7324   emit_int8((unsigned char)(0xC0 | encode));
7325 }
7326 
7327 void Assembler::movq(Register dst, Register src) {
7328   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7329   emit_int8((unsigned char)0x8B);
7330   emit_int8((unsigned char)(0xC0 | encode));
7331 }
7332 
7333 void Assembler::movq(Register dst, Address src) {
7334   InstructionMark im(this);
7335   prefixq(src, dst);
7336   emit_int8((unsigned char)0x8B);
7337   emit_operand(dst, src);
7338 }
7339 
7340 void Assembler::movq(Address dst, Register src) {
7341   InstructionMark im(this);
7342   prefixq(dst, src);


7435   emit_int8((unsigned char)0x0F);
7436   emit_int8((unsigned char)0xB7);
7437   emit_int8((unsigned char)(0xC0 | encode));
7438 }
7439 
7440 void Assembler::mulq(Address src) {
7441   InstructionMark im(this);
7442   prefixq(src);
7443   emit_int8((unsigned char)0xF7);
7444   emit_operand(rsp, src);
7445 }
7446 
7447 void Assembler::mulq(Register src) {
7448   int encode = prefixq_and_encode(src->encoding());
7449   emit_int8((unsigned char)0xF7);
7450   emit_int8((unsigned char)(0xE0 | encode));
7451 }
7452 
7453 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
7454   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7455   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7456   int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
7457   emit_int8((unsigned char)0xF6);
7458   emit_int8((unsigned char)(0xC0 | encode));
7459 }
7460 
7461 void Assembler::negq(Register dst) {
7462   int encode = prefixq_and_encode(dst->encoding());
7463   emit_int8((unsigned char)0xF7);
7464   emit_int8((unsigned char)(0xD8 | encode));
7465 }
7466 
7467 void Assembler::notq(Register dst) {
7468   int encode = prefixq_and_encode(dst->encoding());
7469   emit_int8((unsigned char)0xF7);
7470   emit_int8((unsigned char)(0xD0 | encode));
7471 }
7472 
7473 void Assembler::orq(Address dst, int32_t imm32) {
7474   InstructionMark im(this);
7475   prefixq(dst);
7476   emit_int8((unsigned char)0x81);


7598     emit_int8((unsigned char)(0xD8 | encode));
7599     emit_int8(imm8);
7600   }
7601 }
7602 
7603 void Assembler::rorq(Register dst, int imm8) {
7604   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7605   int encode = prefixq_and_encode(dst->encoding());
7606   if (imm8 == 1) {
7607     emit_int8((unsigned char)0xD1);
7608     emit_int8((unsigned char)(0xC8 | encode));
7609   } else {
7610     emit_int8((unsigned char)0xC1);
7611     emit_int8((unsigned char)(0xc8 | encode));
7612     emit_int8(imm8);
7613   }
7614 }
7615 
7616 void Assembler::rorxq(Register dst, Register src, int imm8) {
7617   assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7618   InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7619   int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
7620   emit_int8((unsigned char)0xF0);
7621   emit_int8((unsigned char)(0xC0 | encode));
7622   emit_int8(imm8);
7623 }
7624 
7625 void Assembler::sarq(Register dst, int imm8) {
7626   assert(isShiftCount(imm8 >> 1), "illegal shift count");
7627   int encode = prefixq_and_encode(dst->encoding());
7628   if (imm8 == 1) {
7629     emit_int8((unsigned char)0xD1);
7630     emit_int8((unsigned char)(0xF8 | encode));
7631   } else {
7632     emit_int8((unsigned char)0xC1);
7633     emit_int8((unsigned char)(0xF8 | encode));
7634     emit_int8(imm8);
7635   }
7636 }
7637 
7638 void Assembler::sarq(Register dst) {
7639   int encode = prefixq_and_encode(dst->encoding());


< prev index next >