2182 emit_int8(0x0F);
2183 emit_int8((unsigned char)0xBD);
2184 emit_int8((unsigned char)(0xC0 | encode));
2185 }
2186
2187 // Emit mfence instruction
2188 void Assembler::mfence() {
2189 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2190 emit_int8(0x0F);
2191 emit_int8((unsigned char)0xAE);
2192 emit_int8((unsigned char)0xF0);
2193 }
2194
2195 void Assembler::mov(Register dst, Register src) {
2196 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2197 }
2198
2199 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2200 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2201 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2202 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2203 attributes.set_rex_vex_w_reverted();
2204 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2205 emit_int8(0x28);
2206 emit_int8((unsigned char)(0xC0 | encode));
2207 }
2208
2209 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2210 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2211 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2212 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2213 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2214 emit_int8(0x28);
2215 emit_int8((unsigned char)(0xC0 | encode));
2216 }
2217
2218 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2219 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2220 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2221 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2222 emit_int8(0x16);
2223 emit_int8((unsigned char)(0xC0 | encode));
2224 }
2225
2226 void Assembler::movb(Register dst, Address src) {
2227 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2228 InstructionMark im(this);
2229 prefix(src, dst, true);
2230 emit_int8((unsigned char)0x8A);
2231 emit_operand(dst, src);
2232 }
2233
2234 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2235 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2236 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2237 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2238 attributes.set_rex_vex_w_reverted();
2239 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2240 emit_int8(0x12);
2448 InstructionMark im(this);
2449 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2450 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2451 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2452 emit_int8(0x6E);
2453 emit_operand(dst, src);
2454 }
2455
2456 void Assembler::movdl(Address dst, XMMRegister src) {
2457 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2458 InstructionMark im(this);
2459 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2460 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2461 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2462 emit_int8(0x7E);
2463 emit_operand(src, dst);
2464 }
2465
2466 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2467 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2468 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2469 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2470 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2471 emit_int8(0x6F);
2472 emit_int8((unsigned char)(0xC0 | encode));
2473 }
2474
2475 void Assembler::movdqa(XMMRegister dst, Address src) {
2476 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2477 InstructionMark im(this);
2478 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2479 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2480 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2481 emit_int8(0x6F);
2482 emit_operand(dst, src);
2483 }
2484
2485 void Assembler::movdqu(XMMRegister dst, Address src) {
2486 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2487 InstructionMark im(this);
2488 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2489 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2566 emit_operand(dst, src);
2567 }
2568
2569 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2570 assert(VM_Version::supports_evex(), "");
2571 assert(src != xnoreg, "sanity");
2572 InstructionMark im(this);
2573 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2574 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2575 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2576 attributes.set_is_evex_instruction();
2577 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2578 emit_int8(0x7F);
2579 emit_operand(src, dst);
2580 }
2581
2582 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2583 assert(VM_Version::supports_avx512vlbw(), "");
2584 assert(is_vector_masking(), ""); // For stub code use only
2585 InstructionMark im(this);
2586 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2587 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2588 attributes.set_embedded_opmask_register_specifier(mask);
2589 attributes.set_is_evex_instruction();
2590 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2591 emit_int8(0x6F);
2592 emit_operand(dst, src);
2593 }
2594
2595 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2596 assert(VM_Version::supports_evex(), "");
2597 InstructionMark im(this);
2598 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2599 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2600 attributes.set_is_evex_instruction();
2601 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2602 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2603 emit_int8(0x6F);
2604 emit_operand(dst, src);
2605 }
2606
2607 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2608 assert(is_vector_masking(), "");
2609 assert(VM_Version::supports_avx512vlbw(), "");
2610 InstructionMark im(this);
2611 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2612 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2613 attributes.set_embedded_opmask_register_specifier(mask);
2614 attributes.set_is_evex_instruction();
2615 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2616 emit_int8(0x6F);
2617 emit_operand(dst, src);
2618 }
2619
2620 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2621 assert(VM_Version::supports_evex(), "");
2622 assert(src != xnoreg, "sanity");
2623 InstructionMark im(this);
2624 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2625 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2626 attributes.set_is_evex_instruction();
2627 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2628 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2629 emit_int8(0x7F);
2630 emit_operand(src, dst);
2631 }
2735 InstructionMark im(this);
2736 prefix(dst);
2737 emit_int8((unsigned char)0xC7);
2738 emit_operand(rax, dst, 4);
2739 emit_int32(imm32);
2740 }
2741
2742 void Assembler::movl(Address dst, Register src) {
2743 InstructionMark im(this);
2744 prefix(dst, src);
2745 emit_int8((unsigned char)0x89);
2746 emit_operand(src, dst);
2747 }
2748
2749 // New cpus require to use movsd and movss to avoid partial register stall
2750 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2751 // The selection is done in MacroAssembler::movdbl() and movflt().
2752 void Assembler::movlpd(XMMRegister dst, Address src) {
2753 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2754 InstructionMark im(this);
2755 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2756 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2757 attributes.set_rex_vex_w_reverted();
2758 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2759 emit_int8(0x12);
2760 emit_operand(dst, src);
2761 }
2762
2763 void Assembler::movq( MMXRegister dst, Address src ) {
2764 assert( VM_Version::supports_mmx(), "" );
2765 emit_int8(0x0F);
2766 emit_int8(0x6F);
2767 emit_operand(dst, src);
2768 }
2769
2770 void Assembler::movq( Address dst, MMXRegister src ) {
2771 assert( VM_Version::supports_mmx(), "" );
2772 emit_int8(0x0F);
2773 emit_int8(0x7F);
2774 // workaround gcc (3.2.1-7a) bug
2775 // In that version of gcc with only an emit_operand(MMX, Address)
3495 assert(VM_Version::supports_avx(), "");
3496 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3497 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3498 emit_int8(0x74);
3499 emit_int8((unsigned char)(0xC0 | encode));
3500 }
3501
3502 // In this context, kdst is written the mask used to process the equal components
3503 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3504 assert(VM_Version::supports_avx512bw(), "");
3505 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3506 attributes.set_is_evex_instruction();
3507 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3508 emit_int8(0x74);
3509 emit_int8((unsigned char)(0xC0 | encode));
3510 }
3511
3512 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3513 assert(VM_Version::supports_avx512vlbw(), "");
3514 InstructionMark im(this);
3515 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3516 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3517 attributes.set_is_evex_instruction();
3518 int dst_enc = kdst->encoding();
3519 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3520 emit_int8(0x64);
3521 emit_operand(as_Register(dst_enc), src);
3522 }
3523
3524 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3525 assert(is_vector_masking(), "");
3526 assert(VM_Version::supports_avx512vlbw(), "");
3527 InstructionMark im(this);
3528 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3529 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3530 attributes.reset_is_clear_context();
3531 attributes.set_embedded_opmask_register_specifier(mask);
3532 attributes.set_is_evex_instruction();
3533 int dst_enc = kdst->encoding();
3534 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3535 emit_int8(0x64);
3536 emit_operand(as_Register(dst_enc), src);
3537 }
3538
3539 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3540 assert(VM_Version::supports_avx512vlbw(), "");
3541 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3542 attributes.set_is_evex_instruction();
3543 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3544 emit_int8(0x3E);
3545 emit_int8((unsigned char)(0xC0 | encode));
3546 emit_int8(vcc);
3547 }
3548
3549 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3550 assert(is_vector_masking(), "");
3551 assert(VM_Version::supports_avx512vlbw(), "");
3552 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3553 attributes.reset_is_clear_context();
3554 attributes.set_embedded_opmask_register_specifier(mask);
3555 attributes.set_is_evex_instruction();
3556 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3557 emit_int8(0x3E);
3558 emit_int8((unsigned char)(0xC0 | encode));
3559 emit_int8(vcc);
3560 }
3561
3562 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3563 assert(VM_Version::supports_avx512vlbw(), "");
3564 InstructionMark im(this);
3565 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3566 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3567 attributes.set_is_evex_instruction();
3568 int dst_enc = kdst->encoding();
3569 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3570 emit_int8(0x3E);
3571 emit_operand(as_Register(dst_enc), src);
3572 emit_int8(vcc);
3573 }
3574
3575 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3576 assert(VM_Version::supports_avx512bw(), "");
3577 InstructionMark im(this);
3578 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3579 attributes.set_is_evex_instruction();
3580 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3581 int dst_enc = kdst->encoding();
3582 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3583 emit_int8(0x74);
3584 emit_operand(as_Register(dst_enc), src);
3585 }
3586
3587 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3588 assert(VM_Version::supports_avx512vlbw(), "");
3589 assert(is_vector_masking(), ""); // For stub code use only
3590 InstructionMark im(this);
3591 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3592 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3593 attributes.reset_is_clear_context();
3594 attributes.set_embedded_opmask_register_specifier(mask);
3595 attributes.set_is_evex_instruction();
3596 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3597 emit_int8(0x74);
3598 emit_operand(as_Register(kdst->encoding()), src);
3599 }
3600
3601 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3602 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3603 assert(VM_Version::supports_sse2(), "");
3604 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3605 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3606 emit_int8(0x75);
3607 emit_int8((unsigned char)(0xC0 | encode));
3608 }
3609
3610 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3611 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3724 }
3725
3726 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3727 assert(VM_Version::supports_sse2(), "");
3728 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3729 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3730 emit_int8((unsigned char)0xD7);
3731 emit_int8((unsigned char)(0xC0 | encode));
3732 }
3733
3734 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3735 assert(VM_Version::supports_avx2(), "");
3736 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3737 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3738 emit_int8((unsigned char)0xD7);
3739 emit_int8((unsigned char)(0xC0 | encode));
3740 }
3741
3742 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3743 assert(VM_Version::supports_sse4_1(), "");
3744 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3745 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3746 emit_int8(0x16);
3747 emit_int8((unsigned char)(0xC0 | encode));
3748 emit_int8(imm8);
3749 }
3750
3751 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3752 assert(VM_Version::supports_sse4_1(), "");
3753 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3754 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3755 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3756 emit_int8(0x16);
3757 emit_operand(src, dst);
3758 emit_int8(imm8);
3759 }
3760
3761 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3762 assert(VM_Version::supports_sse4_1(), "");
3763 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3764 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3765 emit_int8(0x16);
3766 emit_int8((unsigned char)(0xC0 | encode));
3767 emit_int8(imm8);
3768 }
3769
3770 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3771 assert(VM_Version::supports_sse4_1(), "");
3772 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3773 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3774 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3775 emit_int8(0x16);
3776 emit_operand(src, dst);
3777 emit_int8(imm8);
3778 }
3779
3780 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3781 assert(VM_Version::supports_sse2(), "");
3782 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3783 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3784 emit_int8((unsigned char)0xC5);
3785 emit_int8((unsigned char)(0xC0 | encode));
3786 emit_int8(imm8);
3787 }
3788
3789 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3790 assert(VM_Version::supports_sse4_1(), "");
3791 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3792 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3793 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3794 emit_int8((unsigned char)0x15);
3795 emit_operand(src, dst);
3796 emit_int8(imm8);
3797 }
3798
3799 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3800 assert(VM_Version::supports_sse4_1(), "");
3801 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3802 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3803 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3804 emit_int8(0x14);
3805 emit_operand(src, dst);
3806 emit_int8(imm8);
3807 }
3808
3809 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3810 assert(VM_Version::supports_sse4_1(), "");
3811 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3812 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3813 emit_int8(0x22);
3814 emit_int8((unsigned char)(0xC0 | encode));
3815 emit_int8(imm8);
3816 }
3817
3818 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3819 assert(VM_Version::supports_sse4_1(), "");
3820 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3821 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3822 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3823 emit_int8(0x22);
3824 emit_operand(dst,src);
3825 emit_int8(imm8);
3826 }
3827
3828 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3829 assert(VM_Version::supports_sse4_1(), "");
3830 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3831 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3832 emit_int8(0x22);
3833 emit_int8((unsigned char)(0xC0 | encode));
3834 emit_int8(imm8);
3835 }
3836
3837 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3838 assert(VM_Version::supports_sse4_1(), "");
3839 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3840 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3841 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3842 emit_int8(0x22);
3843 emit_operand(dst, src);
3844 emit_int8(imm8);
3845 }
3846
3847 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3848 assert(VM_Version::supports_sse2(), "");
3849 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3850 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3851 emit_int8((unsigned char)0xC4);
3852 emit_int8((unsigned char)(0xC0 | encode));
3853 emit_int8(imm8);
3854 }
3855
3856 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3857 assert(VM_Version::supports_sse2(), "");
3858 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3859 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3860 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3861 emit_int8((unsigned char)0xC4);
3862 emit_operand(dst, src);
3863 emit_int8(imm8);
3864 }
3865
3866 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3867 assert(VM_Version::supports_sse4_1(), "");
3868 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3869 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3870 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3871 emit_int8(0x20);
3872 emit_operand(dst, src);
3873 emit_int8(imm8);
3874 }
3875
3876 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3877 assert(VM_Version::supports_sse4_1(), "");
3878 InstructionMark im(this);
3879 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3880 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3881 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3882 emit_int8(0x30);
3883 emit_operand(dst, src);
3884 }
3885
3886 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3887 assert(VM_Version::supports_sse4_1(), "");
3888 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3889 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3890 emit_int8(0x30);
3891 emit_int8((unsigned char)(0xC0 | encode));
3892 }
3893
3894 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3895 assert(VM_Version::supports_avx(), "");
3896 InstructionMark im(this);
3897 assert(dst != xnoreg, "sanity");
3898 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3899 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3900 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3901 emit_int8(0x30);
3902 emit_operand(dst, src);
3903 }
3904
3905 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3906 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3907 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3908 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3909 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3910 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3911 emit_int8(0x30);
3912 emit_int8((unsigned char) (0xC0 | encode));
3913 }
3914
3915
3916 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3917 assert(is_vector_masking(), "");
3918 assert(VM_Version::supports_avx512vlbw(), "");
3919 assert(dst != xnoreg, "sanity");
3920 InstructionMark im(this);
3921 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3922 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3923 attributes.set_embedded_opmask_register_specifier(mask);
3924 attributes.set_is_evex_instruction();
3925 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3926 emit_int8(0x30);
3927 emit_operand(dst, src);
3928 }
3929 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3930 assert(VM_Version::supports_avx512vlbw(), "");
3931 assert(src != xnoreg, "sanity");
3932 InstructionMark im(this);
3933 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3934 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3935 attributes.set_is_evex_instruction();
3936 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3937 emit_int8(0x30);
3938 emit_operand(src, dst);
3939 }
3940
3941 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3942 assert(is_vector_masking(), "");
3943 assert(VM_Version::supports_avx512vlbw(), "");
3944 assert(src != xnoreg, "sanity");
3945 InstructionMark im(this);
3946 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3947 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3948 attributes.reset_is_clear_context();
3949 attributes.set_embedded_opmask_register_specifier(mask);
3950 attributes.set_is_evex_instruction();
3951 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3952 emit_int8(0x30);
3953 emit_operand(src, dst);
3954 }
3955
3956 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
3957 assert(VM_Version::supports_evex(), "");
3958 assert(src != xnoreg, "sanity");
3959 InstructionMark im(this);
3960 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3961 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
3962 attributes.set_is_evex_instruction();
3963 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3964 emit_int8(0x31);
3965 emit_operand(src, dst);
3966 }
3967
3968 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
3969 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3970 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3971 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
3972 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3973 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3974 emit_int8(0x33);
3975 emit_int8((unsigned char)(0xC0 | encode));
3976 }
3977
3978 // generic
3979 void Assembler::pop(Register dst) {
3980 int encode = prefix_and_encode(dst->encoding());
3981 emit_int8(0x58 | encode);
3982 }
3983
3984 void Assembler::popcntl(Register dst, Address src) {
3985 assert(VM_Version::supports_popcnt(), "must support");
3986 InstructionMark im(this);
3987 emit_int8((unsigned char)0xF3);
3988 prefix(src, dst);
3989 emit_int8(0x0F);
3990 emit_int8((unsigned char)0xB8);
3991 emit_operand(dst, src);
3992 }
4065 InstructionMark im(this);
4066 prefetch_prefix(src);
4067 emit_int8(0x18);
4068 emit_operand(rbx, src); // 3, src
4069 }
4070
4071 void Assembler::prefetchw(Address src) {
4072 assert(VM_Version::supports_3dnow_prefetch(), "must support");
4073 InstructionMark im(this);
4074 prefetch_prefix(src);
4075 emit_int8(0x0D);
4076 emit_operand(rcx, src); // 1, src
4077 }
4078
4079 void Assembler::prefix(Prefix p) {
4080 emit_int8(p);
4081 }
4082
4083 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
4084 assert(VM_Version::supports_ssse3(), "");
4085 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4086 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4087 emit_int8(0x00);
4088 emit_int8((unsigned char)(0xC0 | encode));
4089 }
4090
4091 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4092 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4093 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4094 0, "");
4095 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4096 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4097 emit_int8(0x00);
4098 emit_int8((unsigned char)(0xC0 | encode));
4099 }
4100
4101 void Assembler::pshufb(XMMRegister dst, Address src) {
4102 assert(VM_Version::supports_ssse3(), "");
4103 InstructionMark im(this);
4104 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4105 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4106 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4107 emit_int8(0x00);
4108 emit_operand(dst, src);
4109 }
4110
4111 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4112 assert(isByte(mode), "invalid value");
4113 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4114 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4115 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4116 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4117 emit_int8(0x70);
4118 emit_int8((unsigned char)(0xC0 | encode));
4119 emit_int8(mode & 0xFF);
4120 }
4121
4122 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4123 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4124 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4130 emit_int8((unsigned char)(0xC0 | encode));
4131 emit_int8(mode & 0xFF);
4132 }
4133
4134 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4135 assert(isByte(mode), "invalid value");
4136 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4137 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4138 InstructionMark im(this);
4139 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4140 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4141 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4142 emit_int8(0x70);
4143 emit_operand(dst, src);
4144 emit_int8(mode & 0xFF);
4145 }
4146
4147 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4148 assert(isByte(mode), "invalid value");
4149 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4150 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4151 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4152 emit_int8(0x70);
4153 emit_int8((unsigned char)(0xC0 | encode));
4154 emit_int8(mode & 0xFF);
4155 }
4156
4157 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4158 assert(isByte(mode), "invalid value");
4159 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4160 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4161 InstructionMark im(this);
4162 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4163 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4164 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4165 emit_int8(0x70);
4166 emit_operand(dst, src);
4167 emit_int8(mode & 0xFF);
4168 }
4169 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4170 assert(VM_Version::supports_evex(), "requires EVEX support");
4171 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
4172 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4173 attributes.set_is_evex_instruction();
4174 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4175 emit_int8(0x43);
4176 emit_int8((unsigned char)(0xC0 | encode));
4177 emit_int8(imm8 & 0xFF);
4178 }
4179
4180 void Assembler::psrldq(XMMRegister dst, int shift) {
4181 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4182 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4183 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4184 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4185 emit_int8(0x73);
4186 emit_int8((unsigned char)(0xC0 | encode));
4187 emit_int8(shift);
4188 }
4189
4190 void Assembler::pslldq(XMMRegister dst, int shift) {
4191 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4192 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4193 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4194 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4195 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4196 emit_int8(0x73);
4197 emit_int8((unsigned char)(0xC0 | encode));
4198 emit_int8(shift);
4199 }
4200
4201 void Assembler::ptest(XMMRegister dst, Address src) {
4202 assert(VM_Version::supports_sse4_1(), "");
4203 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4204 InstructionMark im(this);
4205 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4206 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4207 emit_int8(0x17);
4208 emit_operand(dst, src);
4209 }
4210
4211 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4212 assert(VM_Version::supports_sse4_1(), "");
4213 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4439 prefix(src, dst);
4440 emit_int8(0x1B);
4441 emit_operand(dst, src);
4442 }
4443
4444 void Assembler::sbbl(Register dst, Register src) {
4445 (void) prefix_and_encode(dst->encoding(), src->encoding());
4446 emit_arith(0x1B, 0xC0, dst, src);
4447 }
4448
4449 void Assembler::setb(Condition cc, Register dst) {
4450 assert(0 <= cc && cc < 16, "illegal cc");
4451 int encode = prefix_and_encode(dst->encoding(), true);
4452 emit_int8(0x0F);
4453 emit_int8((unsigned char)0x90 | cc);
4454 emit_int8((unsigned char)(0xC0 | encode));
4455 }
4456
4457 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4458 assert(VM_Version::supports_ssse3(), "");
4459 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4460 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4461 emit_int8((unsigned char)0x0F);
4462 emit_int8((unsigned char)(0xC0 | encode));
4463 emit_int8(imm8);
4464 }
4465
4466 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4467 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4468 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4469 0, "");
4470 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4471 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4472 emit_int8((unsigned char)0x0F);
4473 emit_int8((unsigned char)(0xC0 | encode));
4474 emit_int8(imm8);
4475 }
4476
4477 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
4478 assert(VM_Version::supports_evex(), "");
4479 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4480 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4481 emit_int8(0x3);
4482 emit_int8((unsigned char)(0xC0 | encode));
4483 emit_int8(imm8);
4484 }
4485
4486 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4487 assert(VM_Version::supports_sse4_1(), "");
4488 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4489 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4490 emit_int8((unsigned char)0x0E);
4491 emit_int8((unsigned char)(0xC0 | encode));
4492 emit_int8(imm8);
4493 }
4494
4495 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4496 assert(VM_Version::supports_sha(), "");
4497 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4498 emit_int8((unsigned char)0xCC);
4499 emit_int8((unsigned char)(0xC0 | encode));
5574 attributes.set_rex_vex_w_reverted();
5575 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5576 emit_int8(0x57);
5577 emit_operand(dst, src);
5578 }
5579
5580 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5581 assert(VM_Version::supports_avx(), "");
5582 InstructionMark im(this);
5583 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5584 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5585 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5586 emit_int8(0x57);
5587 emit_operand(dst, src);
5588 }
5589
5590 // Integer vector arithmetic
5591 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5592 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5593 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5594 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5596 emit_int8(0x01);
5597 emit_int8((unsigned char)(0xC0 | encode));
5598 }
5599
5600 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5601 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5602 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5603 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5604 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5605 emit_int8(0x02);
5606 emit_int8((unsigned char)(0xC0 | encode));
5607 }
5608
5609 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5610 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5611 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5612 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5613 emit_int8((unsigned char)0xFC);
5614 emit_int8((unsigned char)(0xC0 | encode));
5615 }
5616
5617 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5618 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5619 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5620 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5621 emit_int8((unsigned char)0xFD);
5622 emit_int8((unsigned char)(0xC0 | encode));
5623 }
5633 void Assembler::paddd(XMMRegister dst, Address src) {
5634 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5635 InstructionMark im(this);
5636 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5637 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5638 emit_int8((unsigned char)0xFE);
5639 emit_operand(dst, src);
5640 }
5641
5642 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5643 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5644 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5645 attributes.set_rex_vex_w_reverted();
5646 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5647 emit_int8((unsigned char)0xD4);
5648 emit_int8((unsigned char)(0xC0 | encode));
5649 }
5650
5651 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5652 assert(VM_Version::supports_sse3(), "");
5653 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5654 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5655 emit_int8(0x01);
5656 emit_int8((unsigned char)(0xC0 | encode));
5657 }
5658
5659 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5660 assert(VM_Version::supports_sse3(), "");
5661 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5662 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5663 emit_int8(0x02);
5664 emit_int8((unsigned char)(0xC0 | encode));
5665 }
5666
5667 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5668 assert(UseAVX > 0, "requires some form of AVX");
5669 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5670 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5671 emit_int8((unsigned char)0xFC);
5672 emit_int8((unsigned char)(0xC0 | encode));
5673 }
5674
5675 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5676 assert(UseAVX > 0, "requires some form of AVX");
5677 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5678 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5679 emit_int8((unsigned char)0xFD);
5680 emit_int8((unsigned char)(0xC0 | encode));
5681 }
6352 }
6353
6354 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6355 assert(VM_Version::supports_evex(), "requires EVEX support");
6356 assert(dst != xnoreg, "sanity");
6357 InstructionMark im(this);
6358 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6359 attributes.set_is_evex_instruction();
6360 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6361 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6362 emit_int8((unsigned char)0xEF);
6363 emit_operand(dst, src);
6364 }
6365
6366
6367 // vinserti forms
6368
6369 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6370 assert(VM_Version::supports_avx2(), "");
6371 assert(imm8 <= 0x01, "imm8: %u", imm8);
6372 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6373 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6374 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6375 emit_int8(0x38);
6376 emit_int8((unsigned char)(0xC0 | encode));
6377 // 0x00 - insert into lower 128 bits
6378 // 0x01 - insert into upper 128 bits
6379 emit_int8(imm8 & 0x01);
6380 }
6381
6382 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6383 assert(VM_Version::supports_avx2(), "");
6384 assert(dst != xnoreg, "sanity");
6385 assert(imm8 <= 0x01, "imm8: %u", imm8);
6386 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6387 InstructionMark im(this);
6388 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6389 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6390 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6391 emit_int8(0x38);
6392 emit_operand(dst, src);
6393 // 0x00 - insert into lower 128 bits
6394 // 0x01 - insert into upper 128 bits
6395 emit_int8(imm8 & 0x01);
6396 }
6397
6398 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6399 assert(VM_Version::supports_evex(), "");
6400 assert(imm8 <= 0x03, "imm8: %u", imm8);
6401 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6402 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6403 emit_int8(0x38);
6404 emit_int8((unsigned char)(0xC0 | encode));
6405 // 0x00 - insert into q0 128 bits (0..127)
6406 // 0x01 - insert into q1 128 bits (128..255)
6407 // 0x02 - insert into q2 128 bits (256..383)
6408 // 0x03 - insert into q3 128 bits (384..511)
6409 emit_int8(imm8 & 0x03);
6410 }
6411
6412 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6413 assert(VM_Version::supports_avx(), "");
6414 assert(dst != xnoreg, "sanity");
6415 assert(imm8 <= 0x03, "imm8: %u", imm8);
6416 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6417 InstructionMark im(this);
6418 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6419 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6420 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6421 emit_int8(0x18);
6422 emit_operand(dst, src);
6423 // 0x00 - insert into q0 128 bits (0..127)
6424 // 0x01 - insert into q1 128 bits (128..255)
6425 // 0x02 - insert into q2 128 bits (256..383)
6426 // 0x03 - insert into q3 128 bits (384..511)
6427 emit_int8(imm8 & 0x03);
6428 }
6429
6430 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6431 assert(VM_Version::supports_evex(), "");
6432 assert(imm8 <= 0x01, "imm8: %u", imm8);
6433 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6434 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6435 emit_int8(0x38);
6436 emit_int8((unsigned char)(0xC0 | encode));
6437 // 0x00 - insert into lower 256 bits
6438 // 0x01 - insert into upper 256 bits
6439 emit_int8(imm8 & 0x01);
6440 }
6441
6442
6443 // vinsertf forms
6444
6445 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6446 assert(VM_Version::supports_avx(), "");
6447 assert(imm8 <= 0x01, "imm8: %u", imm8);
6448 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6449 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6450 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6451 emit_int8(0x18);
6452 emit_int8((unsigned char)(0xC0 | encode));
6453 // 0x00 - insert into lower 128 bits
6454 // 0x01 - insert into upper 128 bits
6455 emit_int8(imm8 & 0x01);
6456 }
6457
6458 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6459 assert(VM_Version::supports_avx(), "");
6460 assert(dst != xnoreg, "sanity");
6461 assert(imm8 <= 0x01, "imm8: %u", imm8);
6462 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6463 InstructionMark im(this);
6464 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6465 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6466 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6467 emit_int8(0x18);
6468 emit_operand(dst, src);
6469 // 0x00 - insert into lower 128 bits
6470 // 0x01 - insert into upper 128 bits
6471 emit_int8(imm8 & 0x01);
6472 }
6473
6474 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6475 assert(VM_Version::supports_evex(), "");
6476 assert(imm8 <= 0x03, "imm8: %u", imm8);
6477 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6478 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6479 emit_int8(0x18);
6480 emit_int8((unsigned char)(0xC0 | encode));
6481 // 0x00 - insert into q0 128 bits (0..127)
6482 // 0x01 - insert into q1 128 bits (128..255)
6483 // 0x02 - insert into q2 128 bits (256..383)
6484 // 0x03 - insert into q3 128 bits (384..511)
6485 emit_int8(imm8 & 0x03);
6486 }
6487
6488 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6489 assert(VM_Version::supports_avx(), "");
6490 assert(dst != xnoreg, "sanity");
6491 assert(imm8 <= 0x03, "imm8: %u", imm8);
6492 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6493 InstructionMark im(this);
6494 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6495 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6496 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6497 emit_int8(0x18);
6498 emit_operand(dst, src);
6499 // 0x00 - insert into q0 128 bits (0..127)
6500 // 0x01 - insert into q1 128 bits (128..255)
6501 // 0x02 - insert into q2 128 bits (256..383)
6502 // 0x03 - insert into q3 128 bits (384..511)
6503 emit_int8(imm8 & 0x03);
6504 }
6505
6506 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6507 assert(VM_Version::supports_evex(), "");
6508 assert(imm8 <= 0x01, "imm8: %u", imm8);
6509 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6510 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6511 emit_int8(0x1A);
6512 emit_int8((unsigned char)(0xC0 | encode));
6513 // 0x00 - insert into lower 256 bits
6514 // 0x01 - insert into upper 256 bits
6515 emit_int8(imm8 & 0x01);
6516 }
6517
6518 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6519 assert(VM_Version::supports_evex(), "");
6520 assert(dst != xnoreg, "sanity");
6521 assert(imm8 <= 0x01, "imm8: %u", imm8);
6522 InstructionMark im(this);
6523 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6524 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6525 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6526 emit_int8(0x1A);
6527 emit_operand(dst, src);
6528 // 0x00 - insert into lower 256 bits
6529 // 0x01 - insert into upper 256 bits
6530 emit_int8(imm8 & 0x01);
6531 }
6532
6533
6534 // vextracti forms
6535
6536 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6537 assert(VM_Version::supports_avx(), "");
6538 assert(imm8 <= 0x01, "imm8: %u", imm8);
6539 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6540 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6541 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6542 emit_int8(0x39);
6543 emit_int8((unsigned char)(0xC0 | encode));
6544 // 0x00 - extract from lower 128 bits
6545 // 0x01 - extract from upper 128 bits
6546 emit_int8(imm8 & 0x01);
6547 }
6548
6549 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6550 assert(VM_Version::supports_avx2(), "");
6551 assert(src != xnoreg, "sanity");
6552 assert(imm8 <= 0x01, "imm8: %u", imm8);
6553 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6554 InstructionMark im(this);
6555 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6556 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6557 attributes.reset_is_clear_context();
6558 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6559 emit_int8(0x39);
6560 emit_operand(src, dst);
6561 // 0x00 - extract from lower 128 bits
6562 // 0x01 - extract from upper 128 bits
6563 emit_int8(imm8 & 0x01);
6564 }
6565
6566 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6567 assert(VM_Version::supports_avx(), "");
6568 assert(imm8 <= 0x03, "imm8: %u", imm8);
6569 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6570 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6571 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6572 emit_int8(0x39);
6573 emit_int8((unsigned char)(0xC0 | encode));
6574 // 0x00 - extract from bits 127:0
6575 // 0x01 - extract from bits 255:128
6576 // 0x02 - extract from bits 383:256
6577 // 0x03 - extract from bits 511:384
6578 emit_int8(imm8 & 0x03);
6579 }
6580
6581 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6582 assert(VM_Version::supports_evex(), "");
6583 assert(src != xnoreg, "sanity");
6584 assert(imm8 <= 0x03, "imm8: %u", imm8);
6585 InstructionMark im(this);
6586 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6587 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6588 attributes.reset_is_clear_context();
6589 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6590 emit_int8(0x39);
6591 emit_operand(src, dst);
6592 // 0x00 - extract from bits 127:0
6593 // 0x01 - extract from bits 255:128
6594 // 0x02 - extract from bits 383:256
6595 // 0x03 - extract from bits 511:384
6596 emit_int8(imm8 & 0x03);
6597 }
6598
6599 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6600 assert(VM_Version::supports_avx512dq(), "");
6601 assert(imm8 <= 0x03, "imm8: %u", imm8);
6602 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6603 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6604 emit_int8(0x39);
6605 emit_int8((unsigned char)(0xC0 | encode));
6606 // 0x00 - extract from bits 127:0
6607 // 0x01 - extract from bits 255:128
6608 // 0x02 - extract from bits 383:256
6609 // 0x03 - extract from bits 511:384
6610 emit_int8(imm8 & 0x03);
6611 }
6612
6613 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6614 assert(VM_Version::supports_evex(), "");
6615 assert(imm8 <= 0x01, "imm8: %u", imm8);
6616 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6617 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6618 emit_int8(0x3B);
6619 emit_int8((unsigned char)(0xC0 | encode));
6620 // 0x00 - extract from lower 256 bits
6621 // 0x01 - extract from upper 256 bits
6622 emit_int8(imm8 & 0x01);
6623 }
6624
6625
6626 // vextractf forms
6627
6628 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6629 assert(VM_Version::supports_avx(), "");
6630 assert(imm8 <= 0x01, "imm8: %u", imm8);
6631 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6632 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6633 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6634 emit_int8(0x19);
6635 emit_int8((unsigned char)(0xC0 | encode));
6636 // 0x00 - extract from lower 128 bits
6637 // 0x01 - extract from upper 128 bits
6638 emit_int8(imm8 & 0x01);
6639 }
6640
6641 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6642 assert(VM_Version::supports_avx(), "");
6643 assert(src != xnoreg, "sanity");
6644 assert(imm8 <= 0x01, "imm8: %u", imm8);
6645 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6646 InstructionMark im(this);
6647 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6648 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6649 attributes.reset_is_clear_context();
6650 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6651 emit_int8(0x19);
6652 emit_operand(src, dst);
6653 // 0x00 - extract from lower 128 bits
6654 // 0x01 - extract from upper 128 bits
6655 emit_int8(imm8 & 0x01);
6656 }
6657
6658 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6659 assert(VM_Version::supports_avx(), "");
6660 assert(imm8 <= 0x03, "imm8: %u", imm8);
6661 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6662 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6663 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6664 emit_int8(0x19);
6665 emit_int8((unsigned char)(0xC0 | encode));
6666 // 0x00 - extract from bits 127:0
6667 // 0x01 - extract from bits 255:128
6668 // 0x02 - extract from bits 383:256
6669 // 0x03 - extract from bits 511:384
6670 emit_int8(imm8 & 0x03);
6671 }
6672
6673 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6674 assert(VM_Version::supports_evex(), "");
6675 assert(src != xnoreg, "sanity");
6676 assert(imm8 <= 0x03, "imm8: %u", imm8);
6677 InstructionMark im(this);
6678 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6679 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6680 attributes.reset_is_clear_context();
6681 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6682 emit_int8(0x19);
6683 emit_operand(src, dst);
6684 // 0x00 - extract from bits 127:0
6685 // 0x01 - extract from bits 255:128
6686 // 0x02 - extract from bits 383:256
6687 // 0x03 - extract from bits 511:384
6688 emit_int8(imm8 & 0x03);
6689 }
6690
6691 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6692 assert(VM_Version::supports_avx512dq(), "");
6693 assert(imm8 <= 0x03, "imm8: %u", imm8);
6694 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6695 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6696 emit_int8(0x19);
6697 emit_int8((unsigned char)(0xC0 | encode));
6698 // 0x00 - extract from bits 127:0
6699 // 0x01 - extract from bits 255:128
6700 // 0x02 - extract from bits 383:256
6701 // 0x03 - extract from bits 511:384
6702 emit_int8(imm8 & 0x03);
6703 }
6704
6705 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6706 assert(VM_Version::supports_evex(), "");
6707 assert(imm8 <= 0x01, "imm8: %u", imm8);
6708 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6709 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6710 emit_int8(0x1B);
6711 emit_int8((unsigned char)(0xC0 | encode));
6712 // 0x00 - extract from lower 256 bits
6713 // 0x01 - extract from upper 256 bits
6714 emit_int8(imm8 & 0x01);
6715 }
6716
6717 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6718 assert(VM_Version::supports_evex(), "");
6719 assert(src != xnoreg, "sanity");
6720 assert(imm8 <= 0x01, "imm8: %u", imm8);
6721 InstructionMark im(this);
6722 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6723 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
6724 attributes.reset_is_clear_context();
6725 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6726 emit_int8(0x1B);
6727 emit_operand(src, dst);
6728 // 0x00 - extract from lower 256 bits
6729 // 0x01 - extract from upper 256 bits
6730 emit_int8(imm8 & 0x01);
6731 }
6732
6733
6734 // legacy word/dword replicate
6735 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6736 assert(VM_Version::supports_avx2(), "");
6737 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6738 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6739 emit_int8(0x79);
6740 emit_int8((unsigned char)(0xC0 | encode));
6741 }
6742
6743 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6744 assert(VM_Version::supports_avx2(), "");
6745 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6746 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6747 emit_int8(0x58);
6748 emit_int8((unsigned char)(0xC0 | encode));
6749 }
6750
6751
6752 // xmm/mem sourced byte/word/dword/qword replicate
6753
6754 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6755 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6756 assert(VM_Version::supports_evex(), "");
6757 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6758 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6759 emit_int8(0x78);
6760 emit_int8((unsigned char)(0xC0 | encode));
6761 }
6762
6763 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6764 assert(VM_Version::supports_evex(), "");
6765 assert(dst != xnoreg, "sanity");
6766 InstructionMark im(this);
6767 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6768 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6769 // swap src<->dst for encoding
6770 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6771 emit_int8(0x78);
6772 emit_operand(dst, src);
6773 }
6774
6775 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6776 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6777 assert(VM_Version::supports_evex(), "");
6778 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6779 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6780 emit_int8(0x79);
6781 emit_int8((unsigned char)(0xC0 | encode));
6782 }
6783
6784 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6785 assert(VM_Version::supports_evex(), "");
6786 assert(dst != xnoreg, "sanity");
6787 InstructionMark im(this);
6788 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6789 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6790 // swap src<->dst for encoding
6791 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6792 emit_int8(0x79);
6793 emit_operand(dst, src);
6794 }
6795
6796 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6797 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6798 assert(VM_Version::supports_evex(), "");
6799 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6800 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6801 emit_int8(0x58);
6802 emit_int8((unsigned char)(0xC0 | encode));
6803 }
6804
6805 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6806 assert(VM_Version::supports_evex(), "");
6807 assert(dst != xnoreg, "sanity");
6808 InstructionMark im(this);
6809 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6810 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6811 // swap src<->dst for encoding
6812 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6813 emit_int8(0x58);
6814 emit_operand(dst, src);
6815 }
6816
6817 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6818 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6819 assert(VM_Version::supports_evex(), "");
6820 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6821 attributes.set_rex_vex_w_reverted();
6822 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6823 emit_int8(0x59);
6824 emit_int8((unsigned char)(0xC0 | encode));
6825 }
6826
6827 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6828 assert(VM_Version::supports_evex(), "");
6829 assert(dst != xnoreg, "sanity");
6830 InstructionMark im(this);
6831 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6832 attributes.set_rex_vex_w_reverted();
6833 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6834 // swap src<->dst for encoding
6835 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6836 emit_int8(0x59);
6837 emit_operand(dst, src);
6838 }
6839 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
6840 assert(vector_len != Assembler::AVX_128bit, "");
6841 assert(VM_Version::supports_avx512dq(), "");
6842 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6843 attributes.set_rex_vex_w_reverted();
6844 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6845 emit_int8(0x5A);
6846 emit_int8((unsigned char)(0xC0 | encode));
6847 }
6848
6849 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
6850 assert(vector_len != Assembler::AVX_128bit, "");
6851 assert(VM_Version::supports_avx512dq(), "");
6852 assert(dst != xnoreg, "sanity");
6853 InstructionMark im(this);
6854 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6855 attributes.set_rex_vex_w_reverted();
6856 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
6857 // swap src<->dst for encoding
6858 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6859 emit_int8(0x5A);
6860 emit_operand(dst, src);
6861 }
6862
6863 // scalar single/double precision replicate
6864
6865 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6866 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6867 assert(VM_Version::supports_evex(), "");
6868 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6869 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6870 emit_int8(0x18);
6871 emit_int8((unsigned char)(0xC0 | encode));
6872 }
6873
6874 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6875 assert(VM_Version::supports_evex(), "");
6876 assert(dst != xnoreg, "sanity");
6877 InstructionMark im(this);
6878 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6879 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6880 // swap src<->dst for encoding
6881 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6882 emit_int8(0x18);
6883 emit_operand(dst, src);
6884 }
6885
6886 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6887 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6888 assert(VM_Version::supports_evex(), "");
6889 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6890 attributes.set_rex_vex_w_reverted();
6891 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6892 emit_int8(0x19);
6893 emit_int8((unsigned char)(0xC0 | encode));
6894 }
6895
6896 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6897 assert(VM_Version::supports_evex(), "");
6898 assert(dst != xnoreg, "sanity");
6899 InstructionMark im(this);
6900 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6901 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6902 attributes.set_rex_vex_w_reverted();
6903 // swap src<->dst for encoding
6904 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6905 emit_int8(0x19);
6906 emit_operand(dst, src);
6907 }
6908
6909
6910 // gpr source broadcast forms
6911
6912 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6913 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6914 assert(VM_Version::supports_evex(), "");
6915 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6916 attributes.set_is_evex_instruction();
6917 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6918 emit_int8(0x7A);
6919 emit_int8((unsigned char)(0xC0 | encode));
6920 }
6921
6922 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6923 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6924 assert(VM_Version::supports_evex(), "");
6925 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6926 attributes.set_is_evex_instruction();
6927 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6928 emit_int8(0x7B);
6929 emit_int8((unsigned char)(0xC0 | encode));
6930 }
6931
6932 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6933 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6934 assert(VM_Version::supports_evex(), "");
6935 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6936 attributes.set_is_evex_instruction();
6937 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6938 emit_int8(0x7C);
6939 emit_int8((unsigned char)(0xC0 | encode));
6940 }
6941
6942 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6943 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6944 assert(VM_Version::supports_evex(), "");
6950 }
6951
6952 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
6953 assert(VM_Version::supports_evex(), "");
6954 assert(dst != xnoreg, "sanity");
6955 InstructionMark im(this);
6956 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6957 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6958 attributes.reset_is_clear_context();
6959 attributes.set_embedded_opmask_register_specifier(mask);
6960 attributes.set_is_evex_instruction();
6961 // swap src<->dst for encoding
6962 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6963 emit_int8((unsigned char)0x90);
6964 emit_operand(dst, src);
6965 }
6966
6967 // Carry-Less Multiplication Quadword
6968 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6969 assert(VM_Version::supports_clmul(), "");
6970 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6971 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6972 emit_int8(0x44);
6973 emit_int8((unsigned char)(0xC0 | encode));
6974 emit_int8((unsigned char)mask);
6975 }
6976
6977 // Carry-Less Multiplication Quadword
6978 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6979 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6980 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6981 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6982 emit_int8(0x44);
6983 emit_int8((unsigned char)(0xC0 | encode));
6984 emit_int8((unsigned char)mask);
6985 }
6986
6987 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
6988 assert(VM_Version::supports_vpclmulqdq(), "Requires vector carryless multiplication support");
6989 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6990 attributes.set_is_evex_instruction();
6991 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6992 emit_int8(0x44);
6993 emit_int8((unsigned char)(0xC0 | encode));
6994 emit_int8((unsigned char)mask);
6995 }
6996
6997 void Assembler::vzeroupper() {
6998 if (VM_Version::supports_vzeroupper()) {
6999 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7000 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7580 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7581 // last is EVEX.z for zero/merge actions
7582 if (_attributes->is_no_reg_mask() == false) {
7583 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7584 }
7585 emit_int8(byte4);
7586 }
7587
7588 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7589 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7590 bool vex_b = adr.base_needs_rex();
7591 bool vex_x;
7592 if (adr.isxmmindex()) {
7593 vex_x = adr.xmmindex_needs_rex();
7594 } else {
7595 vex_x = adr.index_needs_rex();
7596 }
7597 set_attributes(attributes);
7598 attributes->set_current_assembler(this);
7599
7600 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7601 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7602 switch (attributes->get_vector_len()) {
7603 case AVX_128bit:
7604 case AVX_256bit:
7605 attributes->set_is_legacy_mode();
7606 break;
7607 }
7608 }
7609
7610 // For pure EVEX check and see if this instruction
7611 // is allowed in legacy mode and has resources which will
7612 // fit in it. Pure EVEX instructions will use set_is_evex_instruction in their definition,
7613 // else that field is set when we encode to EVEX
7614 if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7615 !_is_managed && !attributes->is_evex_instruction()) {
7616 if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7617 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7618 if (check_register_bank) {
7619 // check nds_enc and xreg_enc for upper bank usage
7620 if (nds_enc < 16 && xreg_enc < 16) {
7621 attributes->set_is_legacy_mode();
7622 }
7623 } else {
7624 attributes->set_is_legacy_mode();
7625 }
7626 }
7627 }
7628
7629 _is_managed = false;
7630 if (UseAVX > 2 && !attributes->is_legacy_mode())
7631 {
7632 bool evex_r = (xreg_enc >= 16);
7633 bool evex_v;
7634 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
7635 if (adr.isxmmindex()) {
7636 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
7637 } else {
7638 evex_v = (nds_enc >= 16);
7639 }
7640 attributes->set_is_evex_instruction();
7641 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7642 } else {
7643 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7644 attributes->set_rex_vex_w(false);
7645 }
7646 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7647 }
7648 }
7649
7650 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7651 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7652 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7653 bool vex_x = false;
7654 set_attributes(attributes);
7655 attributes->set_current_assembler(this);
7656 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7657
7658 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7659 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7660 switch (attributes->get_vector_len()) {
7661 case AVX_128bit:
7662 case AVX_256bit:
7663 if (check_register_bank) {
7664 if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7665 // up propagate arithmetic instructions to meet RA requirements
7666 attributes->set_vector_len(AVX_512bit);
7667 } else {
7668 attributes->set_is_legacy_mode();
7669 }
7670 } else {
7671 attributes->set_is_legacy_mode();
7672 }
7673 break;
7674 }
7675 }
7676
7677 // For pure EVEX check and see if this instruction
7678 // is allowed in legacy mode and has resources which will
7679 // fit in it. Pure EVEX instructions will use set_is_evex_instruction in their definition,
7680 // else that field is set when we encode to EVEX
7681 if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7682 !_is_managed && !attributes->is_evex_instruction()) {
7683 if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7684 if (check_register_bank) {
7685 // check dst_enc, nds_enc and src_enc for upper bank usage
7686 if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7687 attributes->set_is_legacy_mode();
7688 }
7689 } else {
7690 attributes->set_is_legacy_mode();
7691 }
7692 }
7693 }
7694
7695 _is_managed = false;
7696 if (UseAVX > 2 && !attributes->is_legacy_mode())
7697 {
7698 bool evex_r = (dst_enc >= 16);
7699 bool evex_v = (nds_enc >= 16);
7700 // can use vex_x as bank extender on rm encoding
7701 vex_x = (src_enc >= 16);
7702 attributes->set_is_evex_instruction();
7703 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7704 } else {
7705 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7706 attributes->set_rex_vex_w(false);
7707 }
7708 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7709 }
7710
7711 // return modrm byte components for operands
7712 return (((dst_enc & 7) << 3) | (src_enc & 7));
7724 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7725 }
7726 }
7727
7728 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7729 VexOpcode opc, InstructionAttr *attributes) {
7730 int dst_enc = dst->encoding();
7731 int src_enc = src->encoding();
7732 if (UseAVX > 0) {
7733 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7734 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7735 } else {
7736 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7737 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7738 }
7739 }
7740
7741 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7742 assert(VM_Version::supports_avx(), "");
7743 assert(!VM_Version::supports_evex(), "");
7744 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7745 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7746 emit_int8((unsigned char)0xC2);
7747 emit_int8((unsigned char)(0xC0 | encode));
7748 emit_int8((unsigned char)(0xF & cop));
7749 }
7750
7751 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7752 assert(VM_Version::supports_avx(), "");
7753 assert(!VM_Version::supports_evex(), "");
7754 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7755 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7756 emit_int8((unsigned char)0x4B);
7757 emit_int8((unsigned char)(0xC0 | encode));
7758 int src2_enc = src2->encoding();
7759 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7760 }
7761
7762 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7763 assert(VM_Version::supports_avx(), "");
7764 assert(!VM_Version::supports_evex(), "");
7765 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7766 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7767 emit_int8((unsigned char)0xC2);
7768 emit_int8((unsigned char)(0xC0 | encode));
7769 emit_int8((unsigned char)(0xF & cop));
7770 }
7771
7772 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7773 assert(VM_Version::supports_avx(), "");
7774 assert(!VM_Version::supports_evex(), "");
7775 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7776 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7777 emit_int8((unsigned char)0x4A);
7778 emit_int8((unsigned char)(0xC0 | encode));
7779 int src2_enc = src2->encoding();
7780 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7781 }
7782
7783 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7784 assert(VM_Version::supports_avx2(), "");
7785 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7786 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7787 emit_int8((unsigned char)0x02);
7788 emit_int8((unsigned char)(0xC0 | encode));
7789 emit_int8((unsigned char)imm8);
7790 }
7791
7792 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7793 assert(VM_Version::supports_bmi2(), "");
7794 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7795 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7796 emit_int8((unsigned char)0xF7);
7797 emit_int8((unsigned char)(0xC0 | encode));
7798 }
7799
7800 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7801 assert(VM_Version::supports_bmi2(), "");
7802 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7803 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7804 emit_int8((unsigned char)0xF7);
7805 emit_int8((unsigned char)(0xC0 | encode));
7806 }
7807
7808 #ifndef _LP64
7809
7810 void Assembler::incl(Register dst) {
7811 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7812 emit_int8(0x40 | dst->encoding());
7813 }
7814
7815 void Assembler::lea(Register dst, Address src) {
7816 leal(dst, src);
7817 }
7818
7819 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7820 InstructionMark im(this);
7821 emit_int8((unsigned char)0xC7);
7822 emit_operand(rax, dst);
|
2182 emit_int8(0x0F);
2183 emit_int8((unsigned char)0xBD);
2184 emit_int8((unsigned char)(0xC0 | encode));
2185 }
2186
2187 // Emit mfence instruction
2188 void Assembler::mfence() {
2189 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2190 emit_int8(0x0F);
2191 emit_int8((unsigned char)0xAE);
2192 emit_int8((unsigned char)0xF0);
2193 }
2194
2195 void Assembler::mov(Register dst, Register src) {
2196 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2197 }
2198
2199 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2200 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2201 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2202 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2203 attributes.set_rex_vex_w_reverted();
2204 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2205 emit_int8(0x28);
2206 emit_int8((unsigned char)(0xC0 | encode));
2207 }
2208
2209 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2210 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2211 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2212 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2213 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2214 emit_int8(0x28);
2215 emit_int8((unsigned char)(0xC0 | encode));
2216 }
2217
2218 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2219 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2220 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2221 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2222 emit_int8(0x16);
2223 emit_int8((unsigned char)(0xC0 | encode));
2224 }
2225
2226 void Assembler::movb(Register dst, Address src) {
2227 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2228 InstructionMark im(this);
2229 prefix(src, dst, true);
2230 emit_int8((unsigned char)0x8A);
2231 emit_operand(dst, src);
2232 }
2233
2234 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2235 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2236 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2237 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2238 attributes.set_rex_vex_w_reverted();
2239 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2240 emit_int8(0x12);
2448 InstructionMark im(this);
2449 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2450 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2451 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2452 emit_int8(0x6E);
2453 emit_operand(dst, src);
2454 }
2455
2456 void Assembler::movdl(Address dst, XMMRegister src) {
2457 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2458 InstructionMark im(this);
2459 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2460 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2461 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2462 emit_int8(0x7E);
2463 emit_operand(src, dst);
2464 }
2465
2466 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2467 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2468 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2469 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2470 emit_int8(0x6F);
2471 emit_int8((unsigned char)(0xC0 | encode));
2472 }
2473
2474 void Assembler::movdqa(XMMRegister dst, Address src) {
2475 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2476 InstructionMark im(this);
2477 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2478 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2479 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2480 emit_int8(0x6F);
2481 emit_operand(dst, src);
2482 }
2483
2484 void Assembler::movdqu(XMMRegister dst, Address src) {
2485 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2486 InstructionMark im(this);
2487 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2488 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2565 emit_operand(dst, src);
2566 }
2567
2568 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2569 assert(VM_Version::supports_evex(), "");
2570 assert(src != xnoreg, "sanity");
2571 InstructionMark im(this);
2572 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2573 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2574 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2575 attributes.set_is_evex_instruction();
2576 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2577 emit_int8(0x7F);
2578 emit_operand(src, dst);
2579 }
2580
2581 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2582 assert(VM_Version::supports_avx512vlbw(), "");
2583 assert(is_vector_masking(), ""); // For stub code use only
2584 InstructionMark im(this);
2585 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2586 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2587 attributes.set_embedded_opmask_register_specifier(mask);
2588 attributes.set_is_evex_instruction();
2589 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2590 emit_int8(0x6F);
2591 emit_operand(dst, src);
2592 }
2593
2594 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2595 assert(VM_Version::supports_evex(), "");
2596 InstructionMark im(this);
2597 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2598 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2599 attributes.set_is_evex_instruction();
2600 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2601 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2602 emit_int8(0x6F);
2603 emit_operand(dst, src);
2604 }
2605
2606 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2607 assert(is_vector_masking(), "");
2608 assert(VM_Version::supports_avx512vlbw(), "");
2609 InstructionMark im(this);
2610 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2611 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2612 attributes.set_embedded_opmask_register_specifier(mask);
2613 attributes.set_is_evex_instruction();
2614 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2615 emit_int8(0x6F);
2616 emit_operand(dst, src);
2617 }
2618
2619 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2620 assert(VM_Version::supports_evex(), "");
2621 assert(src != xnoreg, "sanity");
2622 InstructionMark im(this);
2623 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2624 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2625 attributes.set_is_evex_instruction();
2626 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2627 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2628 emit_int8(0x7F);
2629 emit_operand(src, dst);
2630 }
2734 InstructionMark im(this);
2735 prefix(dst);
2736 emit_int8((unsigned char)0xC7);
2737 emit_operand(rax, dst, 4);
2738 emit_int32(imm32);
2739 }
2740
2741 void Assembler::movl(Address dst, Register src) {
2742 InstructionMark im(this);
2743 prefix(dst, src);
2744 emit_int8((unsigned char)0x89);
2745 emit_operand(src, dst);
2746 }
2747
2748 // New cpus require to use movsd and movss to avoid partial register stall
2749 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2750 // The selection is done in MacroAssembler::movdbl() and movflt().
2751 void Assembler::movlpd(XMMRegister dst, Address src) {
2752 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2753 InstructionMark im(this);
2754 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2755 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2756 attributes.set_rex_vex_w_reverted();
2757 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2758 emit_int8(0x12);
2759 emit_operand(dst, src);
2760 }
2761
2762 void Assembler::movq( MMXRegister dst, Address src ) {
2763 assert( VM_Version::supports_mmx(), "" );
2764 emit_int8(0x0F);
2765 emit_int8(0x6F);
2766 emit_operand(dst, src);
2767 }
2768
2769 void Assembler::movq( Address dst, MMXRegister src ) {
2770 assert( VM_Version::supports_mmx(), "" );
2771 emit_int8(0x0F);
2772 emit_int8(0x7F);
2773 // workaround gcc (3.2.1-7a) bug
2774 // In that version of gcc with only an emit_operand(MMX, Address)
3494 assert(VM_Version::supports_avx(), "");
3495 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3496 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3497 emit_int8(0x74);
3498 emit_int8((unsigned char)(0xC0 | encode));
3499 }
3500
3501 // In this context, kdst is written the mask used to process the equal components
3502 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3503 assert(VM_Version::supports_avx512bw(), "");
3504 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3505 attributes.set_is_evex_instruction();
3506 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3507 emit_int8(0x74);
3508 emit_int8((unsigned char)(0xC0 | encode));
3509 }
3510
3511 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3512 assert(VM_Version::supports_avx512vlbw(), "");
3513 InstructionMark im(this);
3514 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3515 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3516 attributes.set_is_evex_instruction();
3517 int dst_enc = kdst->encoding();
3518 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3519 emit_int8(0x64);
3520 emit_operand(as_Register(dst_enc), src);
3521 }
3522
3523 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3524 assert(is_vector_masking(), "");
3525 assert(VM_Version::supports_avx512vlbw(), "");
3526 InstructionMark im(this);
3527 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3528 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3529 attributes.reset_is_clear_context();
3530 attributes.set_embedded_opmask_register_specifier(mask);
3531 attributes.set_is_evex_instruction();
3532 int dst_enc = kdst->encoding();
3533 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3534 emit_int8(0x64);
3535 emit_operand(as_Register(dst_enc), src);
3536 }
3537
3538 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3539 assert(VM_Version::supports_avx512vlbw(), "");
3540 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3541 attributes.set_is_evex_instruction();
3542 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3543 emit_int8(0x3E);
3544 emit_int8((unsigned char)(0xC0 | encode));
3545 emit_int8(vcc);
3546 }
3547
3548 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3549 assert(is_vector_masking(), "");
3550 assert(VM_Version::supports_avx512vlbw(), "");
3551 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3552 attributes.reset_is_clear_context();
3553 attributes.set_embedded_opmask_register_specifier(mask);
3554 attributes.set_is_evex_instruction();
3555 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3556 emit_int8(0x3E);
3557 emit_int8((unsigned char)(0xC0 | encode));
3558 emit_int8(vcc);
3559 }
3560
3561 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3562 assert(VM_Version::supports_avx512vlbw(), "");
3563 InstructionMark im(this);
3564 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3565 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3566 attributes.set_is_evex_instruction();
3567 int dst_enc = kdst->encoding();
3568 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3569 emit_int8(0x3E);
3570 emit_operand(as_Register(dst_enc), src);
3571 emit_int8(vcc);
3572 }
3573
3574 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3575 assert(VM_Version::supports_avx512bw(), "");
3576 InstructionMark im(this);
3577 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3578 attributes.set_is_evex_instruction();
3579 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3580 int dst_enc = kdst->encoding();
3581 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3582 emit_int8(0x74);
3583 emit_operand(as_Register(dst_enc), src);
3584 }
3585
3586 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3587 assert(VM_Version::supports_avx512vlbw(), "");
3588 assert(is_vector_masking(), ""); // For stub code use only
3589 InstructionMark im(this);
3590 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ true);
3591 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3592 attributes.reset_is_clear_context();
3593 attributes.set_embedded_opmask_register_specifier(mask);
3594 attributes.set_is_evex_instruction();
3595 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3596 emit_int8(0x74);
3597 emit_operand(as_Register(kdst->encoding()), src);
3598 }
3599
3600 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3601 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3602 assert(VM_Version::supports_sse2(), "");
3603 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3604 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3605 emit_int8(0x75);
3606 emit_int8((unsigned char)(0xC0 | encode));
3607 }
3608
3609 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3610 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3723 }
3724
3725 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3726 assert(VM_Version::supports_sse2(), "");
3727 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3728 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3729 emit_int8((unsigned char)0xD7);
3730 emit_int8((unsigned char)(0xC0 | encode));
3731 }
3732
3733 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3734 assert(VM_Version::supports_avx2(), "");
3735 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3736 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3737 emit_int8((unsigned char)0xD7);
3738 emit_int8((unsigned char)(0xC0 | encode));
3739 }
3740
3741 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3742 assert(VM_Version::supports_sse4_1(), "");
3743 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3744 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3745 emit_int8(0x16);
3746 emit_int8((unsigned char)(0xC0 | encode));
3747 emit_int8(imm8);
3748 }
3749
3750 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3751 assert(VM_Version::supports_sse4_1(), "");
3752 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3753 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3754 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3755 emit_int8(0x16);
3756 emit_operand(src, dst);
3757 emit_int8(imm8);
3758 }
3759
3760 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3761 assert(VM_Version::supports_sse4_1(), "");
3762 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3763 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3764 emit_int8(0x16);
3765 emit_int8((unsigned char)(0xC0 | encode));
3766 emit_int8(imm8);
3767 }
3768
3769 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3770 assert(VM_Version::supports_sse4_1(), "");
3771 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3772 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3773 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3774 emit_int8(0x16);
3775 emit_operand(src, dst);
3776 emit_int8(imm8);
3777 }
3778
3779 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3780 assert(VM_Version::supports_sse2(), "");
3781 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3782 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3783 emit_int8((unsigned char)0xC5);
3784 emit_int8((unsigned char)(0xC0 | encode));
3785 emit_int8(imm8);
3786 }
3787
3788 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3789 assert(VM_Version::supports_sse4_1(), "");
3790 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3791 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3792 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3793 emit_int8((unsigned char)0x15);
3794 emit_operand(src, dst);
3795 emit_int8(imm8);
3796 }
3797
3798 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3799 assert(VM_Version::supports_sse4_1(), "");
3800 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3801 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3802 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3803 emit_int8(0x14);
3804 emit_operand(src, dst);
3805 emit_int8(imm8);
3806 }
3807
3808 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3809 assert(VM_Version::supports_sse4_1(), "");
3810 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3811 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3812 emit_int8(0x22);
3813 emit_int8((unsigned char)(0xC0 | encode));
3814 emit_int8(imm8);
3815 }
3816
3817 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3818 assert(VM_Version::supports_sse4_1(), "");
3819 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3820 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3821 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3822 emit_int8(0x22);
3823 emit_operand(dst,src);
3824 emit_int8(imm8);
3825 }
3826
3827 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3828 assert(VM_Version::supports_sse4_1(), "");
3829 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3830 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3831 emit_int8(0x22);
3832 emit_int8((unsigned char)(0xC0 | encode));
3833 emit_int8(imm8);
3834 }
3835
3836 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3837 assert(VM_Version::supports_sse4_1(), "");
3838 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3839 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3840 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3841 emit_int8(0x22);
3842 emit_operand(dst, src);
3843 emit_int8(imm8);
3844 }
3845
3846 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3847 assert(VM_Version::supports_sse2(), "");
3848 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3849 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3850 emit_int8((unsigned char)0xC4);
3851 emit_int8((unsigned char)(0xC0 | encode));
3852 emit_int8(imm8);
3853 }
3854
3855 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3856 assert(VM_Version::supports_sse2(), "");
3857 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3858 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3859 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3860 emit_int8((unsigned char)0xC4);
3861 emit_operand(dst, src);
3862 emit_int8(imm8);
3863 }
3864
3865 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3866 assert(VM_Version::supports_sse4_1(), "");
3867 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3868 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3869 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3870 emit_int8(0x20);
3871 emit_operand(dst, src);
3872 emit_int8(imm8);
3873 }
3874
3875 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3876 assert(VM_Version::supports_sse4_1(), "");
3877 InstructionMark im(this);
3878 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3879 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3880 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3881 emit_int8(0x30);
3882 emit_operand(dst, src);
3883 }
3884
3885 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3886 assert(VM_Version::supports_sse4_1(), "");
3887 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3888 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3889 emit_int8(0x30);
3890 emit_int8((unsigned char)(0xC0 | encode));
3891 }
3892
3893 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3894 assert(VM_Version::supports_avx(), "");
3895 InstructionMark im(this);
3896 assert(dst != xnoreg, "sanity");
3897 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3898 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3899 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3900 emit_int8(0x30);
3901 emit_operand(dst, src);
3902 }
3903
3904 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3905 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3906 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3907 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3908 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3909 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3910 emit_int8(0x30);
3911 emit_int8((unsigned char) (0xC0 | encode));
3912 }
3913
3914
3915 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3916 assert(is_vector_masking(), "");
3917 assert(VM_Version::supports_avx512vlbw(), "");
3918 assert(dst != xnoreg, "sanity");
3919 InstructionMark im(this);
3920 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3921 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3922 attributes.set_embedded_opmask_register_specifier(mask);
3923 attributes.set_is_evex_instruction();
3924 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3925 emit_int8(0x30);
3926 emit_operand(dst, src);
3927 }
3928 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3929 assert(VM_Version::supports_avx512vlbw(), "");
3930 assert(src != xnoreg, "sanity");
3931 InstructionMark im(this);
3932 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3933 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3934 attributes.set_is_evex_instruction();
3935 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3936 emit_int8(0x30);
3937 emit_operand(src, dst);
3938 }
3939
3940 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3941 assert(is_vector_masking(), "");
3942 assert(VM_Version::supports_avx512vlbw(), "");
3943 assert(src != xnoreg, "sanity");
3944 InstructionMark im(this);
3945 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3946 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3947 attributes.reset_is_clear_context();
3948 attributes.set_embedded_opmask_register_specifier(mask);
3949 attributes.set_is_evex_instruction();
3950 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3951 emit_int8(0x30);
3952 emit_operand(src, dst);
3953 }
3954
3955 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
3956 assert(VM_Version::supports_evex(), "");
3957 assert(src != xnoreg, "sanity");
3958 InstructionMark im(this);
3959 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3960 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
3961 attributes.set_is_evex_instruction();
3962 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3963 emit_int8(0x31);
3964 emit_operand(src, dst);
3965 }
3966
3967 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
3968 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3969 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3970 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
3971 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3972 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3973 emit_int8(0x33);
3974 emit_int8((unsigned char)(0xC0 | encode));
3975 }
3976
3977 // generic
3978 void Assembler::pop(Register dst) {
3979 int encode = prefix_and_encode(dst->encoding());
3980 emit_int8(0x58 | encode);
3981 }
3982
3983 void Assembler::popcntl(Register dst, Address src) {
3984 assert(VM_Version::supports_popcnt(), "must support");
3985 InstructionMark im(this);
3986 emit_int8((unsigned char)0xF3);
3987 prefix(src, dst);
3988 emit_int8(0x0F);
3989 emit_int8((unsigned char)0xB8);
3990 emit_operand(dst, src);
3991 }
4064 InstructionMark im(this);
4065 prefetch_prefix(src);
4066 emit_int8(0x18);
4067 emit_operand(rbx, src); // 3, src
4068 }
4069
4070 void Assembler::prefetchw(Address src) {
4071 assert(VM_Version::supports_3dnow_prefetch(), "must support");
4072 InstructionMark im(this);
4073 prefetch_prefix(src);
4074 emit_int8(0x0D);
4075 emit_operand(rcx, src); // 1, src
4076 }
4077
4078 void Assembler::prefix(Prefix p) {
4079 emit_int8(p);
4080 }
4081
4082 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
4083 assert(VM_Version::supports_ssse3(), "");
4084 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4085 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4086 emit_int8(0x00);
4087 emit_int8((unsigned char)(0xC0 | encode));
4088 }
4089
4090 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4091 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4092 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4093 0, "");
4094 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4095 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4096 emit_int8(0x00);
4097 emit_int8((unsigned char)(0xC0 | encode));
4098 }
4099
4100 void Assembler::pshufb(XMMRegister dst, Address src) {
4101 assert(VM_Version::supports_ssse3(), "");
4102 InstructionMark im(this);
4103 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4104 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4105 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4106 emit_int8(0x00);
4107 emit_operand(dst, src);
4108 }
4109
4110 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4111 assert(isByte(mode), "invalid value");
4112 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4113 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4114 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4115 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4116 emit_int8(0x70);
4117 emit_int8((unsigned char)(0xC0 | encode));
4118 emit_int8(mode & 0xFF);
4119 }
4120
4121 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4122 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4123 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4129 emit_int8((unsigned char)(0xC0 | encode));
4130 emit_int8(mode & 0xFF);
4131 }
4132
4133 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4134 assert(isByte(mode), "invalid value");
4135 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4136 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4137 InstructionMark im(this);
4138 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4139 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4140 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4141 emit_int8(0x70);
4142 emit_operand(dst, src);
4143 emit_int8(mode & 0xFF);
4144 }
4145
4146 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4147 assert(isByte(mode), "invalid value");
4148 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4149 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4150 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4151 emit_int8(0x70);
4152 emit_int8((unsigned char)(0xC0 | encode));
4153 emit_int8(mode & 0xFF);
4154 }
4155
4156 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4157 assert(isByte(mode), "invalid value");
4158 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4159 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4160 InstructionMark im(this);
4161 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4162 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4163 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4164 emit_int8(0x70);
4165 emit_operand(dst, src);
4166 emit_int8(mode & 0xFF);
4167 }
4168 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4169 assert(VM_Version::supports_evex(), "requires EVEX support");
4170 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
4171 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4172 attributes.set_is_evex_instruction();
4173 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4174 emit_int8(0x43);
4175 emit_int8((unsigned char)(0xC0 | encode));
4176 emit_int8(imm8 & 0xFF);
4177 }
4178
4179 void Assembler::psrldq(XMMRegister dst, int shift) {
4180 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4181 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4182 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4183 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4184 emit_int8(0x73);
4185 emit_int8((unsigned char)(0xC0 | encode));
4186 emit_int8(shift);
4187 }
4188
4189 void Assembler::pslldq(XMMRegister dst, int shift) {
4190 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4191 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4192 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4193 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4194 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4195 emit_int8(0x73);
4196 emit_int8((unsigned char)(0xC0 | encode));
4197 emit_int8(shift);
4198 }
4199
4200 void Assembler::ptest(XMMRegister dst, Address src) {
4201 assert(VM_Version::supports_sse4_1(), "");
4202 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4203 InstructionMark im(this);
4204 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4205 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4206 emit_int8(0x17);
4207 emit_operand(dst, src);
4208 }
4209
4210 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4211 assert(VM_Version::supports_sse4_1(), "");
4212 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4438 prefix(src, dst);
4439 emit_int8(0x1B);
4440 emit_operand(dst, src);
4441 }
4442
4443 void Assembler::sbbl(Register dst, Register src) {
4444 (void) prefix_and_encode(dst->encoding(), src->encoding());
4445 emit_arith(0x1B, 0xC0, dst, src);
4446 }
4447
4448 void Assembler::setb(Condition cc, Register dst) {
4449 assert(0 <= cc && cc < 16, "illegal cc");
4450 int encode = prefix_and_encode(dst->encoding(), true);
4451 emit_int8(0x0F);
4452 emit_int8((unsigned char)0x90 | cc);
4453 emit_int8((unsigned char)(0xC0 | encode));
4454 }
4455
4456 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4457 assert(VM_Version::supports_ssse3(), "");
4458 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4459 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4460 emit_int8((unsigned char)0x0F);
4461 emit_int8((unsigned char)(0xC0 | encode));
4462 emit_int8(imm8);
4463 }
4464
4465 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4466 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4467 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4468 0, "");
4469 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4470 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4471 emit_int8((unsigned char)0x0F);
4472 emit_int8((unsigned char)(0xC0 | encode));
4473 emit_int8(imm8);
4474 }
4475
4476 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
4477 assert(VM_Version::supports_evex(), "");
4478 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4479 attributes.set_is_evex_instruction();
4480 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4481 emit_int8(0x3);
4482 emit_int8((unsigned char)(0xC0 | encode));
4483 emit_int8(imm8);
4484 }
4485
4486 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4487 assert(VM_Version::supports_sse4_1(), "");
4488 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4489 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4490 emit_int8((unsigned char)0x0E);
4491 emit_int8((unsigned char)(0xC0 | encode));
4492 emit_int8(imm8);
4493 }
4494
4495 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4496 assert(VM_Version::supports_sha(), "");
4497 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4498 emit_int8((unsigned char)0xCC);
4499 emit_int8((unsigned char)(0xC0 | encode));
5574 attributes.set_rex_vex_w_reverted();
5575 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5576 emit_int8(0x57);
5577 emit_operand(dst, src);
5578 }
5579
5580 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5581 assert(VM_Version::supports_avx(), "");
5582 InstructionMark im(this);
5583 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5584 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5585 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5586 emit_int8(0x57);
5587 emit_operand(dst, src);
5588 }
5589
5590 // Integer vector arithmetic
5591 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5592 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5593 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5594 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
5595 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5596 emit_int8(0x01);
5597 emit_int8((unsigned char)(0xC0 | encode));
5598 }
5599
5600 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5601 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5602 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5603 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
5604 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5605 emit_int8(0x02);
5606 emit_int8((unsigned char)(0xC0 | encode));
5607 }
5608
5609 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5610 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5611 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5612 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5613 emit_int8((unsigned char)0xFC);
5614 emit_int8((unsigned char)(0xC0 | encode));
5615 }
5616
5617 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5618 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5619 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5620 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5621 emit_int8((unsigned char)0xFD);
5622 emit_int8((unsigned char)(0xC0 | encode));
5623 }
5633 void Assembler::paddd(XMMRegister dst, Address src) {
5634 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5635 InstructionMark im(this);
5636 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5637 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5638 emit_int8((unsigned char)0xFE);
5639 emit_operand(dst, src);
5640 }
5641
5642 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5643 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5644 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5645 attributes.set_rex_vex_w_reverted();
5646 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5647 emit_int8((unsigned char)0xD4);
5648 emit_int8((unsigned char)(0xC0 | encode));
5649 }
5650
5651 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5652 assert(VM_Version::supports_sse3(), "");
5653 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
5654 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5655 emit_int8(0x01);
5656 emit_int8((unsigned char)(0xC0 | encode));
5657 }
5658
5659 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5660 assert(VM_Version::supports_sse3(), "");
5661 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
5662 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5663 emit_int8(0x02);
5664 emit_int8((unsigned char)(0xC0 | encode));
5665 }
5666
5667 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5668 assert(UseAVX > 0, "requires some form of AVX");
5669 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5670 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5671 emit_int8((unsigned char)0xFC);
5672 emit_int8((unsigned char)(0xC0 | encode));
5673 }
5674
5675 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5676 assert(UseAVX > 0, "requires some form of AVX");
5677 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5678 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5679 emit_int8((unsigned char)0xFD);
5680 emit_int8((unsigned char)(0xC0 | encode));
5681 }
6352 }
6353
6354 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6355 assert(VM_Version::supports_evex(), "requires EVEX support");
6356 assert(dst != xnoreg, "sanity");
6357 InstructionMark im(this);
6358 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6359 attributes.set_is_evex_instruction();
6360 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6361 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6362 emit_int8((unsigned char)0xEF);
6363 emit_operand(dst, src);
6364 }
6365
6366
6367 // vinserti forms
6368
6369 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6370 assert(VM_Version::supports_avx2(), "");
6371 assert(imm8 <= 0x01, "imm8: %u", imm8);
6372 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6373 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6374 emit_int8(0x38);
6375 emit_int8((unsigned char)(0xC0 | encode));
6376 // 0x00 - insert into lower 128 bits
6377 // 0x01 - insert into upper 128 bits
6378 emit_int8(imm8 & 0x01);
6379 }
6380
6381 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6382 assert(VM_Version::supports_avx2(), "");
6383 assert(dst != xnoreg, "sanity");
6384 assert(imm8 <= 0x01, "imm8: %u", imm8);
6385 InstructionMark im(this);
6386 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6387 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6388 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6389 emit_int8(0x38);
6390 emit_operand(dst, src);
6391 // 0x00 - insert into lower 128 bits
6392 // 0x01 - insert into upper 128 bits
6393 emit_int8(imm8 & 0x01);
6394 }
6395
6396 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6397 assert(VM_Version::supports_evex(), "");
6398 assert(imm8 <= 0x03, "imm8: %u", imm8);
6399 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6400 attributes.set_is_evex_instruction();
6401 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6402 emit_int8(0x38);
6403 emit_int8((unsigned char)(0xC0 | encode));
6404 // 0x00 - insert into q0 128 bits (0..127)
6405 // 0x01 - insert into q1 128 bits (128..255)
6406 // 0x02 - insert into q2 128 bits (256..383)
6407 // 0x03 - insert into q3 128 bits (384..511)
6408 emit_int8(imm8 & 0x03);
6409 }
6410
6411 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6412 assert(VM_Version::supports_avx(), "");
6413 assert(dst != xnoreg, "sanity");
6414 assert(imm8 <= 0x03, "imm8: %u", imm8);
6415 InstructionMark im(this);
6416 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6417 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6418 attributes.set_is_evex_instruction();
6419 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6420 emit_int8(0x18);
6421 emit_operand(dst, src);
6422 // 0x00 - insert into q0 128 bits (0..127)
6423 // 0x01 - insert into q1 128 bits (128..255)
6424 // 0x02 - insert into q2 128 bits (256..383)
6425 // 0x03 - insert into q3 128 bits (384..511)
6426 emit_int8(imm8 & 0x03);
6427 }
6428
6429 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6430 assert(VM_Version::supports_evex(), "");
6431 assert(imm8 <= 0x01, "imm8: %u", imm8);
6432 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6433 attributes.set_is_evex_instruction();
6434 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6435 emit_int8(0x3A);
6436 emit_int8((unsigned char)(0xC0 | encode));
6437 // 0x00 - insert into lower 256 bits
6438 // 0x01 - insert into upper 256 bits
6439 emit_int8(imm8 & 0x01);
6440 }
6441
6442
6443 // vinsertf forms
6444
6445 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6446 assert(VM_Version::supports_avx(), "");
6447 assert(imm8 <= 0x01, "imm8: %u", imm8);
6448 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6449 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6450 emit_int8(0x18);
6451 emit_int8((unsigned char)(0xC0 | encode));
6452 // 0x00 - insert into lower 128 bits
6453 // 0x01 - insert into upper 128 bits
6454 emit_int8(imm8 & 0x01);
6455 }
6456
6457 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6458 assert(VM_Version::supports_avx(), "");
6459 assert(dst != xnoreg, "sanity");
6460 assert(imm8 <= 0x01, "imm8: %u", imm8);
6461 InstructionMark im(this);
6462 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6463 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6464 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6465 emit_int8(0x18);
6466 emit_operand(dst, src);
6467 // 0x00 - insert into lower 128 bits
6468 // 0x01 - insert into upper 128 bits
6469 emit_int8(imm8 & 0x01);
6470 }
6471
6472 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6473 assert(VM_Version::supports_avx2(), "");
6474 assert(imm8 <= 0x03, "imm8: %u", imm8);
6475 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6476 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6477 emit_int8(0x18);
6478 emit_int8((unsigned char)(0xC0 | encode));
6479 // 0x00 - insert into q0 128 bits (0..127)
6480 // 0x01 - insert into q1 128 bits (128..255)
6481 // 0x02 - insert into q0 128 bits (256..383)
6482 // 0x03 - insert into q1 128 bits (384..512)
6483 emit_int8(imm8 & 0x03);
6484 }
6485
6486 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6487 assert(VM_Version::supports_avx(), "");
6488 assert(dst != xnoreg, "sanity");
6489 assert(imm8 <= 0x03, "imm8: %u", imm8);
6490 InstructionMark im(this);
6491 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6492 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6493 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6494 emit_int8(0x18);
6495 emit_operand(dst, src);
6496 // 0x00 - insert into q0 128 bits (0..127)
6497 // 0x01 - insert into q1 128 bits (128..255)
6498 // 0x02 - insert into q0 128 bits (256..383)
6499 // 0x03 - insert into q1 128 bits (384..512)
6500 emit_int8(imm8 & 0x03);
6501 }
6502
6503 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6504 assert(VM_Version::supports_evex(), "");
6505 assert(imm8 <= 0x01, "imm8: %u", imm8);
6506 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6507 attributes.set_is_evex_instruction();
6508 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6509 emit_int8(0x1A);
6510 emit_int8((unsigned char)(0xC0 | encode));
6511 // 0x00 - insert into lower 256 bits
6512 // 0x01 - insert into upper 256 bits
6513 emit_int8(imm8 & 0x01);
6514 }
6515
6516 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6517 assert(VM_Version::supports_evex(), "");
6518 assert(dst != xnoreg, "sanity");
6519 assert(imm8 <= 0x01, "imm8: %u", imm8);
6520 InstructionMark im(this);
6521 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6522 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6523 attributes.set_is_evex_instruction();
6524 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6525 emit_int8(0x1A);
6526 emit_operand(dst, src);
6527 // 0x00 - insert into lower 256 bits
6528 // 0x01 - insert into upper 256 bits
6529 emit_int8(imm8 & 0x01);
6530 }
6531
6532
6533 // vextracti forms
6534
6535 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6536 assert(VM_Version::supports_avx2(), "");
6537 assert(imm8 <= 0x01, "imm8: %u", imm8);
6538 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6539 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6540 emit_int8(0x39);
6541 emit_int8((unsigned char)(0xC0 | encode));
6542 // 0x00 - extract from lower 128 bits
6543 // 0x01 - extract from upper 128 bits
6544 emit_int8(imm8 & 0x01);
6545 }
6546
6547 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6548 assert(VM_Version::supports_avx2(), "");
6549 assert(src != xnoreg, "sanity");
6550 assert(imm8 <= 0x01, "imm8: %u", imm8);
6551 InstructionMark im(this);
6552 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6553 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6554 attributes.reset_is_clear_context();
6555 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6556 emit_int8(0x39);
6557 emit_operand(src, dst);
6558 // 0x00 - extract from lower 128 bits
6559 // 0x01 - extract from upper 128 bits
6560 emit_int8(imm8 & 0x01);
6561 }
6562
6563 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6564 assert(VM_Version::supports_evex(), "");
6565 assert(imm8 <= 0x03, "imm8: %u", imm8);
6566 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6567 attributes.set_is_evex_instruction();
6568 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6569 emit_int8(0x39);
6570 emit_int8((unsigned char)(0xC0 | encode));
6571 // 0x00 - extract from bits 127:0
6572 // 0x01 - extract from bits 255:128
6573 // 0x02 - extract from bits 383:256
6574 // 0x03 - extract from bits 511:384
6575 emit_int8(imm8 & 0x03);
6576 }
6577
6578 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6579 assert(VM_Version::supports_evex(), "");
6580 assert(src != xnoreg, "sanity");
6581 assert(imm8 <= 0x03, "imm8: %u", imm8);
6582 InstructionMark im(this);
6583 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6584 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6585 attributes.reset_is_clear_context();
6586 attributes.set_is_evex_instruction();
6587 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6588 emit_int8(0x39);
6589 emit_operand(src, dst);
6590 // 0x00 - extract from bits 127:0
6591 // 0x01 - extract from bits 255:128
6592 // 0x02 - extract from bits 383:256
6593 // 0x03 - extract from bits 511:384
6594 emit_int8(imm8 & 0x03);
6595 }
6596
6597 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6598 assert(VM_Version::supports_avx512dq(), "");
6599 assert(imm8 <= 0x03, "imm8: %u", imm8);
6600 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6601 attributes.set_is_evex_instruction();
6602 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6603 emit_int8(0x39);
6604 emit_int8((unsigned char)(0xC0 | encode));
6605 // 0x00 - extract from bits 127:0
6606 // 0x01 - extract from bits 255:128
6607 // 0x02 - extract from bits 383:256
6608 // 0x03 - extract from bits 511:384
6609 emit_int8(imm8 & 0x03);
6610 }
6611
6612 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6613 assert(VM_Version::supports_evex(), "");
6614 assert(imm8 <= 0x01, "imm8: %u", imm8);
6615 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6616 attributes.set_is_evex_instruction();
6617 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6618 emit_int8(0x3B);
6619 emit_int8((unsigned char)(0xC0 | encode));
6620 // 0x00 - extract from lower 256 bits
6621 // 0x01 - extract from upper 256 bits
6622 emit_int8(imm8 & 0x01);
6623 }
6624
6625 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
6626 assert(VM_Version::supports_evex(), "");
6627 assert(src != xnoreg, "sanity");
6628 assert(imm8 <= 0x01, "imm8: %u", imm8);
6629 InstructionMark im(this);
6630 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6631 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6632 attributes.reset_is_clear_context();
6633 attributes.set_is_evex_instruction();
6634 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6635 emit_int8(0x38);
6636 emit_operand(src, dst);
6637 // 0x00 - extract from lower 256 bits
6638 // 0x01 - extract from upper 256 bits
6639 emit_int8(imm8 & 0x01);
6640 }
6641 // vextractf forms
6642
6643 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6644 assert(VM_Version::supports_avx(), "");
6645 assert(imm8 <= 0x01, "imm8: %u", imm8);
6646 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6647 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6648 emit_int8(0x19);
6649 emit_int8((unsigned char)(0xC0 | encode));
6650 // 0x00 - extract from lower 128 bits
6651 // 0x01 - extract from upper 128 bits
6652 emit_int8(imm8 & 0x01);
6653 }
6654
6655 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6656 assert(VM_Version::supports_avx(), "");
6657 assert(src != xnoreg, "sanity");
6658 assert(imm8 <= 0x01, "imm8: %u", imm8);
6659 InstructionMark im(this);
6660 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6661 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6662 attributes.reset_is_clear_context();
6663 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6664 emit_int8(0x19);
6665 emit_operand(src, dst);
6666 // 0x00 - extract from lower 128 bits
6667 // 0x01 - extract from upper 128 bits
6668 emit_int8(imm8 & 0x01);
6669 }
6670
6671 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6672 assert(VM_Version::supports_evex(), "");
6673 assert(imm8 <= 0x03, "imm8: %u", imm8);
6674 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6675 attributes.set_is_evex_instruction();
6676 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6677 emit_int8(0x19);
6678 emit_int8((unsigned char)(0xC0 | encode));
6679 // 0x00 - extract from bits 127:0
6680 // 0x01 - extract from bits 255:128
6681 // 0x02 - extract from bits 383:256
6682 // 0x03 - extract from bits 511:384
6683 emit_int8(imm8 & 0x03);
6684 }
6685
6686 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6687 assert(VM_Version::supports_evex(), "");
6688 assert(src != xnoreg, "sanity");
6689 assert(imm8 <= 0x03, "imm8: %u", imm8);
6690 InstructionMark im(this);
6691 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6692 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6693 attributes.reset_is_clear_context();
6694 attributes.set_is_evex_instruction();
6695 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6696 emit_int8(0x19);
6697 emit_operand(src, dst);
6698 // 0x00 - extract from bits 127:0
6699 // 0x01 - extract from bits 255:128
6700 // 0x02 - extract from bits 383:256
6701 // 0x03 - extract from bits 511:384
6702 emit_int8(imm8 & 0x03);
6703 }
6704
6705 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6706 assert(VM_Version::supports_avx512dq(), "");
6707 assert(imm8 <= 0x03, "imm8: %u", imm8);
6708 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6709 attributes.set_is_evex_instruction();
6710 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6711 emit_int8(0x19);
6712 emit_int8((unsigned char)(0xC0 | encode));
6713 // 0x00 - extract from bits 127:0
6714 // 0x01 - extract from bits 255:128
6715 // 0x02 - extract from bits 383:256
6716 // 0x03 - extract from bits 511:384
6717 emit_int8(imm8 & 0x03);
6718 }
6719
6720 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6721 assert(VM_Version::supports_evex(), "");
6722 assert(imm8 <= 0x01, "imm8: %u", imm8);
6723 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6724 attributes.set_is_evex_instruction();
6725 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6726 emit_int8(0x1B);
6727 emit_int8((unsigned char)(0xC0 | encode));
6728 // 0x00 - extract from lower 256 bits
6729 // 0x01 - extract from upper 256 bits
6730 emit_int8(imm8 & 0x01);
6731 }
6732
6733 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6734 assert(VM_Version::supports_evex(), "");
6735 assert(src != xnoreg, "sanity");
6736 assert(imm8 <= 0x01, "imm8: %u", imm8);
6737 InstructionMark im(this);
6738 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6739 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
6740 attributes.reset_is_clear_context();
6741 attributes.set_is_evex_instruction();
6742 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6743 emit_int8(0x1B);
6744 emit_operand(src, dst);
6745 // 0x00 - extract from lower 256 bits
6746 // 0x01 - extract from upper 256 bits
6747 emit_int8(imm8 & 0x01);
6748 }
6749
6750 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6751 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6752 assert(VM_Version::supports_avx2(), "");
6753 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6754 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6755 emit_int8(0x78);
6756 emit_int8((unsigned char)(0xC0 | encode));
6757 }
6758
6759 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6760 assert(VM_Version::supports_avx2(), "");
6761 assert(dst != xnoreg, "sanity");
6762 InstructionMark im(this);
6763 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6764 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6765 // swap src<->dst for encoding
6766 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6767 emit_int8(0x78);
6768 emit_operand(dst, src);
6769 }
6770
6771 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6772 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6773 assert(VM_Version::supports_avx2(), "");
6774 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6775 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6776 emit_int8(0x79);
6777 emit_int8((unsigned char)(0xC0 | encode));
6778 }
6779
6780 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6781 assert(VM_Version::supports_avx2(), "");
6782 assert(dst != xnoreg, "sanity");
6783 InstructionMark im(this);
6784 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6785 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6786 // swap src<->dst for encoding
6787 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6788 emit_int8(0x79);
6789 emit_operand(dst, src);
6790 }
6791
6792 // xmm/mem sourced byte/word/dword/qword replicate
6793
6794 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6795 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6796 assert(UseAVX >= 2, "");
6797 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6798 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6799 emit_int8(0x58);
6800 emit_int8((unsigned char)(0xC0 | encode));
6801 }
6802
6803 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6804 assert(VM_Version::supports_avx2(), "");
6805 assert(dst != xnoreg, "sanity");
6806 InstructionMark im(this);
6807 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6808 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6809 // swap src<->dst for encoding
6810 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6811 emit_int8(0x58);
6812 emit_operand(dst, src);
6813 }
6814
6815 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6816 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6817 assert(VM_Version::supports_avx2(), "");
6818 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6819 attributes.set_rex_vex_w_reverted();
6820 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6821 emit_int8(0x59);
6822 emit_int8((unsigned char)(0xC0 | encode));
6823 }
6824
6825 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6826 assert(VM_Version::supports_avx2(), "");
6827 assert(dst != xnoreg, "sanity");
6828 InstructionMark im(this);
6829 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6830 attributes.set_rex_vex_w_reverted();
6831 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6832 // swap src<->dst for encoding
6833 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6834 emit_int8(0x59);
6835 emit_operand(dst, src);
6836 }
6837 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
6838 assert(vector_len != Assembler::AVX_128bit, "");
6839 assert(VM_Version::supports_avx512dq(), "");
6840 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6841 attributes.set_rex_vex_w_reverted();
6842 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6843 emit_int8(0x5A);
6844 emit_int8((unsigned char)(0xC0 | encode));
6845 }
6846
6847 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
6848 assert(vector_len != Assembler::AVX_128bit, "");
6849 assert(VM_Version::supports_avx512dq(), "");
6850 assert(dst != xnoreg, "sanity");
6851 InstructionMark im(this);
6852 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6853 attributes.set_rex_vex_w_reverted();
6854 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
6855 // swap src<->dst for encoding
6856 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6857 emit_int8(0x5A);
6858 emit_operand(dst, src);
6859 }
6860
6861 // scalar single/double precision replicate
6862
6863 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6864 void Assembler::vpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6865 assert(VM_Version::supports_avx(), "");
6866 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6867 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6868 emit_int8(0x18);
6869 emit_int8((unsigned char)(0xC0 | encode));
6870 }
6871
6872 void Assembler::vpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6873 assert(VM_Version::supports_avx(), "");
6874 assert(dst != xnoreg, "sanity");
6875 InstructionMark im(this);
6876 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6877 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6878 // swap src<->dst for encoding
6879 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6880 emit_int8(0x18);
6881 emit_operand(dst, src);
6882 }
6883
6884 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6885 void Assembler::vpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6886 assert(VM_Version::supports_avx(), "");
6887 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6888 attributes.set_rex_vex_w_reverted();
6889 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6890 emit_int8(0x19);
6891 emit_int8((unsigned char)(0xC0 | encode));
6892 }
6893
6894 void Assembler::vpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6895 assert(VM_Version::supports_avx(), "");
6896 assert(dst != xnoreg, "sanity");
6897 InstructionMark im(this);
6898 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6899 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6900 attributes.set_rex_vex_w_reverted();
6901 // swap src<->dst for encoding
6902 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6903 emit_int8(0x19);
6904 emit_operand(dst, src);
6905 }
6906
6907
6908 // gpr source broadcast forms
6909
6910 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6911 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6912 assert(VM_Version::supports_avx512bw(), "");
6913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6914 attributes.set_is_evex_instruction();
6915 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6916 emit_int8(0x7A);
6917 emit_int8((unsigned char)(0xC0 | encode));
6918 }
6919
6920 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6921 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6922 assert(VM_Version::supports_avx512bw(), "");
6923 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6924 attributes.set_is_evex_instruction();
6925 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6926 emit_int8(0x7B);
6927 emit_int8((unsigned char)(0xC0 | encode));
6928 }
6929
6930 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6931 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6932 assert(VM_Version::supports_evex(), "");
6933 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6934 attributes.set_is_evex_instruction();
6935 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6936 emit_int8(0x7C);
6937 emit_int8((unsigned char)(0xC0 | encode));
6938 }
6939
6940 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6941 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6942 assert(VM_Version::supports_evex(), "");
6948 }
6949
6950 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
6951 assert(VM_Version::supports_evex(), "");
6952 assert(dst != xnoreg, "sanity");
6953 InstructionMark im(this);
6954 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6955 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6956 attributes.reset_is_clear_context();
6957 attributes.set_embedded_opmask_register_specifier(mask);
6958 attributes.set_is_evex_instruction();
6959 // swap src<->dst for encoding
6960 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6961 emit_int8((unsigned char)0x90);
6962 emit_operand(dst, src);
6963 }
6964
6965 // Carry-Less Multiplication Quadword
6966 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6967 assert(VM_Version::supports_clmul(), "");
6968 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
6969 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6970 emit_int8(0x44);
6971 emit_int8((unsigned char)(0xC0 | encode));
6972 emit_int8((unsigned char)mask);
6973 }
6974
6975 // Carry-Less Multiplication Quadword
6976 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6977 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6978 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
6979 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6980 emit_int8(0x44);
6981 emit_int8((unsigned char)(0xC0 | encode));
6982 emit_int8((unsigned char)mask);
6983 }
6984
6985 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
6986 assert(VM_Version::supports_vpclmulqdq(), "Requires vector carryless multiplication support");
6987 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6988 attributes.set_is_evex_instruction();
6989 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6990 emit_int8(0x44);
6991 emit_int8((unsigned char)(0xC0 | encode));
6992 emit_int8((unsigned char)mask);
6993 }
6994
6995 void Assembler::vzeroupper() {
6996 if (VM_Version::supports_vzeroupper()) {
6997 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6998 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7578 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7579 // last is EVEX.z for zero/merge actions
7580 if (_attributes->is_no_reg_mask() == false) {
7581 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7582 }
7583 emit_int8(byte4);
7584 }
7585
7586 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7587 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7588 bool vex_b = adr.base_needs_rex();
7589 bool vex_x;
7590 if (adr.isxmmindex()) {
7591 vex_x = adr.xmmindex_needs_rex();
7592 } else {
7593 vex_x = adr.index_needs_rex();
7594 }
7595 set_attributes(attributes);
7596 attributes->set_current_assembler(this);
7597
7598 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
7599 // is allowed in legacy mode and has resources which will fit in it.
7600 // Pure EVEX instructions will have is_evex_instruction set in their definition.
7601 if (!attributes->is_legacy_mode()) {
7602 if (UseAVX > 2 && !attributes->is_evex_instruction() && !_is_managed) {
7603 if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
7604 attributes->set_is_legacy_mode();
7605 }
7606 }
7607 }
7608
7609 if (UseAVX > 2) {
7610 assert(((!attributes->uses_vl()) ||
7611 (attributes->get_vector_len() == AVX_512bit) ||
7612 (!_legacy_mode_vl) ||
7613 (attributes->is_legacy_mode())),"XMM register should be 0-15");
7614 assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
7615 }
7616
7617 _is_managed = false;
7618 if (UseAVX > 2 && !attributes->is_legacy_mode())
7619 {
7620 bool evex_r = (xreg_enc >= 16);
7621 bool evex_v;
7622 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
7623 if (adr.isxmmindex()) {
7624 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
7625 } else {
7626 evex_v = (nds_enc >= 16);
7627 }
7628 attributes->set_is_evex_instruction();
7629 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7630 } else {
7631 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7632 attributes->set_rex_vex_w(false);
7633 }
7634 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7635 }
7636 }
7637
7638 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7639 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7640 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7641 bool vex_x = false;
7642 set_attributes(attributes);
7643 attributes->set_current_assembler(this);
7644
7645 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
7646 // is allowed in legacy mode and has resources which will fit in it.
7647 // Pure EVEX instructions will have is_evex_instruction set in their definition.
7648 if (!attributes->is_legacy_mode()) {
7649 if (UseAVX > 2 && !attributes->is_evex_instruction() && !_is_managed) {
7650 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
7651 (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
7652 attributes->set_is_legacy_mode();
7653 }
7654 }
7655 }
7656
7657 if (UseAVX > 2) {
7658 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
7659 // Instruction with uses_vl true are vector instructions
7660 // All the vector instructions with AVX_512bit length can have legacy_mode as false
7661 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
7662 // Rest all should have legacy_mode set as true
7663 assert(((!attributes->uses_vl()) ||
7664 (attributes->get_vector_len() == AVX_512bit) ||
7665 (!_legacy_mode_vl) ||
7666 (attributes->is_legacy_mode())),"XMM register should be 0-15");
7667 // Instruction with legacy_mode true should have dst, nds and src < 15
7668 assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
7669 }
7670
7671 _is_managed = false;
7672 if (UseAVX > 2 && !attributes->is_legacy_mode())
7673 {
7674 bool evex_r = (dst_enc >= 16);
7675 bool evex_v = (nds_enc >= 16);
7676 // can use vex_x as bank extender on rm encoding
7677 vex_x = (src_enc >= 16);
7678 attributes->set_is_evex_instruction();
7679 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7680 } else {
7681 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7682 attributes->set_rex_vex_w(false);
7683 }
7684 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7685 }
7686
7687 // return modrm byte components for operands
7688 return (((dst_enc & 7) << 3) | (src_enc & 7));
7700 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7701 }
7702 }
7703
7704 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7705 VexOpcode opc, InstructionAttr *attributes) {
7706 int dst_enc = dst->encoding();
7707 int src_enc = src->encoding();
7708 if (UseAVX > 0) {
7709 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7710 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7711 } else {
7712 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7713 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7714 }
7715 }
7716
7717 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7718 assert(VM_Version::supports_avx(), "");
7719 assert(!VM_Version::supports_evex(), "");
7720 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
7721 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7722 emit_int8((unsigned char)0xC2);
7723 emit_int8((unsigned char)(0xC0 | encode));
7724 emit_int8((unsigned char)(0xF & cop));
7725 }
7726
7727 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7728 assert(VM_Version::supports_avx(), "");
7729 assert(!VM_Version::supports_evex(), "");
7730 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
7731 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7732 emit_int8((unsigned char)0x4B);
7733 emit_int8((unsigned char)(0xC0 | encode));
7734 int src2_enc = src2->encoding();
7735 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7736 }
7737
7738 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7739 assert(VM_Version::supports_avx(), "");
7740 assert(!VM_Version::supports_evex(), "");
7741 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
7742 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7743 emit_int8((unsigned char)0xC2);
7744 emit_int8((unsigned char)(0xC0 | encode));
7745 emit_int8((unsigned char)(0xF & cop));
7746 }
7747
7748 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7749 assert(VM_Version::supports_avx(), "");
7750 assert(!VM_Version::supports_evex(), "");
7751 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
7752 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7753 emit_int8((unsigned char)0x4A);
7754 emit_int8((unsigned char)(0xC0 | encode));
7755 int src2_enc = src2->encoding();
7756 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7757 }
7758
7759 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7760 assert(VM_Version::supports_avx2(), "");
7761 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
7762 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7763 emit_int8((unsigned char)0x02);
7764 emit_int8((unsigned char)(0xC0 | encode));
7765 emit_int8((unsigned char)imm8);
7766 }
7767
7768 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7769 assert(VM_Version::supports_bmi2(), "");
7770 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7771 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7772 emit_int8((unsigned char)0xF7);
7773 emit_int8((unsigned char)(0xC0 | encode));
7774 }
7775
7776 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7777 assert(VM_Version::supports_bmi2(), "");
7778 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7779 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7780 emit_int8((unsigned char)0xF7);
7781 emit_int8((unsigned char)(0xC0 | encode));
7782 }
7783
7784 #ifndef _LP64
7785
7786 void Assembler::incl(Register dst) {
7787 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7788 emit_int8(0x40 | dst->encoding());
7789 }
7790
7791 void Assembler::lea(Register dst, Address src) {
7792 leal(dst, src);
7793 }
7794
7795 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7796 InstructionMark im(this);
7797 emit_int8((unsigned char)0xC7);
7798 emit_operand(rax, dst);
|