2315 }
2316
2317 // This instruction produces ZF or CF flags
2318 void Assembler::kortestql(KRegister src1, KRegister src2) {
2319 assert(VM_Version::supports_avx512bw(), "");
2320 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2321 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2322 emit_int8((unsigned char)0x98);
2323 emit_int8((unsigned char)(0xC0 | encode));
2324 }
2325
2326 // This instruction produces ZF or CF flags
2327 void Assembler::ktestql(KRegister src1, KRegister src2) {
2328 assert(VM_Version::supports_avx512bw(), "");
2329 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2330 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2331 emit_int8((unsigned char)0x99);
2332 emit_int8((unsigned char)(0xC0 | encode));
2333 }
2334
2335 void Assembler::movb(Address dst, int imm8) {
2336 InstructionMark im(this);
2337 prefix(dst);
2338 emit_int8((unsigned char)0xC6);
2339 emit_operand(rax, dst, 1);
2340 emit_int8(imm8);
2341 }
2342
2343
2344 void Assembler::movb(Address dst, Register src) {
2345 assert(src->has_byte_register(), "must have byte register");
2346 InstructionMark im(this);
2347 prefix(dst, src, true);
2348 emit_int8((unsigned char)0x88);
2349 emit_operand(src, dst);
2350 }
2351
2352 void Assembler::movdl(XMMRegister dst, Register src) {
2353 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2354 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2483 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2484 attributes.set_is_evex_instruction();
2485 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2486 emit_int8(0x6F);
2487 emit_operand(dst, src);
2488 }
2489
2490 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2491 assert(VM_Version::supports_evex(), "");
2492 assert(src != xnoreg, "sanity");
2493 InstructionMark im(this);
2494 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2495 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2496 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2497 attributes.set_is_evex_instruction();
2498 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2499 emit_int8(0x7F);
2500 emit_operand(src, dst);
2501 }
2502
2503 void Assembler::evmovdqub(KRegister mask, XMMRegister dst, Address src, int vector_len) {
2504 assert(VM_Version::supports_avx512vlbw(), "");
2505 assert(is_vector_masking(), ""); // For stub code use only
2506 InstructionMark im(this);
2507 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2508 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2509 attributes.set_embedded_opmask_register_specifier(mask);
2510 attributes.set_is_evex_instruction();
2511 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2512 emit_int8(0x6F);
2513 emit_operand(dst, src);
2514 }
2515
2516 void Assembler::evmovdquw(XMMRegister dst, XMMRegister src, int vector_len) {
2517 assert(VM_Version::supports_evex(), "");
2518 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2519 attributes.set_is_evex_instruction();
2520 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2521 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2522 emit_int8(0x6F);
2523 emit_int8((unsigned char)(0xC0 | encode));
2524 }
2525
2526 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2527 assert(VM_Version::supports_evex(), "");
2528 InstructionMark im(this);
2529 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2530 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2531 attributes.set_is_evex_instruction();
2532 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2533 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2534 emit_int8(0x6F);
2535 emit_operand(dst, src);
2536 }
2537
2538 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2539 assert(VM_Version::supports_evex(), "");
2540 assert(src != xnoreg, "sanity");
2541 InstructionMark im(this);
2542 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2543 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2544 attributes.set_is_evex_instruction();
2545 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2546 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2547 emit_int8(0x7F);
2548 emit_operand(src, dst);
2549 }
2550
2551 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2552 assert(VM_Version::supports_evex(), "");
2553 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2554 attributes.set_is_evex_instruction();
2555 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2556 emit_int8(0x6F);
2557 emit_int8((unsigned char)(0xC0 | encode));
2558 }
2559
2560 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2561 assert(VM_Version::supports_evex(), "");
2562 InstructionMark im(this);
2563 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2564 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2565 attributes.set_is_evex_instruction();
2566 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2567 emit_int8(0x6F);
2568 emit_operand(dst, src);
2569 }
2570
3278
3279 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3280 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3281 assert(VM_Version::supports_avx(), "");
3282 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3283 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3284 emit_int8(0x74);
3285 emit_int8((unsigned char)(0xC0 | encode));
3286 }
3287
3288 // In this context, kdst is written the mask used to process the equal components
3289 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3290 assert(VM_Version::supports_avx512bw(), "");
3291 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3292 attributes.set_is_evex_instruction();
3293 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3294 emit_int8(0x74);
3295 emit_int8((unsigned char)(0xC0 | encode));
3296 }
3297
3298 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3299 assert(VM_Version::supports_avx512bw(), "");
3300 InstructionMark im(this);
3301 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3302 attributes.set_is_evex_instruction();
3303 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3304 int dst_enc = kdst->encoding();
3305 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3306 emit_int8(0x74);
3307 emit_operand(as_Register(dst_enc), src);
3308 }
3309
3310 void Assembler::evpcmpeqb(KRegister mask, KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3311 assert(VM_Version::supports_avx512vlbw(), "");
3312 assert(is_vector_masking(), ""); // For stub code use only
3313 InstructionMark im(this);
3314 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3315 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3316 attributes.set_embedded_opmask_register_specifier(mask);
3317 attributes.set_is_evex_instruction();
3318 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3319 emit_int8(0x74);
3320 emit_operand(as_Register(kdst->encoding()), src);
3321 }
3322
3323 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3324 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3325 assert(VM_Version::supports_sse2(), "");
3326 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3327 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3328 emit_int8(0x75);
3329 emit_int8((unsigned char)(0xC0 | encode));
3330 }
3603
3604 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3605 assert(VM_Version::supports_sse4_1(), "");
3606 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3607 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3608 emit_int8(0x30);
3609 emit_int8((unsigned char)(0xC0 | encode));
3610 }
3611
3612 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3613 assert(VM_Version::supports_avx(), "");
3614 InstructionMark im(this);
3615 assert(dst != xnoreg, "sanity");
3616 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3617 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3618 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3619 emit_int8(0x30);
3620 emit_operand(dst, src);
3621 }
3622
3623 // generic
3624 void Assembler::pop(Register dst) {
3625 int encode = prefix_and_encode(dst->encoding());
3626 emit_int8(0x58 | encode);
3627 }
3628
3629 void Assembler::popcntl(Register dst, Address src) {
3630 assert(VM_Version::supports_popcnt(), "must support");
3631 InstructionMark im(this);
3632 emit_int8((unsigned char)0xF3);
3633 prefix(src, dst);
3634 emit_int8(0x0F);
3635 emit_int8((unsigned char)0xB8);
3636 emit_operand(dst, src);
3637 }
3638
3639 void Assembler::popcntl(Register dst, Register src) {
3640 assert(VM_Version::supports_popcnt(), "must support");
3641 emit_int8((unsigned char)0xF3);
3642 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6389 emit_int8((unsigned char)mask);
6390 }
6391
6392 // Carry-Less Multiplication Quadword
6393 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6394 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6395 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6396 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6397 emit_int8(0x44);
6398 emit_int8((unsigned char)(0xC0 | encode));
6399 emit_int8((unsigned char)mask);
6400 }
6401
6402 void Assembler::vzeroupper() {
6403 assert(VM_Version::supports_avx(), "");
6404 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6405 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6406 emit_int8(0x77);
6407 }
6408
6409
6410 #ifndef _LP64
6411 // 32bit only pieces of the assembler
6412
6413 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6414 // NO PREFIX AS NEVER 64BIT
6415 InstructionMark im(this);
6416 emit_int8((unsigned char)0x81);
6417 emit_int8((unsigned char)(0xF8 | src1->encoding()));
6418 emit_data(imm32, rspec, 0);
6419 }
6420
6421 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6422 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6423 InstructionMark im(this);
6424 emit_int8((unsigned char)0x81);
6425 emit_operand(rdi, src1);
6426 emit_data(imm32, rspec, 0);
6427 }
6428
6429 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6956 // P0: byte 2, initialized to RXBR`00mm
6957 // instead of not'd
6958 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6959 byte2 = (~byte2) & 0xF0;
6960 // confine opc opcode extensions in mm bits to lower two bits
6961 // of form {0F, 0F_38, 0F_3A}
6962 byte2 |= opc;
6963 emit_int8(byte2);
6964
6965 // P1: byte 3 as Wvvvv1pp
6966 int byte3 = ((~nds_enc) & 0xf) << 3;
6967 // p[10] is always 1
6968 byte3 |= EVEX_F;
6969 byte3 |= (vex_w & 1) << 7;
6970 // confine pre opcode extensions in pp bits to lower two bits
6971 // of form {66, F3, F2}
6972 byte3 |= pre;
6973 emit_int8(byte3);
6974
6975 // P2: byte 4 as zL'Lbv'aaa
6976 int byte4 = (_attributes->is_no_reg_mask()) ? 0 : _attributes->get_embedded_opmask_register_specifier(); // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6977 // EVEX.v` for extending EVEX.vvvv or VIDX
6978 byte4 |= (evex_v ? 0: EVEX_V);
6979 // third EXEC.b for broadcast actions
6980 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6981 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6982 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6983 // last is EVEX.z for zero/merge actions
6984 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6985 emit_int8(byte4);
6986 }
6987
6988 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6989 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6990 bool vex_b = adr.base_needs_rex();
6991 bool vex_x = adr.index_needs_rex();
6992 set_attributes(attributes);
6993 attributes->set_current_assembler(this);
6994
6995 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6996 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
|
2315 }
2316
2317 // This instruction produces ZF or CF flags
2318 void Assembler::kortestql(KRegister src1, KRegister src2) {
2319 assert(VM_Version::supports_avx512bw(), "");
2320 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2321 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2322 emit_int8((unsigned char)0x98);
2323 emit_int8((unsigned char)(0xC0 | encode));
2324 }
2325
2326 // This instruction produces ZF or CF flags
2327 void Assembler::ktestql(KRegister src1, KRegister src2) {
2328 assert(VM_Version::supports_avx512bw(), "");
2329 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2330 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2331 emit_int8((unsigned char)0x99);
2332 emit_int8((unsigned char)(0xC0 | encode));
2333 }
2334
2335 void Assembler::ktestq(KRegister src1, KRegister src2) {
2336 assert(VM_Version::supports_avx512bw(), "");
2337 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2338 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2339 emit_int8((unsigned char)0x99);
2340 emit_int8((unsigned char)(0xC0 | encode));
2341 }
2342
2343 void Assembler::ktestd(KRegister src1, KRegister src2) {
2344 assert(VM_Version::supports_avx512bw(), "");
2345 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2346 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2347 emit_int8((unsigned char)0x99);
2348 emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350
2351 void Assembler::movb(Address dst, int imm8) {
2352 InstructionMark im(this);
2353 prefix(dst);
2354 emit_int8((unsigned char)0xC6);
2355 emit_operand(rax, dst, 1);
2356 emit_int8(imm8);
2357 }
2358
2359
2360 void Assembler::movb(Address dst, Register src) {
2361 assert(src->has_byte_register(), "must have byte register");
2362 InstructionMark im(this);
2363 prefix(dst, src, true);
2364 emit_int8((unsigned char)0x88);
2365 emit_operand(src, dst);
2366 }
2367
2368 void Assembler::movdl(XMMRegister dst, Register src) {
2369 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2370 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2499 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2500 attributes.set_is_evex_instruction();
2501 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2502 emit_int8(0x6F);
2503 emit_operand(dst, src);
2504 }
2505
2506 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2507 assert(VM_Version::supports_evex(), "");
2508 assert(src != xnoreg, "sanity");
2509 InstructionMark im(this);
2510 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2511 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2512 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2513 attributes.set_is_evex_instruction();
2514 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2515 emit_int8(0x7F);
2516 emit_operand(src, dst);
2517 }
2518
2519 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2520 assert(VM_Version::supports_avx512vlbw(), "");
2521 assert(is_vector_masking(), ""); // For stub code use only
2522 InstructionMark im(this);
2523 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2524 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2525 attributes.set_embedded_opmask_register_specifier(mask);
2526 attributes.set_is_evex_instruction();
2527 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2528 emit_int8(0x6F);
2529 emit_operand(dst, src);
2530 }
2531
2532 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2533 assert(VM_Version::supports_evex(), "");
2534 InstructionMark im(this);
2535 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2536 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2537 attributes.set_is_evex_instruction();
2538 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2539 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2540 emit_int8(0x6F);
2541 emit_operand(dst, src);
2542 }
2543
2544 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2545 assert(is_vector_masking(), "");
2546 assert(VM_Version::supports_avx512vlbw(), "");
2547 InstructionMark im(this);
2548 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2549 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2550 attributes.set_embedded_opmask_register_specifier(mask);
2551 attributes.set_is_evex_instruction();
2552 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2553 emit_int8(0x6F);
2554 emit_operand(dst, src);
2555 }
2556
2557 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2558 assert(VM_Version::supports_evex(), "");
2559 assert(src != xnoreg, "sanity");
2560 InstructionMark im(this);
2561 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2562 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2563 attributes.set_is_evex_instruction();
2564 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2565 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2566 emit_int8(0x7F);
2567 emit_operand(src, dst);
2568 }
2569
2570 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2571 assert(VM_Version::supports_avx512vlbw(), "");
2572 assert(src != xnoreg, "sanity");
2573 InstructionMark im(this);
2574 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2575 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2576 attributes.set_embedded_opmask_register_specifier(mask);
2577 attributes.set_is_evex_instruction();
2578 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2579 emit_int8(0x7F);
2580 emit_operand(src, dst);
2581 }
2582
2583 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2584 assert(VM_Version::supports_evex(), "");
2585 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2586 attributes.set_is_evex_instruction();
2587 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2588 emit_int8(0x6F);
2589 emit_int8((unsigned char)(0xC0 | encode));
2590 }
2591
2592 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2593 assert(VM_Version::supports_evex(), "");
2594 InstructionMark im(this);
2595 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2596 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2597 attributes.set_is_evex_instruction();
2598 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2599 emit_int8(0x6F);
2600 emit_operand(dst, src);
2601 }
2602
3310
3311 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3312 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3313 assert(VM_Version::supports_avx(), "");
3314 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3315 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3316 emit_int8(0x74);
3317 emit_int8((unsigned char)(0xC0 | encode));
3318 }
3319
3320 // In this context, kdst is written the mask used to process the equal components
3321 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3322 assert(VM_Version::supports_avx512bw(), "");
3323 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3324 attributes.set_is_evex_instruction();
3325 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3326 emit_int8(0x74);
3327 emit_int8((unsigned char)(0xC0 | encode));
3328 }
3329
3330 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3331 assert(VM_Version::supports_avx512vlbw(), "");
3332 InstructionMark im(this);
3333 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3334 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3335 attributes.set_is_evex_instruction();
3336 int dst_enc = kdst->encoding();
3337 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3338 emit_int8(0x64);
3339 emit_operand(as_Register(dst_enc), src);
3340 }
3341
3342 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3343 assert(is_vector_masking(), "");
3344 assert(VM_Version::supports_avx512vlbw(), "");
3345 InstructionMark im(this);
3346 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3347 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3348 attributes.set_embedded_opmask_register_specifier(mask);
3349 attributes.set_is_evex_instruction();
3350 int dst_enc = kdst->encoding();
3351 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3352 emit_int8(0x64);
3353 emit_operand(as_Register(dst_enc), src);
3354 }
3355
3356 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3357 assert(VM_Version::supports_avx512vlbw(), "");
3358 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3359 attributes.set_is_evex_instruction();
3360 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3361 emit_int8(0x3E);
3362 emit_int8((unsigned char)(0xC0 | encode));
3363 emit_int8(vcc);
3364 }
3365
3366 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3367 assert(is_vector_masking(), "");
3368 assert(VM_Version::supports_avx512vlbw(), "");
3369 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3370 attributes.set_embedded_opmask_register_specifier(mask);
3371 attributes.set_is_evex_instruction();
3372 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3373 emit_int8(0x3E);
3374 emit_int8((unsigned char)(0xC0 | encode));
3375 emit_int8(vcc);
3376 }
3377
3378 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3379 assert(VM_Version::supports_avx512vlbw(), "");
3380 InstructionMark im(this);
3381 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3382 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3383 attributes.set_is_evex_instruction();
3384 int dst_enc = kdst->encoding();
3385 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3386 emit_int8(0x3E);
3387 emit_operand(as_Register(dst_enc), src);
3388 emit_int8(vcc);
3389 }
3390
3391 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3392 assert(VM_Version::supports_avx512bw(), "");
3393 InstructionMark im(this);
3394 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3395 attributes.set_is_evex_instruction();
3396 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3397 int dst_enc = kdst->encoding();
3398 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3399 emit_int8(0x74);
3400 emit_operand(as_Register(dst_enc), src);
3401 }
3402
3403 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3404 assert(VM_Version::supports_avx512vlbw(), "");
3405 assert(is_vector_masking(), ""); // For stub code use only
3406 InstructionMark im(this);
3407 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3408 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3409 attributes.set_embedded_opmask_register_specifier(mask);
3410 attributes.set_is_evex_instruction();
3411 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3412 emit_int8(0x74);
3413 emit_operand(as_Register(kdst->encoding()), src);
3414 }
3415
3416 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3417 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3418 assert(VM_Version::supports_sse2(), "");
3419 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3420 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3421 emit_int8(0x75);
3422 emit_int8((unsigned char)(0xC0 | encode));
3423 }
3696
3697 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3698 assert(VM_Version::supports_sse4_1(), "");
3699 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3700 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3701 emit_int8(0x30);
3702 emit_int8((unsigned char)(0xC0 | encode));
3703 }
3704
3705 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3706 assert(VM_Version::supports_avx(), "");
3707 InstructionMark im(this);
3708 assert(dst != xnoreg, "sanity");
3709 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3710 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3711 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3712 emit_int8(0x30);
3713 emit_operand(dst, src);
3714 }
3715
3716 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3717 assert(is_vector_masking(), "");
3718 assert(VM_Version::supports_avx512vlbw(), "");
3719 assert(dst != xnoreg, "sanity");
3720 InstructionMark im(this);
3721 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3722 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3723 attributes.set_embedded_opmask_register_specifier(mask);
3724 attributes.set_is_evex_instruction();
3725 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3726 emit_int8(0x30);
3727 emit_operand(dst, src);
3728 }
3729
3730 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3731 assert(VM_Version::supports_avx512vlbw(), "");
3732 assert(src != xnoreg, "sanity");
3733 InstructionMark im(this);
3734 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3735 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3736 attributes.set_is_evex_instruction();
3737 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3738 emit_int8(0x30);
3739 emit_operand(src, dst);
3740 }
3741
3742 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3743 assert(is_vector_masking(), "");
3744 assert(VM_Version::supports_avx512vlbw(), "");
3745 assert(src != xnoreg, "sanity");
3746 InstructionMark im(this);
3747 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3748 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3749 attributes.set_embedded_opmask_register_specifier(mask);
3750 attributes.set_is_evex_instruction();
3751 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3752 emit_int8(0x30);
3753 emit_operand(src, dst);
3754 }
3755
3756 // generic
3757 void Assembler::pop(Register dst) {
3758 int encode = prefix_and_encode(dst->encoding());
3759 emit_int8(0x58 | encode);
3760 }
3761
3762 void Assembler::popcntl(Register dst, Address src) {
3763 assert(VM_Version::supports_popcnt(), "must support");
3764 InstructionMark im(this);
3765 emit_int8((unsigned char)0xF3);
3766 prefix(src, dst);
3767 emit_int8(0x0F);
3768 emit_int8((unsigned char)0xB8);
3769 emit_operand(dst, src);
3770 }
3771
3772 void Assembler::popcntl(Register dst, Register src) {
3773 assert(VM_Version::supports_popcnt(), "must support");
3774 emit_int8((unsigned char)0xF3);
3775 int encode = prefix_and_encode(dst->encoding(), src->encoding());
6522 emit_int8((unsigned char)mask);
6523 }
6524
6525 // Carry-Less Multiplication Quadword
6526 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6527 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6528 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6529 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6530 emit_int8(0x44);
6531 emit_int8((unsigned char)(0xC0 | encode));
6532 emit_int8((unsigned char)mask);
6533 }
6534
6535 void Assembler::vzeroupper() {
6536 assert(VM_Version::supports_avx(), "");
6537 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6538 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6539 emit_int8(0x77);
6540 }
6541
6542 #ifndef _LP64
6543 // 32bit only pieces of the assembler
6544
6545 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6546 // NO PREFIX AS NEVER 64BIT
6547 InstructionMark im(this);
6548 emit_int8((unsigned char)0x81);
6549 emit_int8((unsigned char)(0xF8 | src1->encoding()));
6550 emit_data(imm32, rspec, 0);
6551 }
6552
6553 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6554 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6555 InstructionMark im(this);
6556 emit_int8((unsigned char)0x81);
6557 emit_operand(rdi, src1);
6558 emit_data(imm32, rspec, 0);
6559 }
6560
6561 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
7088 // P0: byte 2, initialized to RXBR`00mm
7089 // instead of not'd
7090 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7091 byte2 = (~byte2) & 0xF0;
7092 // confine opc opcode extensions in mm bits to lower two bits
7093 // of form {0F, 0F_38, 0F_3A}
7094 byte2 |= opc;
7095 emit_int8(byte2);
7096
7097 // P1: byte 3 as Wvvvv1pp
7098 int byte3 = ((~nds_enc) & 0xf) << 3;
7099 // p[10] is always 1
7100 byte3 |= EVEX_F;
7101 byte3 |= (vex_w & 1) << 7;
7102 // confine pre opcode extensions in pp bits to lower two bits
7103 // of form {66, F3, F2}
7104 byte3 |= pre;
7105 emit_int8(byte3);
7106
7107 // P2: byte 4 as zL'Lbv'aaa
7108 // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7109 int byte4 = (_attributes->is_no_reg_mask()) ?
7110 0 :
7111 _attributes->get_embedded_opmask_register_specifier();
7112 // EVEX.v` for extending EVEX.vvvv or VIDX
7113 byte4 |= (evex_v ? 0: EVEX_V);
7114 // third EXEC.b for broadcast actions
7115 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7116 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7117 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7118 // last is EVEX.z for zero/merge actions
7119 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7120 emit_int8(byte4);
7121 }
7122
7123 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7124 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7125 bool vex_b = adr.base_needs_rex();
7126 bool vex_x = adr.index_needs_rex();
7127 set_attributes(attributes);
7128 attributes->set_current_assembler(this);
7129
7130 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7131 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
|