1 /*
2 * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableBarrierSet.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/biasedLocking.hpp"
34 #include "runtime/objectMonitor.hpp"
35 #include "runtime/os.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/stubRoutines.hpp"
38 #include "utilities/macros.hpp"
39
40 #ifdef PRODUCT
41 #define BLOCK_COMMENT(str) /* nothing */
42 #define STOP(error) stop(error)
43 #else
44 #define BLOCK_COMMENT(str) block_comment(str)
45 #define STOP(error) block_comment(error); stop(error)
46 #endif
47
48 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
49 // Implementation of AddressLiteral
50
51 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
52 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
53 // -----------------Table 4.5 -------------------- //
54 16, 32, 64, // EVEX_FV(0)
55 4, 4, 4, // EVEX_FV(1) - with Evex.b
56 16, 32, 64, // EVEX_FV(2) - with Evex.w
57 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
58 8, 16, 32, // EVEX_HV(0)
59 4, 4, 4, // EVEX_HV(1) - with Evex.b
60 // -----------------Table 4.6 -------------------- //
61 16, 32, 64, // EVEX_FVM(0)
62 1, 1, 1, // EVEX_T1S(0)
63 2, 2, 2, // EVEX_T1S(1)
64 4, 4, 4, // EVEX_T1S(2)
65 8, 8, 8, // EVEX_T1S(3)
66 4, 4, 4, // EVEX_T1F(0)
67 8, 8, 8, // EVEX_T1F(1)
68 8, 8, 8, // EVEX_T2(0)
69 0, 16, 16, // EVEX_T2(1)
70 0, 16, 16, // EVEX_T4(0)
71 0, 0, 32, // EVEX_T4(1)
72 0, 0, 32, // EVEX_T8(0)
73 8, 16, 32, // EVEX_HVM(0)
74 4, 8, 16, // EVEX_QVM(0)
75 2, 4, 8, // EVEX_OVM(0)
76 16, 16, 16, // EVEX_M128(0)
77 8, 32, 64, // EVEX_DUP(0)
78 0, 0, 0 // EVEX_NTUP
79 };
80
81 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
82 _is_lval = false;
83 _target = target;
84 switch (rtype) {
85 case relocInfo::oop_type:
86 case relocInfo::metadata_type:
87 // Oops are a special case. Normally they would be their own section
88 // but in cases like icBuffer they are literals in the code stream that
89 // we don't have a section for. We use none so that we get a literal address
90 // which is always patchable.
91 break;
92 case relocInfo::external_word_type:
93 _rspec = external_word_Relocation::spec(target);
94 break;
95 case relocInfo::internal_word_type:
96 _rspec = internal_word_Relocation::spec(target);
97 break;
98 case relocInfo::opt_virtual_call_type:
99 _rspec = opt_virtual_call_Relocation::spec();
100 break;
101 case relocInfo::static_call_type:
102 _rspec = static_call_Relocation::spec();
103 break;
104 case relocInfo::runtime_call_type:
105 _rspec = runtime_call_Relocation::spec();
106 break;
107 case relocInfo::poll_type:
108 case relocInfo::poll_return_type:
109 _rspec = Relocation::spec_simple(rtype);
110 break;
111 case relocInfo::none:
112 break;
113 default:
114 ShouldNotReachHere();
115 break;
116 }
117 }
118
119 // Implementation of Address
120
121 #ifdef _LP64
122
123 Address Address::make_array(ArrayAddress adr) {
124 // Not implementable on 64bit machines
125 // Should have been handled higher up the call chain.
126 ShouldNotReachHere();
127 return Address();
128 }
129
130 // exceedingly dangerous constructor
131 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
132 _base = noreg;
133 _index = noreg;
134 _scale = no_scale;
135 _disp = disp;
136 _xmmindex = xnoreg;
137 _isxmmindex = false;
138 switch (rtype) {
139 case relocInfo::external_word_type:
140 _rspec = external_word_Relocation::spec(loc);
141 break;
142 case relocInfo::internal_word_type:
143 _rspec = internal_word_Relocation::spec(loc);
144 break;
145 case relocInfo::runtime_call_type:
146 // HMM
147 _rspec = runtime_call_Relocation::spec();
148 break;
149 case relocInfo::poll_type:
150 case relocInfo::poll_return_type:
151 _rspec = Relocation::spec_simple(rtype);
152 break;
153 case relocInfo::none:
154 break;
155 default:
156 ShouldNotReachHere();
157 }
158 }
159 #else // LP64
160
161 Address Address::make_array(ArrayAddress adr) {
162 AddressLiteral base = adr.base();
163 Address index = adr.index();
164 assert(index._disp == 0, "must not have disp"); // maybe it can?
165 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
166 array._rspec = base._rspec;
167 return array;
168 }
169
170 // exceedingly dangerous constructor
171 Address::Address(address loc, RelocationHolder spec) {
172 _base = noreg;
173 _index = noreg;
174 _scale = no_scale;
175 _disp = (intptr_t) loc;
176 _rspec = spec;
177 _xmmindex = xnoreg;
178 _isxmmindex = false;
179 }
180
181 #endif // _LP64
182
183
184
185 // Convert the raw encoding form into the form expected by the constructor for
186 // Address. An index of 4 (rsp) corresponds to having no index, so convert
187 // that to noreg for the Address constructor.
188 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
189 RelocationHolder rspec;
190 if (disp_reloc != relocInfo::none) {
191 rspec = Relocation::spec_simple(disp_reloc);
192 }
193 bool valid_index = index != rsp->encoding();
194 if (valid_index) {
195 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
196 madr._rspec = rspec;
197 return madr;
198 } else {
199 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
200 madr._rspec = rspec;
201 return madr;
202 }
203 }
204
205 // Implementation of Assembler
206
207 int AbstractAssembler::code_fill_byte() {
208 return (u_char)'\xF4'; // hlt
209 }
210
211 // make this go away someday
212 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
213 if (rtype == relocInfo::none)
214 emit_int32(data);
215 else
216 emit_data(data, Relocation::spec_simple(rtype), format);
217 }
218
219 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
220 assert(imm_operand == 0, "default format must be immediate in this file");
221 assert(inst_mark() != NULL, "must be inside InstructionMark");
222 if (rspec.type() != relocInfo::none) {
223 #ifdef ASSERT
224 check_relocation(rspec, format);
225 #endif
226 // Do not use AbstractAssembler::relocate, which is not intended for
227 // embedded words. Instead, relocate to the enclosing instruction.
228
229 // hack. call32 is too wide for mask so use disp32
230 if (format == call32_operand)
231 code_section()->relocate(inst_mark(), rspec, disp32_operand);
232 else
233 code_section()->relocate(inst_mark(), rspec, format);
234 }
235 emit_int32(data);
236 }
237
238 static int encode(Register r) {
239 int enc = r->encoding();
240 if (enc >= 8) {
241 enc -= 8;
242 }
243 return enc;
244 }
245
246 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
247 assert(dst->has_byte_register(), "must have byte register");
248 assert(isByte(op1) && isByte(op2), "wrong opcode");
249 assert(isByte(imm8), "not a byte");
250 assert((op1 & 0x01) == 0, "should be 8bit operation");
251 emit_int24(op1, (op2 | encode(dst)), imm8);
252 }
253
254
255 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
256 assert(isByte(op1) && isByte(op2), "wrong opcode");
257 assert((op1 & 0x01) == 1, "should be 32bit operation");
258 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
259 if (is8bit(imm32)) {
260 emit_int24(op1 | 0x02, // set sign bit
261 op2 | encode(dst),
262 imm32 & 0xFF);
263 } else {
264 emit_int16(op1, (op2 | encode(dst)));
265 emit_int32(imm32);
266 }
267 }
268
269 // Force generation of a 4 byte immediate value even if it fits into 8bit
270 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
271 assert(isByte(op1) && isByte(op2), "wrong opcode");
272 assert((op1 & 0x01) == 1, "should be 32bit operation");
273 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
274 emit_int16(op1, (op2 | encode(dst)));
275 emit_int32(imm32);
276 }
277
278 // immediate-to-memory forms
279 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
280 assert((op1 & 0x01) == 1, "should be 32bit operation");
281 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
282 if (is8bit(imm32)) {
283 emit_int8(op1 | 0x02); // set sign bit
284 emit_operand(rm, adr, 1);
285 emit_int8(imm32 & 0xFF);
286 } else {
287 emit_int8(op1);
288 emit_operand(rm, adr, 4);
289 emit_int32(imm32);
290 }
291 }
292
293
294 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
295 assert(isByte(op1) && isByte(op2), "wrong opcode");
296 emit_int16(op1, (op2 | encode(dst) << 3 | encode(src)));
297 }
298
299
300 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
301 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
302 int mod_idx = 0;
303 // We will test if the displacement fits the compressed format and if so
304 // apply the compression to the displacment iff the result is8bit.
305 if (VM_Version::supports_evex() && is_evex_inst) {
306 switch (cur_tuple_type) {
307 case EVEX_FV:
308 if ((cur_encoding & VEX_W) == VEX_W) {
309 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
310 } else {
311 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
312 }
313 break;
314
315 case EVEX_HV:
316 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
317 break;
318
319 case EVEX_FVM:
320 break;
321
322 case EVEX_T1S:
323 switch (in_size_in_bits) {
324 case EVEX_8bit:
325 break;
326
327 case EVEX_16bit:
328 mod_idx = 1;
329 break;
330
331 case EVEX_32bit:
332 mod_idx = 2;
333 break;
334
335 case EVEX_64bit:
336 mod_idx = 3;
337 break;
338 }
339 break;
340
341 case EVEX_T1F:
342 case EVEX_T2:
343 case EVEX_T4:
344 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
345 break;
346
347 case EVEX_T8:
348 break;
349
350 case EVEX_HVM:
351 break;
352
353 case EVEX_QVM:
354 break;
355
356 case EVEX_OVM:
357 break;
358
359 case EVEX_M128:
360 break;
361
362 case EVEX_DUP:
363 break;
364
365 default:
366 assert(0, "no valid evex tuple_table entry");
367 break;
368 }
369
370 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
371 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
372 if ((disp % disp_factor) == 0) {
373 int new_disp = disp / disp_factor;
374 if ((-0x80 <= new_disp && new_disp < 0x80)) {
375 disp = new_disp;
376 }
377 } else {
378 return false;
379 }
380 }
381 }
382 return (-0x80 <= disp && disp < 0x80);
383 }
384
385
386 bool Assembler::emit_compressed_disp_byte(int &disp) {
387 int mod_idx = 0;
388 // We will test if the displacement fits the compressed format and if so
389 // apply the compression to the displacment iff the result is8bit.
390 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
391 int evex_encoding = _attributes->get_evex_encoding();
392 int tuple_type = _attributes->get_tuple_type();
393 switch (tuple_type) {
394 case EVEX_FV:
395 if ((evex_encoding & VEX_W) == VEX_W) {
396 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
397 } else {
398 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
399 }
400 break;
401
402 case EVEX_HV:
403 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
404 break;
405
406 case EVEX_FVM:
407 break;
408
409 case EVEX_T1S:
410 switch (_attributes->get_input_size()) {
411 case EVEX_8bit:
412 break;
413
414 case EVEX_16bit:
415 mod_idx = 1;
416 break;
417
418 case EVEX_32bit:
419 mod_idx = 2;
420 break;
421
422 case EVEX_64bit:
423 mod_idx = 3;
424 break;
425 }
426 break;
427
428 case EVEX_T1F:
429 case EVEX_T2:
430 case EVEX_T4:
431 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
432 break;
433
434 case EVEX_T8:
435 break;
436
437 case EVEX_HVM:
438 break;
439
440 case EVEX_QVM:
441 break;
442
443 case EVEX_OVM:
444 break;
445
446 case EVEX_M128:
447 break;
448
449 case EVEX_DUP:
450 break;
451
452 default:
453 assert(0, "no valid evex tuple_table entry");
454 break;
455 }
456
457 int vector_len = _attributes->get_vector_len();
458 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
459 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
460 if ((disp % disp_factor) == 0) {
461 int new_disp = disp / disp_factor;
462 if (is8bit(new_disp)) {
463 disp = new_disp;
464 }
465 } else {
466 return false;
467 }
468 }
469 }
470 return is8bit(disp);
471 }
472
473 static bool is_valid_encoding(int reg_enc) {
474 return reg_enc >= 0;
475 }
476
477 static int raw_encode(Register reg) {
478 assert(reg == noreg || reg->is_valid(), "sanity");
479 int reg_enc = (intptr_t)reg;
480 assert(reg_enc == -1 || is_valid_encoding(reg_enc), "sanity");
481 return reg_enc;
482 }
483
484 static int raw_encode(XMMRegister xmmreg) {
485 assert(xmmreg == xnoreg || xmmreg->is_valid(), "sanity");
486 int xmmreg_enc = (intptr_t)xmmreg;
487 assert(xmmreg_enc == -1 || is_valid_encoding(xmmreg_enc), "sanity");
488 return xmmreg_enc;
489 }
490
491 static int modrm_encoding(int mod, int dst_enc, int src_enc) {
492 return (mod & 3) << 6 | (dst_enc & 7) << 3 | (src_enc & 7);
493 }
494
495 static int sib_encoding(Address::ScaleFactor scale, int index_enc, int base_enc) {
496 return (scale & 3) << 6 | (index_enc & 7) << 3 | (base_enc & 7);
497 }
498
499 inline void Assembler::emit_modrm(int mod, int dst_enc, int src_enc) {
500 assert((mod & 3) != 0b11, "forbidden");
501 int modrm = modrm_encoding(mod, dst_enc, src_enc);
502 emit_int8(modrm);
503 }
504
505 inline void Assembler::emit_modrm_disp8(int mod, int dst_enc, int src_enc,
506 int disp) {
507 int modrm = modrm_encoding(mod, dst_enc, src_enc);
508 emit_int16(modrm, disp & 0xFF);
509 }
510
511 inline void Assembler::emit_modrm_sib(int mod, int dst_enc, int src_enc,
512 Address::ScaleFactor scale, int index_enc, int base_enc) {
513 int modrm = modrm_encoding(mod, dst_enc, src_enc);
514 int sib = sib_encoding(scale, index_enc, base_enc);
515 emit_int16(modrm, sib);
516 }
517
518 inline void Assembler::emit_modrm_sib_disp8(int mod, int dst_enc, int src_enc,
519 Address::ScaleFactor scale, int index_enc, int base_enc,
520 int disp) {
521 int modrm = modrm_encoding(mod, dst_enc, src_enc);
522 int sib = sib_encoding(scale, index_enc, base_enc);
523 emit_int24(modrm, sib, disp & 0xFF);
524 }
525
526 void Assembler::emit_operand_helper(int reg_enc, int base_enc, int index_enc,
527 Address::ScaleFactor scale, int disp,
528 RelocationHolder const& rspec,
529 int rip_relative_correction) {
530 bool no_relocation = (rspec.type() == relocInfo::none);
531
532 if (is_valid_encoding(base_enc)) {
533 if (is_valid_encoding(index_enc)) {
534 assert(scale != Address::no_scale, "inconsistent address");
535 // [base + index*scale + disp]
536 if (disp == 0 && no_relocation &&
537 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
538 // [base + index*scale]
539 // [00 reg 100][ss index base]
540 emit_modrm_sib(0b00, reg_enc, 0b100,
541 scale, index_enc, base_enc);
542 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
543 // [base + index*scale + imm8]
544 // [01 reg 100][ss index base] imm8
545 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
546 scale, index_enc, base_enc,
547 disp);
548 } else {
549 // [base + index*scale + disp32]
550 // [10 reg 100][ss index base] disp32
551 emit_modrm_sib(0b10, reg_enc, 0b100,
552 scale, index_enc, base_enc);
553 emit_data(disp, rspec, disp32_operand);
554 }
555 } else if (base_enc == rsp->encoding() LP64_ONLY(|| base_enc == r12->encoding())) {
556 // [rsp + disp]
557 if (disp == 0 && no_relocation) {
558 // [rsp]
559 // [00 reg 100][00 100 100]
560 emit_modrm_sib(0b00, reg_enc, 0b100,
561 Address::times_1, 0b100, 0b100);
562 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
563 // [rsp + imm8]
564 // [01 reg 100][00 100 100] disp8
565 emit_modrm_sib_disp8(0b01, reg_enc, 0b100,
566 Address::times_1, 0b100, 0b100,
567 disp);
568 } else {
569 // [rsp + imm32]
570 // [10 reg 100][00 100 100] disp32
571 emit_modrm_sib(0b10, reg_enc, 0b100,
572 Address::times_1, 0b100, 0b100);
573 emit_data(disp, rspec, disp32_operand);
574 }
575 } else {
576 // [base + disp]
577 assert(base_enc != rsp->encoding() LP64_ONLY(&& base_enc != r12->encoding()), "illegal addressing mode");
578 if (disp == 0 && no_relocation &&
579 base_enc != rbp->encoding() LP64_ONLY(&& base_enc != r13->encoding())) {
580 // [base]
581 // [00 reg base]
582 emit_modrm(0, reg_enc, base_enc);
583 } else if (emit_compressed_disp_byte(disp) && no_relocation) {
584 // [base + disp8]
585 // [01 reg base] disp8
586 emit_modrm_disp8(0b01, reg_enc, base_enc,
587 disp);
588 } else {
589 // [base + disp32]
590 // [10 reg base] disp32
591 emit_modrm(0b10, reg_enc, base_enc);
592 emit_data(disp, rspec, disp32_operand);
593 }
594 }
595 } else {
596 if (is_valid_encoding(index_enc)) {
597 assert(scale != Address::no_scale, "inconsistent address");
598 // base == noreg
599 // [index*scale + disp]
600 // [00 reg 100][ss index 101] disp32
601 emit_modrm_sib(0b00, reg_enc, 0b100,
602 scale, index_enc, 0b101 /* no base */);
603 emit_data(disp, rspec, disp32_operand);
604 } else if (!no_relocation) {
605 // base == noreg, index == noreg
606 // [disp] (64bit) RIP-RELATIVE (32bit) abs
607 // [00 reg 101] disp32
608
609 emit_modrm(0b00, reg_enc, 0b101 /* no base */);
610 // Note that the RIP-rel. correction applies to the generated
611 // disp field, but _not_ to the target address in the rspec.
612
613 // disp was created by converting the target address minus the pc
614 // at the start of the instruction. That needs more correction here.
615 // intptr_t disp = target - next_ip;
616 assert(inst_mark() != NULL, "must be inside InstructionMark");
617 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
618 int64_t adjusted = disp;
619 // Do rip-rel adjustment for 64bit
620 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
621 assert(is_simm32(adjusted),
622 "must be 32bit offset (RIP relative address)");
623 emit_data((int32_t) adjusted, rspec, disp32_operand);
624
625 } else {
626 // base == noreg, index == noreg, no_relocation == true
627 // 32bit never did this, did everything as the rip-rel/disp code above
628 // [disp] ABSOLUTE
629 // [00 reg 100][00 100 101] disp32
630 emit_modrm_sib(0b00, reg_enc, 0b100 /* no base */,
631 Address::times_1, 0b100, 0b101);
632 emit_data(disp, rspec, disp32_operand);
633 }
634 }
635 }
636
637 void Assembler::emit_operand(Register reg, Register base, Register index,
638 Address::ScaleFactor scale, int disp,
639 RelocationHolder const& rspec,
640 int rip_relative_correction) {
641 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
642 emit_operand_helper(raw_encode(reg), raw_encode(base), raw_encode(index),
643 scale, disp, rspec, rip_relative_correction);
644
645 }
646 void Assembler::emit_operand(XMMRegister xmmreg, Register base, Register index,
647 Address::ScaleFactor scale, int disp,
648 RelocationHolder const& rspec) {
649 assert(!index->is_valid() || index != rsp, "illegal addressing mode");
650 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
651 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(index),
652 scale, disp, rspec);
653 }
654
655 void Assembler::emit_operand(XMMRegister xmmreg, Register base, XMMRegister xmmindex,
656 Address::ScaleFactor scale, int disp,
657 RelocationHolder const& rspec) {
658 assert(xmmreg->encoding() < 16 || UseAVX > 2, "not supported");
659 assert(xmmindex->encoding() < 16 || UseAVX > 2, "not supported");
660 emit_operand_helper(raw_encode(xmmreg), raw_encode(base), raw_encode(xmmindex),
661 scale, disp, rspec, /* rip_relative_correction */ 0);
662 }
663
664 // Secret local extension to Assembler::WhichOperand:
665 #define end_pc_operand (_WhichOperand_limit)
666
667 address Assembler::locate_operand(address inst, WhichOperand which) {
668 // Decode the given instruction, and return the address of
669 // an embedded 32-bit operand word.
670
671 // If "which" is disp32_operand, selects the displacement portion
672 // of an effective address specifier.
673 // If "which" is imm64_operand, selects the trailing immediate constant.
674 // If "which" is call32_operand, selects the displacement of a call or jump.
675 // Caller is responsible for ensuring that there is such an operand,
676 // and that it is 32/64 bits wide.
677
678 // If "which" is end_pc_operand, find the end of the instruction.
679
680 address ip = inst;
681 bool is_64bit = false;
682
683 debug_only(bool has_disp32 = false);
684 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
685
686 again_after_prefix:
687 switch (0xFF & *ip++) {
688
689 // These convenience macros generate groups of "case" labels for the switch.
690 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
691 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
692 case (x)+4: case (x)+5: case (x)+6: case (x)+7
693 #define REP16(x) REP8((x)+0): \
694 case REP8((x)+8)
695
696 case CS_segment:
697 case SS_segment:
698 case DS_segment:
699 case ES_segment:
700 case FS_segment:
701 case GS_segment:
702 // Seems dubious
703 LP64_ONLY(assert(false, "shouldn't have that prefix"));
704 assert(ip == inst+1, "only one prefix allowed");
705 goto again_after_prefix;
706
707 case 0x67:
708 case REX:
709 case REX_B:
710 case REX_X:
711 case REX_XB:
712 case REX_R:
713 case REX_RB:
714 case REX_RX:
715 case REX_RXB:
716 NOT_LP64(assert(false, "64bit prefixes"));
717 goto again_after_prefix;
718
719 case REX_W:
720 case REX_WB:
721 case REX_WX:
722 case REX_WXB:
723 case REX_WR:
724 case REX_WRB:
725 case REX_WRX:
726 case REX_WRXB:
727 NOT_LP64(assert(false, "64bit prefixes"));
728 is_64bit = true;
729 goto again_after_prefix;
730
731 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
732 case 0x88: // movb a, r
733 case 0x89: // movl a, r
734 case 0x8A: // movb r, a
735 case 0x8B: // movl r, a
736 case 0x8F: // popl a
737 debug_only(has_disp32 = true);
738 break;
739
740 case 0x68: // pushq #32
741 if (which == end_pc_operand) {
742 return ip + 4;
743 }
744 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
745 return ip; // not produced by emit_operand
746
747 case 0x66: // movw ... (size prefix)
748 again_after_size_prefix2:
749 switch (0xFF & *ip++) {
750 case REX:
751 case REX_B:
752 case REX_X:
753 case REX_XB:
754 case REX_R:
755 case REX_RB:
756 case REX_RX:
757 case REX_RXB:
758 case REX_W:
759 case REX_WB:
760 case REX_WX:
761 case REX_WXB:
762 case REX_WR:
763 case REX_WRB:
764 case REX_WRX:
765 case REX_WRXB:
766 NOT_LP64(assert(false, "64bit prefix found"));
767 goto again_after_size_prefix2;
768 case 0x8B: // movw r, a
769 case 0x89: // movw a, r
770 debug_only(has_disp32 = true);
771 break;
772 case 0xC7: // movw a, #16
773 debug_only(has_disp32 = true);
774 tail_size = 2; // the imm16
775 break;
776 case 0x0F: // several SSE/SSE2 variants
777 ip--; // reparse the 0x0F
778 goto again_after_prefix;
779 default:
780 ShouldNotReachHere();
781 }
782 break;
783
784 case REP8(0xB8): // movl/q r, #32/#64(oop?)
785 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
786 // these asserts are somewhat nonsensical
787 #ifndef _LP64
788 assert(which == imm_operand || which == disp32_operand,
789 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
790 #else
791 assert((which == call32_operand || which == imm_operand) && is_64bit ||
792 which == narrow_oop_operand && !is_64bit,
793 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
794 #endif // _LP64
795 return ip;
796
797 case 0x69: // imul r, a, #32
798 case 0xC7: // movl a, #32(oop?)
799 tail_size = 4;
800 debug_only(has_disp32 = true); // has both kinds of operands!
801 break;
802
803 case 0x0F: // movx..., etc.
804 switch (0xFF & *ip++) {
805 case 0x3A: // pcmpestri
806 tail_size = 1;
807 case 0x38: // ptest, pmovzxbw
808 ip++; // skip opcode
809 debug_only(has_disp32 = true); // has both kinds of operands!
810 break;
811
812 case 0x70: // pshufd r, r/a, #8
813 debug_only(has_disp32 = true); // has both kinds of operands!
814 case 0x73: // psrldq r, #8
815 tail_size = 1;
816 break;
817
818 case 0x12: // movlps
819 case 0x28: // movaps
820 case 0x2E: // ucomiss
821 case 0x2F: // comiss
822 case 0x54: // andps
823 case 0x55: // andnps
824 case 0x56: // orps
825 case 0x57: // xorps
826 case 0x58: // addpd
827 case 0x59: // mulpd
828 case 0x6E: // movd
829 case 0x7E: // movd
830 case 0x6F: // movdq
831 case 0x7F: // movdq
832 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
833 case 0xFE: // paddd
834 debug_only(has_disp32 = true);
835 break;
836
837 case 0xAD: // shrd r, a, %cl
838 case 0xAF: // imul r, a
839 case 0xBE: // movsbl r, a (movsxb)
840 case 0xBF: // movswl r, a (movsxw)
841 case 0xB6: // movzbl r, a (movzxb)
842 case 0xB7: // movzwl r, a (movzxw)
843 case REP16(0x40): // cmovl cc, r, a
844 case 0xB0: // cmpxchgb
845 case 0xB1: // cmpxchg
846 case 0xC1: // xaddl
847 case 0xC7: // cmpxchg8
848 case REP16(0x90): // setcc a
849 debug_only(has_disp32 = true);
850 // fall out of the switch to decode the address
851 break;
852
853 case 0xC4: // pinsrw r, a, #8
854 debug_only(has_disp32 = true);
855 case 0xC5: // pextrw r, r, #8
856 tail_size = 1; // the imm8
857 break;
858
859 case 0xAC: // shrd r, a, #8
860 debug_only(has_disp32 = true);
861 tail_size = 1; // the imm8
862 break;
863
864 case REP16(0x80): // jcc rdisp32
865 if (which == end_pc_operand) return ip + 4;
866 assert(which == call32_operand, "jcc has no disp32 or imm");
867 return ip;
868 default:
869 ShouldNotReachHere();
870 }
871 break;
872
873 case 0x81: // addl a, #32; addl r, #32
874 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
875 // on 32bit in the case of cmpl, the imm might be an oop
876 tail_size = 4;
877 debug_only(has_disp32 = true); // has both kinds of operands!
878 break;
879
880 case 0x83: // addl a, #8; addl r, #8
881 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
882 debug_only(has_disp32 = true); // has both kinds of operands!
883 tail_size = 1;
884 break;
885
886 case 0x9B:
887 switch (0xFF & *ip++) {
888 case 0xD9: // fnstcw a
889 debug_only(has_disp32 = true);
890 break;
891 default:
892 ShouldNotReachHere();
893 }
894 break;
895
896 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
897 case REP4(0x10): // adc...
898 case REP4(0x20): // and...
899 case REP4(0x30): // xor...
900 case REP4(0x08): // or...
901 case REP4(0x18): // sbb...
902 case REP4(0x28): // sub...
903 case 0xF7: // mull a
904 case 0x8D: // lea r, a
905 case 0x87: // xchg r, a
906 case REP4(0x38): // cmp...
907 case 0x85: // test r, a
908 debug_only(has_disp32 = true); // has both kinds of operands!
909 break;
910
911 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
912 case 0xC6: // movb a, #8
913 case 0x80: // cmpb a, #8
914 case 0x6B: // imul r, a, #8
915 debug_only(has_disp32 = true); // has both kinds of operands!
916 tail_size = 1; // the imm8
917 break;
918
919 case 0xC4: // VEX_3bytes
920 case 0xC5: // VEX_2bytes
921 assert((UseAVX > 0), "shouldn't have VEX prefix");
922 assert(ip == inst+1, "no prefixes allowed");
923 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
924 // but they have prefix 0x0F and processed when 0x0F processed above.
925 //
926 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
927 // instructions (these instructions are not supported in 64-bit mode).
928 // To distinguish them bits [7:6] are set in the VEX second byte since
929 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
930 // those VEX bits REX and vvvv bits are inverted.
931 //
932 // Fortunately C2 doesn't generate these instructions so we don't need
933 // to check for them in product version.
934
935 // Check second byte
936 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
937
938 int vex_opcode;
939 // First byte
940 if ((0xFF & *inst) == VEX_3bytes) {
941 vex_opcode = VEX_OPCODE_MASK & *ip;
942 ip++; // third byte
943 is_64bit = ((VEX_W & *ip) == VEX_W);
944 } else {
945 vex_opcode = VEX_OPCODE_0F;
946 }
947 ip++; // opcode
948 // To find the end of instruction (which == end_pc_operand).
949 switch (vex_opcode) {
950 case VEX_OPCODE_0F:
951 switch (0xFF & *ip) {
952 case 0x70: // pshufd r, r/a, #8
953 case 0x71: // ps[rl|ra|ll]w r, #8
954 case 0x72: // ps[rl|ra|ll]d r, #8
955 case 0x73: // ps[rl|ra|ll]q r, #8
956 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
957 case 0xC4: // pinsrw r, r, r/a, #8
958 case 0xC5: // pextrw r/a, r, #8
959 case 0xC6: // shufp[s|d] r, r, r/a, #8
960 tail_size = 1; // the imm8
961 break;
962 }
963 break;
964 case VEX_OPCODE_0F_3A:
965 tail_size = 1;
966 break;
967 }
968 ip++; // skip opcode
969 debug_only(has_disp32 = true); // has both kinds of operands!
970 break;
971
972 case 0x62: // EVEX_4bytes
973 assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
974 assert(ip == inst+1, "no prefixes allowed");
975 // no EVEX collisions, all instructions that have 0x62 opcodes
976 // have EVEX versions and are subopcodes of 0x66
977 ip++; // skip P0 and exmaine W in P1
978 is_64bit = ((VEX_W & *ip) == VEX_W);
979 ip++; // move to P2
980 ip++; // skip P2, move to opcode
981 // To find the end of instruction (which == end_pc_operand).
982 switch (0xFF & *ip) {
983 case 0x22: // pinsrd r, r/a, #8
984 case 0x61: // pcmpestri r, r/a, #8
985 case 0x70: // pshufd r, r/a, #8
986 case 0x73: // psrldq r, #8
987 tail_size = 1; // the imm8
988 break;
989 default:
990 break;
991 }
992 ip++; // skip opcode
993 debug_only(has_disp32 = true); // has both kinds of operands!
994 break;
995
996 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
997 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
998 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
999 case 0xDD: // fld_d a; fst_d a; fstp_d a
1000 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
1001 case 0xDF: // fild_d a; fistp_d a
1002 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
1003 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
1004 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
1005 debug_only(has_disp32 = true);
1006 break;
1007
1008 case 0xE8: // call rdisp32
1009 case 0xE9: // jmp rdisp32
1010 if (which == end_pc_operand) return ip + 4;
1011 assert(which == call32_operand, "call has no disp32 or imm");
1012 return ip;
1013
1014 case 0xF0: // Lock
1015 goto again_after_prefix;
1016
1017 case 0xF3: // For SSE
1018 case 0xF2: // For SSE2
1019 switch (0xFF & *ip++) {
1020 case REX:
1021 case REX_B:
1022 case REX_X:
1023 case REX_XB:
1024 case REX_R:
1025 case REX_RB:
1026 case REX_RX:
1027 case REX_RXB:
1028 case REX_W:
1029 case REX_WB:
1030 case REX_WX:
1031 case REX_WXB:
1032 case REX_WR:
1033 case REX_WRB:
1034 case REX_WRX:
1035 case REX_WRXB:
1036 NOT_LP64(assert(false, "found 64bit prefix"));
1037 ip++;
1038 default:
1039 ip++;
1040 }
1041 debug_only(has_disp32 = true); // has both kinds of operands!
1042 break;
1043
1044 default:
1045 ShouldNotReachHere();
1046
1047 #undef REP8
1048 #undef REP16
1049 }
1050
1051 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1052 #ifdef _LP64
1053 assert(which != imm_operand, "instruction is not a movq reg, imm64");
1054 #else
1055 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1056 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1057 #endif // LP64
1058 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1059
1060 // parse the output of emit_operand
1061 int op2 = 0xFF & *ip++;
1062 int base = op2 & 0x07;
1063 int op3 = -1;
1064 const int b100 = 4;
1065 const int b101 = 5;
1066 if (base == b100 && (op2 >> 6) != 3) {
1067 op3 = 0xFF & *ip++;
1068 base = op3 & 0x07; // refetch the base
1069 }
1070 // now ip points at the disp (if any)
1071
1072 switch (op2 >> 6) {
1073 case 0:
1074 // [00 reg 100][ss index base]
1075 // [00 reg 100][00 100 esp]
1076 // [00 reg base]
1077 // [00 reg 100][ss index 101][disp32]
1078 // [00 reg 101] [disp32]
1079
1080 if (base == b101) {
1081 if (which == disp32_operand)
1082 return ip; // caller wants the disp32
1083 ip += 4; // skip the disp32
1084 }
1085 break;
1086
1087 case 1:
1088 // [01 reg 100][ss index base][disp8]
1089 // [01 reg 100][00 100 esp][disp8]
1090 // [01 reg base] [disp8]
1091 ip += 1; // skip the disp8
1092 break;
1093
1094 case 2:
1095 // [10 reg 100][ss index base][disp32]
1096 // [10 reg 100][00 100 esp][disp32]
1097 // [10 reg base] [disp32]
1098 if (which == disp32_operand)
1099 return ip; // caller wants the disp32
1100 ip += 4; // skip the disp32
1101 break;
1102
1103 case 3:
1104 // [11 reg base] (not a memory addressing mode)
1105 break;
1106 }
1107
1108 if (which == end_pc_operand) {
1109 return ip + tail_size;
1110 }
1111
1112 #ifdef _LP64
1113 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1114 #else
1115 assert(which == imm_operand, "instruction has only an imm field");
1116 #endif // LP64
1117 return ip;
1118 }
1119
1120 address Assembler::locate_next_instruction(address inst) {
1121 // Secretly share code with locate_operand:
1122 return locate_operand(inst, end_pc_operand);
1123 }
1124
1125
1126 #ifdef ASSERT
1127 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1128 address inst = inst_mark();
1129 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1130 address opnd;
1131
1132 Relocation* r = rspec.reloc();
1133 if (r->type() == relocInfo::none) {
1134 return;
1135 } else if (r->is_call() || format == call32_operand) {
1136 // assert(format == imm32_operand, "cannot specify a nonzero format");
1137 opnd = locate_operand(inst, call32_operand);
1138 } else if (r->is_data()) {
1139 assert(format == imm_operand || format == disp32_operand
1140 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1141 opnd = locate_operand(inst, (WhichOperand)format);
1142 } else {
1143 assert(format == imm_operand, "cannot specify a format");
1144 return;
1145 }
1146 assert(opnd == pc(), "must put operand where relocs can find it");
1147 }
1148 #endif // ASSERT
1149
1150 void Assembler::emit_operand(Register reg, Address adr,
1151 int rip_relative_correction) {
1152 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1153 adr._rspec,
1154 rip_relative_correction);
1155 }
1156
1157 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1158 if (adr.isxmmindex()) {
1159 emit_operand(reg, adr._base, adr._xmmindex, adr._scale, adr._disp, adr._rspec);
1160 } else {
1161 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1162 adr._rspec);
1163 }
1164 }
1165
1166 // Now the Assembler instructions (identical for 32/64 bits)
1167
1168 void Assembler::adcl(Address dst, int32_t imm32) {
1169 InstructionMark im(this);
1170 prefix(dst);
1171 emit_arith_operand(0x81, rdx, dst, imm32);
1172 }
1173
1174 void Assembler::adcl(Address dst, Register src) {
1175 InstructionMark im(this);
1176 prefix(dst, src);
1177 emit_int8(0x11);
1178 emit_operand(src, dst);
1179 }
1180
1181 void Assembler::adcl(Register dst, int32_t imm32) {
1182 prefix(dst);
1183 emit_arith(0x81, 0xD0, dst, imm32);
1184 }
1185
1186 void Assembler::adcl(Register dst, Address src) {
1187 InstructionMark im(this);
1188 prefix(src, dst);
1189 emit_int8(0x13);
1190 emit_operand(dst, src);
1191 }
1192
1193 void Assembler::adcl(Register dst, Register src) {
1194 (void) prefix_and_encode(dst->encoding(), src->encoding());
1195 emit_arith(0x13, 0xC0, dst, src);
1196 }
1197
1198 void Assembler::addl(Address dst, int32_t imm32) {
1199 InstructionMark im(this);
1200 prefix(dst);
1201 emit_arith_operand(0x81, rax, dst, imm32);
1202 }
1203
1204 void Assembler::addb(Address dst, int imm8) {
1205 InstructionMark im(this);
1206 prefix(dst);
1207 emit_int8((unsigned char)0x80);
1208 emit_operand(rax, dst, 1);
1209 emit_int8(imm8);
1210 }
1211
1212 void Assembler::addw(Address dst, int imm16) {
1213 InstructionMark im(this);
1214 emit_int8(0x66);
1215 prefix(dst);
1216 emit_int8((unsigned char)0x81);
1217 emit_operand(rax, dst, 2);
1218 emit_int16(imm16);
1219 }
1220
1221 void Assembler::addl(Address dst, Register src) {
1222 InstructionMark im(this);
1223 prefix(dst, src);
1224 emit_int8(0x01);
1225 emit_operand(src, dst);
1226 }
1227
1228 void Assembler::addl(Register dst, int32_t imm32) {
1229 prefix(dst);
1230 emit_arith(0x81, 0xC0, dst, imm32);
1231 }
1232
1233 void Assembler::addl(Register dst, Address src) {
1234 InstructionMark im(this);
1235 prefix(src, dst);
1236 emit_int8(0x03);
1237 emit_operand(dst, src);
1238 }
1239
1240 void Assembler::addl(Register dst, Register src) {
1241 (void) prefix_and_encode(dst->encoding(), src->encoding());
1242 emit_arith(0x03, 0xC0, dst, src);
1243 }
1244
1245 void Assembler::addr_nop_4() {
1246 assert(UseAddressNop, "no CPU support");
1247 // 4 bytes: NOP DWORD PTR [EAX+0]
1248 emit_int32(0x0F,
1249 0x1F,
1250 0x40, // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1251 0); // 8-bits offset (1 byte)
1252 }
1253
1254 void Assembler::addr_nop_5() {
1255 assert(UseAddressNop, "no CPU support");
1256 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1257 emit_int32(0x0F,
1258 0x1F,
1259 0x44, // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1260 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1261 emit_int8(0); // 8-bits offset (1 byte)
1262 }
1263
1264 void Assembler::addr_nop_7() {
1265 assert(UseAddressNop, "no CPU support");
1266 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1267 emit_int24(0x0F,
1268 0x1F,
1269 (unsigned char)0x80);
1270 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1271 emit_int32(0); // 32-bits offset (4 bytes)
1272 }
1273
1274 void Assembler::addr_nop_8() {
1275 assert(UseAddressNop, "no CPU support");
1276 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1277 emit_int32(0x0F,
1278 0x1F,
1279 (unsigned char)0x84,
1280 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1281 0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1282 emit_int32(0); // 32-bits offset (4 bytes)
1283 }
1284
1285 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1286 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1287 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1288 attributes.set_rex_vex_w_reverted();
1289 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1290 emit_int16(0x58, (0xC0 | encode));
1291 }
1292
1293 void Assembler::addsd(XMMRegister dst, Address src) {
1294 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1295 InstructionMark im(this);
1296 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1297 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1298 attributes.set_rex_vex_w_reverted();
1299 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1300 emit_int8(0x58);
1301 emit_operand(dst, src);
1302 }
1303
1304 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1305 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1306 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1307 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1308 emit_int16(0x58, (0xC0 | encode));
1309 }
1310
1311 void Assembler::addss(XMMRegister dst, Address src) {
1312 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1313 InstructionMark im(this);
1314 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1315 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1316 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1317 emit_int8(0x58);
1318 emit_operand(dst, src);
1319 }
1320
1321 void Assembler::aesdec(XMMRegister dst, Address src) {
1322 assert(VM_Version::supports_aes(), "");
1323 InstructionMark im(this);
1324 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1325 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1326 emit_int8((unsigned char)0xDE);
1327 emit_operand(dst, src);
1328 }
1329
1330 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1331 assert(VM_Version::supports_aes(), "");
1332 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1333 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1335 }
1336
1337 void Assembler::vaesdec(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1338 assert(VM_Version::supports_avx512_vaes(), "");
1339 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1340 attributes.set_is_evex_instruction();
1341 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342 emit_int16((unsigned char)0xDE, (0xC0 | encode));
1343 }
1344
1345
1346 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1347 assert(VM_Version::supports_aes(), "");
1348 InstructionMark im(this);
1349 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1350 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1351 emit_int8((unsigned char)0xDF);
1352 emit_operand(dst, src);
1353 }
1354
1355 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1356 assert(VM_Version::supports_aes(), "");
1357 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1358 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1359 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1360 }
1361
1362 void Assembler::vaesdeclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1363 assert(VM_Version::supports_avx512_vaes(), "");
1364 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1365 attributes.set_is_evex_instruction();
1366 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1367 emit_int16((unsigned char)0xDF, (0xC0 | encode));
1368 }
1369
1370 void Assembler::aesenc(XMMRegister dst, Address src) {
1371 assert(VM_Version::supports_aes(), "");
1372 InstructionMark im(this);
1373 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1374 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1375 emit_int8((unsigned char)0xDC);
1376 emit_operand(dst, src);
1377 }
1378
1379 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1380 assert(VM_Version::supports_aes(), "");
1381 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1382 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1383 emit_int16((unsigned char)0xDC, 0xC0 | encode);
1384 }
1385
1386 void Assembler::vaesenc(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1387 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1388 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1389 attributes.set_is_evex_instruction();
1390 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1391 emit_int16((unsigned char)0xDC, (0xC0 | encode));
1392 }
1393
1394 void Assembler::aesenclast(XMMRegister dst, Address src) {
1395 assert(VM_Version::supports_aes(), "");
1396 InstructionMark im(this);
1397 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1398 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1399 emit_int8((unsigned char)0xDD);
1400 emit_operand(dst, src);
1401 }
1402
1403 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1404 assert(VM_Version::supports_aes(), "");
1405 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1406 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1407 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1408 }
1409
1410 void Assembler::vaesenclast(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1411 assert(VM_Version::supports_avx512_vaes(), "requires vaes support/enabling");
1412 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1413 attributes.set_is_evex_instruction();
1414 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1415 emit_int16((unsigned char)0xDD, (0xC0 | encode));
1416 }
1417
1418 void Assembler::andl(Address dst, int32_t imm32) {
1419 InstructionMark im(this);
1420 prefix(dst);
1421 emit_int8((unsigned char)0x81);
1422 emit_operand(rsp, dst, 4);
1423 emit_int32(imm32);
1424 }
1425
1426 void Assembler::andl(Register dst, int32_t imm32) {
1427 prefix(dst);
1428 emit_arith(0x81, 0xE0, dst, imm32);
1429 }
1430
1431 void Assembler::andl(Register dst, Address src) {
1432 InstructionMark im(this);
1433 prefix(src, dst);
1434 emit_int8(0x23);
1435 emit_operand(dst, src);
1436 }
1437
1438 void Assembler::andl(Register dst, Register src) {
1439 (void) prefix_and_encode(dst->encoding(), src->encoding());
1440 emit_arith(0x23, 0xC0, dst, src);
1441 }
1442
1443 void Assembler::andnl(Register dst, Register src1, Register src2) {
1444 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1445 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1446 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1447 emit_int16((unsigned char)0xF2, (0xC0 | encode));
1448 }
1449
1450 void Assembler::andnl(Register dst, Register src1, Address src2) {
1451 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452 InstructionMark im(this);
1453 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1454 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455 emit_int8((unsigned char)0xF2);
1456 emit_operand(dst, src2);
1457 }
1458
1459 void Assembler::bsfl(Register dst, Register src) {
1460 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1461 emit_int24(0x0F,
1462 (unsigned char)0xBC,
1463 0xC0 | encode);
1464 }
1465
1466 void Assembler::bsrl(Register dst, Register src) {
1467 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1468 emit_int24(0x0F,
1469 (unsigned char)0xBD,
1470 0xC0 | encode);
1471 }
1472
1473 void Assembler::bswapl(Register reg) { // bswap
1474 int encode = prefix_and_encode(reg->encoding());
1475 emit_int16(0x0F, (0xC8 | encode));
1476 }
1477
1478 void Assembler::blsil(Register dst, Register src) {
1479 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1480 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1481 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1482 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1483 }
1484
1485 void Assembler::blsil(Register dst, Address src) {
1486 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1487 InstructionMark im(this);
1488 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1489 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1490 emit_int8((unsigned char)0xF3);
1491 emit_operand(rbx, src);
1492 }
1493
1494 void Assembler::blsmskl(Register dst, Register src) {
1495 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1496 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1497 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1498 emit_int16((unsigned char)0xF3,
1499 0xC0 | encode);
1500 }
1501
1502 void Assembler::blsmskl(Register dst, Address src) {
1503 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1504 InstructionMark im(this);
1505 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1506 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1507 emit_int8((unsigned char)0xF3);
1508 emit_operand(rdx, src);
1509 }
1510
1511 void Assembler::blsrl(Register dst, Register src) {
1512 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1513 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1514 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1515 emit_int16((unsigned char)0xF3, (0xC0 | encode));
1516 }
1517
1518 void Assembler::blsrl(Register dst, Address src) {
1519 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1520 InstructionMark im(this);
1521 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1522 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1523 emit_int8((unsigned char)0xF3);
1524 emit_operand(rcx, src);
1525 }
1526
1527 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1528 // suspect disp32 is always good
1529 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1530
1531 if (L.is_bound()) {
1532 const int long_size = 5;
1533 int offs = (int)( target(L) - pc() );
1534 assert(offs <= 0, "assembler error");
1535 InstructionMark im(this);
1536 // 1110 1000 #32-bit disp
1537 emit_int8((unsigned char)0xE8);
1538 emit_data(offs - long_size, rtype, operand);
1539 } else {
1540 InstructionMark im(this);
1541 // 1110 1000 #32-bit disp
1542 L.add_patch_at(code(), locator());
1543
1544 emit_int8((unsigned char)0xE8);
1545 emit_data(int(0), rtype, operand);
1546 }
1547 }
1548
1549 void Assembler::call(Register dst) {
1550 int encode = prefix_and_encode(dst->encoding());
1551 emit_int16((unsigned char)0xFF, (0xD0 | encode));
1552 }
1553
1554
1555 void Assembler::call(Address adr) {
1556 InstructionMark im(this);
1557 prefix(adr);
1558 emit_int8((unsigned char)0xFF);
1559 emit_operand(rdx, adr);
1560 }
1561
1562 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1563 InstructionMark im(this);
1564 emit_int8((unsigned char)0xE8);
1565 intptr_t disp = entry - (pc() + sizeof(int32_t));
1566 // Entry is NULL in case of a scratch emit.
1567 assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1568 // Technically, should use call32_operand, but this format is
1569 // implied by the fact that we're emitting a call instruction.
1570
1571 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1572 emit_data((int) disp, rspec, operand);
1573 }
1574
1575 void Assembler::cdql() {
1576 emit_int8((unsigned char)0x99);
1577 }
1578
1579 void Assembler::cld() {
1580 emit_int8((unsigned char)0xFC);
1581 }
1582
1583 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1584 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1585 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1586 emit_int24(0x0F,
1587 0x40 | cc,
1588 0xC0 | encode);
1589 }
1590
1591
1592 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1593 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1594 prefix(src, dst);
1595 emit_int16(0x0F, (0x40 | cc));
1596 emit_operand(dst, src);
1597 }
1598
1599 void Assembler::cmpb(Address dst, int imm8) {
1600 InstructionMark im(this);
1601 prefix(dst);
1602 emit_int8((unsigned char)0x80);
1603 emit_operand(rdi, dst, 1);
1604 emit_int8(imm8);
1605 }
1606
1607 void Assembler::cmpl(Address dst, int32_t imm32) {
1608 InstructionMark im(this);
1609 prefix(dst);
1610 emit_int8((unsigned char)0x81);
1611 emit_operand(rdi, dst, 4);
1612 emit_int32(imm32);
1613 }
1614
1615 void Assembler::cmpl(Register dst, int32_t imm32) {
1616 prefix(dst);
1617 emit_arith(0x81, 0xF8, dst, imm32);
1618 }
1619
1620 void Assembler::cmpl(Register dst, Register src) {
1621 (void) prefix_and_encode(dst->encoding(), src->encoding());
1622 emit_arith(0x3B, 0xC0, dst, src);
1623 }
1624
1625 void Assembler::cmpl(Register dst, Address src) {
1626 InstructionMark im(this);
1627 prefix(src, dst);
1628 emit_int8(0x3B);
1629 emit_operand(dst, src);
1630 }
1631
1632 void Assembler::cmpw(Address dst, int imm16) {
1633 InstructionMark im(this);
1634 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1635 emit_int16(0x66, (unsigned char)0x81);
1636 emit_operand(rdi, dst, 2);
1637 emit_int16(imm16);
1638 }
1639
1640 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1641 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1642 // The ZF is set if the compared values were equal, and cleared otherwise.
1643 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1644 InstructionMark im(this);
1645 prefix(adr, reg);
1646 emit_int16(0x0F, (unsigned char)0xB1);
1647 emit_operand(reg, adr);
1648 }
1649
1650 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1651 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1652 // The ZF is set if the compared values were equal, and cleared otherwise.
1653 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1654 InstructionMark im(this);
1655 prefix(adr, reg, true);
1656 emit_int16(0x0F, (unsigned char)0xB0);
1657 emit_operand(reg, adr);
1658 }
1659
1660 void Assembler::comisd(XMMRegister dst, Address src) {
1661 // NOTE: dbx seems to decode this as comiss even though the
1662 // 0x66 is there. Strangly ucomisd comes out correct
1663 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1664 InstructionMark im(this);
1665 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1666 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1667 attributes.set_rex_vex_w_reverted();
1668 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1669 emit_int8(0x2F);
1670 emit_operand(dst, src);
1671 }
1672
1673 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1674 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1675 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1676 attributes.set_rex_vex_w_reverted();
1677 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1678 emit_int16(0x2F, (0xC0 | encode));
1679 }
1680
1681 void Assembler::comiss(XMMRegister dst, Address src) {
1682 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1683 InstructionMark im(this);
1684 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1685 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1686 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1687 emit_int8(0x2F);
1688 emit_operand(dst, src);
1689 }
1690
1691 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1692 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1693 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1694 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1695 emit_int16(0x2F, (0xC0 | encode));
1696 }
1697
1698 void Assembler::cpuid() {
1699 emit_int16(0x0F, (unsigned char)0xA2);
1700 }
1701
1702 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1703 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1704 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1705 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1706 //
1707 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1708 //
1709 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1710 //
1711 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1712 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1713 assert(VM_Version::supports_sse4_2(), "");
1714 int8_t w = 0x01;
1715 Prefix p = Prefix_EMPTY;
1716
1717 emit_int8((unsigned char)0xF2);
1718 switch (sizeInBytes) {
1719 case 1:
1720 w = 0;
1721 break;
1722 case 2:
1723 case 4:
1724 break;
1725 LP64_ONLY(case 8:)
1726 // This instruction is not valid in 32 bits
1727 // Note:
1728 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1729 //
1730 // Page B - 72 Vol. 2C says
1731 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1732 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1733 // F0!!!
1734 // while 3 - 208 Vol. 2A
1735 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1736 //
1737 // the 0 on a last bit is reserved for a different flavor of this instruction :
1738 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1739 p = REX_W;
1740 break;
1741 default:
1742 assert(0, "Unsupported value for a sizeInBytes argument");
1743 break;
1744 }
1745 LP64_ONLY(prefix(crc, v, p);)
1746 emit_int32(0x0F,
1747 0x38,
1748 0xF0 | w,
1749 0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1750 }
1751
1752 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1753 assert(VM_Version::supports_sse4_2(), "");
1754 InstructionMark im(this);
1755 int8_t w = 0x01;
1756 Prefix p = Prefix_EMPTY;
1757
1758 emit_int8((int8_t)0xF2);
1759 switch (sizeInBytes) {
1760 case 1:
1761 w = 0;
1762 break;
1763 case 2:
1764 case 4:
1765 break;
1766 LP64_ONLY(case 8:)
1767 // This instruction is not valid in 32 bits
1768 p = REX_W;
1769 break;
1770 default:
1771 assert(0, "Unsupported value for a sizeInBytes argument");
1772 break;
1773 }
1774 LP64_ONLY(prefix(crc, adr, p);)
1775 emit_int24(0x0F, 0x38, (0xF0 | w));
1776 emit_operand(crc, adr);
1777 }
1778
1779 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1780 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1781 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1782 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1783 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1784 }
1785
1786 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1787 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1788 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1789 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1790 emit_int16(0x5B, (0xC0 | encode));
1791 }
1792
1793 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1794 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1795 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1796 attributes.set_rex_vex_w_reverted();
1797 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1798 emit_int16(0x5A, (0xC0 | encode));
1799 }
1800
1801 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1802 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1803 InstructionMark im(this);
1804 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1805 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1806 attributes.set_rex_vex_w_reverted();
1807 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1808 emit_int8(0x5A);
1809 emit_operand(dst, src);
1810 }
1811
1812 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1813 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1814 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1815 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1816 emit_int16(0x2A, (0xC0 | encode));
1817 }
1818
1819 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1820 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1821 InstructionMark im(this);
1822 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1823 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1824 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1825 emit_int8(0x2A);
1826 emit_operand(dst, src);
1827 }
1828
1829 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1830 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1831 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1832 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1833 emit_int16(0x2A, (0xC0 | encode));
1834 }
1835
1836 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1837 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1838 InstructionMark im(this);
1839 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1840 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1841 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1842 emit_int8(0x2A);
1843 emit_operand(dst, src);
1844 }
1845
1846 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1847 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1848 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1849 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1850 emit_int16(0x2A, (0xC0 | encode));
1851 }
1852
1853 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1854 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1856 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1857 emit_int16(0x5A, (0xC0 | encode));
1858 }
1859
1860 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1861 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1862 InstructionMark im(this);
1863 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1864 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1865 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1866 emit_int8(0x5A);
1867 emit_operand(dst, src);
1868 }
1869
1870
1871 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1872 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1873 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1874 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1875 emit_int16(0x2C, (0xC0 | encode));
1876 }
1877
1878 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1879 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1880 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1881 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1882 emit_int16(0x2C, (0xC0 | encode));
1883 }
1884
1885 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1886 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1887 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1888 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1889 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1890 emit_int16((unsigned char)0xE6, (0xC0 | encode));
1891 }
1892
1893 void Assembler::pabsb(XMMRegister dst, XMMRegister src) {
1894 assert(VM_Version::supports_ssse3(), "");
1895 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
1896 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1897 emit_int16(0x1C, (0xC0 | encode));
1898 }
1899
1900 void Assembler::pabsw(XMMRegister dst, XMMRegister src) {
1901 assert(VM_Version::supports_ssse3(), "");
1902 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
1903 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1904 emit_int16(0x1D, (0xC0 | encode));
1905 }
1906
1907 void Assembler::pabsd(XMMRegister dst, XMMRegister src) {
1908 assert(VM_Version::supports_ssse3(), "");
1909 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1910 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1911 emit_int16(0x1E, (0xC0 | encode));
1912 }
1913
1914 void Assembler::vpabsb(XMMRegister dst, XMMRegister src, int vector_len) {
1915 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
1916 vector_len == AVX_256bit? VM_Version::supports_avx2() :
1917 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
1918 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
1919 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1920 emit_int16(0x1C, (0xC0 | encode));
1921 }
1922
1923 void Assembler::vpabsw(XMMRegister dst, XMMRegister src, int vector_len) {
1924 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
1925 vector_len == AVX_256bit? VM_Version::supports_avx2() :
1926 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
1927 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
1928 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1929 emit_int16(0x1D, (0xC0 | encode));
1930 }
1931
1932 void Assembler::vpabsd(XMMRegister dst, XMMRegister src, int vector_len) {
1933 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
1934 vector_len == AVX_256bit? VM_Version::supports_avx2() :
1935 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, "");
1936 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1937 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1938 emit_int16(0x1E, (0xC0 | encode));
1939 }
1940
1941 void Assembler::evpabsq(XMMRegister dst, XMMRegister src, int vector_len) {
1942 assert(UseAVX > 2, "");
1943 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
1944 attributes.set_is_evex_instruction();
1945 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1946 emit_int16(0x1F, (0xC0 | encode));
1947 }
1948
1949 void Assembler::decl(Address dst) {
1950 // Don't use it directly. Use MacroAssembler::decrement() instead.
1951 InstructionMark im(this);
1952 prefix(dst);
1953 emit_int8((unsigned char)0xFF);
1954 emit_operand(rcx, dst);
1955 }
1956
1957 void Assembler::divsd(XMMRegister dst, Address src) {
1958 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1959 InstructionMark im(this);
1960 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1961 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1962 attributes.set_rex_vex_w_reverted();
1963 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1964 emit_int8(0x5E);
1965 emit_operand(dst, src);
1966 }
1967
1968 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1969 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1970 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1971 attributes.set_rex_vex_w_reverted();
1972 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1973 emit_int16(0x5E, (0xC0 | encode));
1974 }
1975
1976 void Assembler::divss(XMMRegister dst, Address src) {
1977 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1978 InstructionMark im(this);
1979 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1980 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1981 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1982 emit_int8(0x5E);
1983 emit_operand(dst, src);
1984 }
1985
1986 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1987 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1988 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1989 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1990 emit_int16(0x5E, (0xC0 | encode));
1991 }
1992
1993 void Assembler::hlt() {
1994 emit_int8((unsigned char)0xF4);
1995 }
1996
1997 void Assembler::idivl(Register src) {
1998 int encode = prefix_and_encode(src->encoding());
1999 emit_int16((unsigned char)0xF7, (0xF8 | encode));
2000 }
2001
2002 void Assembler::divl(Register src) { // Unsigned
2003 int encode = prefix_and_encode(src->encoding());
2004 emit_int16((unsigned char)0xF7, (0xF0 | encode));
2005 }
2006
2007 void Assembler::imull(Register src) {
2008 int encode = prefix_and_encode(src->encoding());
2009 emit_int16((unsigned char)0xF7, (0xE8 | encode));
2010 }
2011
2012 void Assembler::imull(Register dst, Register src) {
2013 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2014 emit_int24(0x0F,
2015 (unsigned char)0xAF,
2016 (0xC0 | encode));
2017 }
2018
2019
2020 void Assembler::imull(Register dst, Register src, int value) {
2021 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2022 if (is8bit(value)) {
2023 emit_int24(0x6B, (0xC0 | encode), value & 0xFF);
2024 } else {
2025 emit_int16(0x69, (0xC0 | encode));
2026 emit_int32(value);
2027 }
2028 }
2029
2030 void Assembler::imull(Register dst, Address src) {
2031 InstructionMark im(this);
2032 prefix(src, dst);
2033 emit_int16(0x0F, (unsigned char)0xAF);
2034 emit_operand(dst, src);
2035 }
2036
2037
2038 void Assembler::incl(Address dst) {
2039 // Don't use it directly. Use MacroAssembler::increment() instead.
2040 InstructionMark im(this);
2041 prefix(dst);
2042 emit_int8((unsigned char)0xFF);
2043 emit_operand(rax, dst);
2044 }
2045
2046 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
2047 InstructionMark im(this);
2048 assert((0 <= cc) && (cc < 16), "illegal cc");
2049 if (L.is_bound()) {
2050 address dst = target(L);
2051 assert(dst != NULL, "jcc most probably wrong");
2052
2053 const int short_size = 2;
2054 const int long_size = 6;
2055 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
2056 if (maybe_short && is8bit(offs - short_size)) {
2057 // 0111 tttn #8-bit disp
2058 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2059 } else {
2060 // 0000 1111 1000 tttn #32-bit disp
2061 assert(is_simm32(offs - long_size),
2062 "must be 32bit offset (call4)");
2063 emit_int16(0x0F, (0x80 | cc));
2064 emit_int32(offs - long_size);
2065 }
2066 } else {
2067 // Note: could eliminate cond. jumps to this jump if condition
2068 // is the same however, seems to be rather unlikely case.
2069 // Note: use jccb() if label to be bound is very close to get
2070 // an 8-bit displacement
2071 L.add_patch_at(code(), locator());
2072 emit_int16(0x0F, (0x80 | cc));
2073 emit_int32(0);
2074 }
2075 }
2076
2077 void Assembler::jccb_0(Condition cc, Label& L, const char* file, int line) {
2078 if (L.is_bound()) {
2079 const int short_size = 2;
2080 address entry = target(L);
2081 #ifdef ASSERT
2082 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2083 intptr_t delta = short_branch_delta();
2084 if (delta != 0) {
2085 dist += (dist < 0 ? (-delta) :delta);
2086 }
2087 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2088 #endif
2089 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2090 // 0111 tttn #8-bit disp
2091 emit_int16(0x70 | cc, (offs - short_size) & 0xFF);
2092 } else {
2093 InstructionMark im(this);
2094 L.add_patch_at(code(), locator(), file, line);
2095 emit_int16(0x70 | cc, 0);
2096 }
2097 }
2098
2099 void Assembler::jmp(Address adr) {
2100 InstructionMark im(this);
2101 prefix(adr);
2102 emit_int8((unsigned char)0xFF);
2103 emit_operand(rsp, adr);
2104 }
2105
2106 void Assembler::jmp(Label& L, bool maybe_short) {
2107 if (L.is_bound()) {
2108 address entry = target(L);
2109 assert(entry != NULL, "jmp most probably wrong");
2110 InstructionMark im(this);
2111 const int short_size = 2;
2112 const int long_size = 5;
2113 intptr_t offs = entry - pc();
2114 if (maybe_short && is8bit(offs - short_size)) {
2115 emit_int16((unsigned char)0xEB, ((offs - short_size) & 0xFF));
2116 } else {
2117 emit_int8((unsigned char)0xE9);
2118 emit_int32(offs - long_size);
2119 }
2120 } else {
2121 // By default, forward jumps are always 32-bit displacements, since
2122 // we can't yet know where the label will be bound. If you're sure that
2123 // the forward jump will not run beyond 256 bytes, use jmpb to
2124 // force an 8-bit displacement.
2125 InstructionMark im(this);
2126 L.add_patch_at(code(), locator());
2127 emit_int8((unsigned char)0xE9);
2128 emit_int32(0);
2129 }
2130 }
2131
2132 void Assembler::jmp(Register entry) {
2133 int encode = prefix_and_encode(entry->encoding());
2134 emit_int16((unsigned char)0xFF, (0xE0 | encode));
2135 }
2136
2137 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2138 InstructionMark im(this);
2139 emit_int8((unsigned char)0xE9);
2140 assert(dest != NULL, "must have a target");
2141 intptr_t disp = dest - (pc() + sizeof(int32_t));
2142 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2143 emit_data(disp, rspec.reloc(), call32_operand);
2144 }
2145
2146 void Assembler::jmpb_0(Label& L, const char* file, int line) {
2147 if (L.is_bound()) {
2148 const int short_size = 2;
2149 address entry = target(L);
2150 assert(entry != NULL, "jmp most probably wrong");
2151 #ifdef ASSERT
2152 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2153 intptr_t delta = short_branch_delta();
2154 if (delta != 0) {
2155 dist += (dist < 0 ? (-delta) :delta);
2156 }
2157 assert(is8bit(dist), "Dispacement too large for a short jmp at %s:%d", file, line);
2158 #endif
2159 intptr_t offs = entry - pc();
2160 emit_int16((unsigned char)0xEB, (offs - short_size) & 0xFF);
2161 } else {
2162 InstructionMark im(this);
2163 L.add_patch_at(code(), locator(), file, line);
2164 emit_int16((unsigned char)0xEB, 0);
2165 }
2166 }
2167
2168 void Assembler::ldmxcsr( Address src) {
2169 if (UseAVX > 0 ) {
2170 InstructionMark im(this);
2171 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2172 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2173 emit_int8((unsigned char)0xAE);
2174 emit_operand(as_Register(2), src);
2175 } else {
2176 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2177 InstructionMark im(this);
2178 prefix(src);
2179 emit_int16(0x0F, (unsigned char)0xAE);
2180 emit_operand(as_Register(2), src);
2181 }
2182 }
2183
2184 void Assembler::leal(Register dst, Address src) {
2185 InstructionMark im(this);
2186 #ifdef _LP64
2187 emit_int8(0x67); // addr32
2188 prefix(src, dst);
2189 #endif // LP64
2190 emit_int8((unsigned char)0x8D);
2191 emit_operand(dst, src);
2192 }
2193
2194 void Assembler::lfence() {
2195 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xE8);
2196 }
2197
2198 void Assembler::lock() {
2199 emit_int8((unsigned char)0xF0);
2200 }
2201
2202 void Assembler::lzcntl(Register dst, Register src) {
2203 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2204 emit_int8((unsigned char)0xF3);
2205 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2206 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
2207 }
2208
2209 // Emit mfence instruction
2210 void Assembler::mfence() {
2211 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2212 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF0);
2213 }
2214
2215 // Emit sfence instruction
2216 void Assembler::sfence() {
2217 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2218 emit_int24(0x0F, (unsigned char)0xAE, (unsigned char)0xF8);
2219 }
2220
2221 void Assembler::mov(Register dst, Register src) {
2222 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2223 }
2224
2225 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2226 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2227 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2228 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2229 attributes.set_rex_vex_w_reverted();
2230 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2231 emit_int16(0x28, (0xC0 | encode));
2232 }
2233
2234 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2235 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2236 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2237 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2238 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2239 emit_int16(0x28, (0xC0 | encode));
2240 }
2241
2242 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2243 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2244 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2245 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2246 emit_int16(0x16, (0xC0 | encode));
2247 }
2248
2249 void Assembler::movb(Register dst, Address src) {
2250 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2251 InstructionMark im(this);
2252 prefix(src, dst, true);
2253 emit_int8((unsigned char)0x8A);
2254 emit_operand(dst, src);
2255 }
2256
2257 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2258 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2259 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2260 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2261 attributes.set_rex_vex_w_reverted();
2262 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2263 emit_int16(0x12, 0xC0 | encode);
2264 }
2265
2266 void Assembler::kmovbl(KRegister dst, Register src) {
2267 assert(VM_Version::supports_avx512dq(), "");
2268 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2269 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2270 emit_int16((unsigned char)0x92, (0xC0 | encode));
2271 }
2272
2273 void Assembler::kmovbl(Register dst, KRegister src) {
2274 assert(VM_Version::supports_avx512dq(), "");
2275 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2276 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2277 emit_int16((unsigned char)0x93, (0xC0 | encode));
2278 }
2279
2280 void Assembler::kmovwl(KRegister dst, Register src) {
2281 assert(VM_Version::supports_evex(), "");
2282 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2283 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2284 emit_int16((unsigned char)0x92, (0xC0 | encode));
2285 }
2286
2287 void Assembler::kmovwl(Register dst, KRegister src) {
2288 assert(VM_Version::supports_evex(), "");
2289 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2290 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2291 emit_int16((unsigned char)0x93, (0xC0 | encode));
2292 }
2293
2294 void Assembler::kmovwl(KRegister dst, Address src) {
2295 assert(VM_Version::supports_evex(), "");
2296 InstructionMark im(this);
2297 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2298 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2299 emit_int8((unsigned char)0x90);
2300 emit_operand((Register)dst, src);
2301 }
2302
2303 void Assembler::kmovdl(KRegister dst, Register src) {
2304 assert(VM_Version::supports_avx512bw(), "");
2305 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2306 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2307 emit_int16((unsigned char)0x92, (0xC0 | encode));
2308 }
2309
2310 void Assembler::kmovdl(Register dst, KRegister src) {
2311 assert(VM_Version::supports_avx512bw(), "");
2312 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2313 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2314 emit_int16((unsigned char)0x93, (0xC0 | encode));
2315 }
2316
2317 void Assembler::kmovql(KRegister dst, KRegister src) {
2318 assert(VM_Version::supports_avx512bw(), "");
2319 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2320 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2321 emit_int16((unsigned char)0x90, (0xC0 | encode));
2322 }
2323
2324 void Assembler::kmovql(KRegister dst, Address src) {
2325 assert(VM_Version::supports_avx512bw(), "");
2326 InstructionMark im(this);
2327 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2328 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2329 emit_int8((unsigned char)0x90);
2330 emit_operand((Register)dst, src);
2331 }
2332
2333 void Assembler::kmovql(Address dst, KRegister src) {
2334 assert(VM_Version::supports_avx512bw(), "");
2335 InstructionMark im(this);
2336 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2337 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2338 emit_int8((unsigned char)0x90);
2339 emit_operand((Register)src, dst);
2340 }
2341
2342 void Assembler::kmovql(KRegister dst, Register src) {
2343 assert(VM_Version::supports_avx512bw(), "");
2344 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2345 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2346 emit_int16((unsigned char)0x92, (0xC0 | encode));
2347 }
2348
2349 void Assembler::kmovql(Register dst, KRegister src) {
2350 assert(VM_Version::supports_avx512bw(), "");
2351 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2352 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2353 emit_int16((unsigned char)0x93, (0xC0 | encode));
2354 }
2355
2356 void Assembler::knotwl(KRegister dst, KRegister src) {
2357 assert(VM_Version::supports_evex(), "");
2358 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2359 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2360 emit_int16(0x44, (0xC0 | encode));
2361 }
2362
2363 // This instruction produces ZF or CF flags
2364 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2365 assert(VM_Version::supports_avx512dq(), "");
2366 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2367 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2368 emit_int16((unsigned char)0x98, (0xC0 | encode));
2369 }
2370
2371 // This instruction produces ZF or CF flags
2372 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2373 assert(VM_Version::supports_evex(), "");
2374 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2375 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2376 emit_int16((unsigned char)0x98, (0xC0 | encode));
2377 }
2378
2379 // This instruction produces ZF or CF flags
2380 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2381 assert(VM_Version::supports_avx512bw(), "");
2382 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2383 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2384 emit_int16((unsigned char)0x98, (0xC0 | encode));
2385 }
2386
2387 // This instruction produces ZF or CF flags
2388 void Assembler::kortestql(KRegister src1, KRegister src2) {
2389 assert(VM_Version::supports_avx512bw(), "");
2390 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2391 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2392 emit_int16((unsigned char)0x98, (0xC0 | encode));
2393 }
2394
2395 // This instruction produces ZF or CF flags
2396 void Assembler::ktestql(KRegister src1, KRegister src2) {
2397 assert(VM_Version::supports_avx512bw(), "");
2398 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2399 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2400 emit_int16((unsigned char)0x99, (0xC0 | encode));
2401 }
2402
2403 void Assembler::ktestq(KRegister src1, KRegister src2) {
2404 assert(VM_Version::supports_avx512bw(), "");
2405 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2406 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2407 emit_int16((unsigned char)0x99, (0xC0 | encode));
2408 }
2409
2410 void Assembler::ktestd(KRegister src1, KRegister src2) {
2411 assert(VM_Version::supports_avx512bw(), "");
2412 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2413 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2414 emit_int16((unsigned char)0x99, (0xC0 | encode));
2415 }
2416
2417 void Assembler::movb(Address dst, int imm8) {
2418 InstructionMark im(this);
2419 prefix(dst);
2420 emit_int8((unsigned char)0xC6);
2421 emit_operand(rax, dst, 1);
2422 emit_int8(imm8);
2423 }
2424
2425
2426 void Assembler::movb(Address dst, Register src) {
2427 assert(src->has_byte_register(), "must have byte register");
2428 InstructionMark im(this);
2429 prefix(dst, src, true);
2430 emit_int8((unsigned char)0x88);
2431 emit_operand(src, dst);
2432 }
2433
2434 void Assembler::movdl(XMMRegister dst, Register src) {
2435 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2436 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2437 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2438 emit_int16(0x6E, (0xC0 | encode));
2439 }
2440
2441 void Assembler::movdl(Register dst, XMMRegister src) {
2442 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2443 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2444 // swap src/dst to get correct prefix
2445 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2446 emit_int16(0x7E, (0xC0 | encode));
2447 }
2448
2449 void Assembler::movdl(XMMRegister dst, Address src) {
2450 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2451 InstructionMark im(this);
2452 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2453 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2454 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2455 emit_int8(0x6E);
2456 emit_operand(dst, src);
2457 }
2458
2459 void Assembler::movdl(Address dst, XMMRegister src) {
2460 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2461 InstructionMark im(this);
2462 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2463 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2464 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2465 emit_int8(0x7E);
2466 emit_operand(src, dst);
2467 }
2468
2469 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2470 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2471 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2472 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2473 emit_int16(0x6F, (0xC0 | encode));
2474 }
2475
2476 void Assembler::movdqa(XMMRegister dst, Address src) {
2477 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2478 InstructionMark im(this);
2479 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2480 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2481 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2482 emit_int8(0x6F);
2483 emit_operand(dst, src);
2484 }
2485
2486 void Assembler::movdqu(XMMRegister dst, Address src) {
2487 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2488 InstructionMark im(this);
2489 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2490 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2491 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2492 emit_int8(0x6F);
2493 emit_operand(dst, src);
2494 }
2495
2496 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2497 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2498 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2499 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2500 emit_int16(0x6F, (0xC0 | encode));
2501 }
2502
2503 void Assembler::movdqu(Address dst, XMMRegister src) {
2504 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2505 InstructionMark im(this);
2506 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2507 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2508 attributes.reset_is_clear_context();
2509 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2510 emit_int8(0x7F);
2511 emit_operand(src, dst);
2512 }
2513
2514 // Move Unaligned 256bit Vector
2515 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2516 assert(UseAVX > 0, "");
2517 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2518 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2519 emit_int16(0x6F, (0xC0 | encode));
2520 }
2521
2522 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2523 assert(UseAVX > 0, "");
2524 InstructionMark im(this);
2525 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2526 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2527 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2528 emit_int8(0x6F);
2529 emit_operand(dst, src);
2530 }
2531
2532 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2533 assert(UseAVX > 0, "");
2534 InstructionMark im(this);
2535 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2536 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2537 attributes.reset_is_clear_context();
2538 // swap src<->dst for encoding
2539 assert(src != xnoreg, "sanity");
2540 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2541 emit_int8(0x7F);
2542 emit_operand(src, dst);
2543 }
2544
2545 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2546 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2547 assert(VM_Version::supports_evex(), "");
2548 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2549 attributes.set_is_evex_instruction();
2550 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2551 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2552 emit_int16(0x6F, (0xC0 | encode));
2553 }
2554
2555 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2556 assert(VM_Version::supports_evex(), "");
2557 InstructionMark im(this);
2558 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2559 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2560 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2561 attributes.set_is_evex_instruction();
2562 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2563 emit_int8(0x6F);
2564 emit_operand(dst, src);
2565 }
2566
2567 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2568 assert(VM_Version::supports_evex(), "");
2569 assert(src != xnoreg, "sanity");
2570 InstructionMark im(this);
2571 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2572 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2573 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2574 attributes.set_is_evex_instruction();
2575 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2576 emit_int8(0x7F);
2577 emit_operand(src, dst);
2578 }
2579
2580 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2581 assert(VM_Version::supports_avx512vlbw(), "");
2582 InstructionMark im(this);
2583 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2584 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2585 attributes.set_embedded_opmask_register_specifier(mask);
2586 attributes.set_is_evex_instruction();
2587 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2588 emit_int8(0x6F);
2589 emit_operand(dst, src);
2590 }
2591
2592 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2593 assert(VM_Version::supports_evex(), "");
2594 InstructionMark im(this);
2595 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2596 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2597 attributes.set_is_evex_instruction();
2598 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2599 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2600 emit_int8(0x6F);
2601 emit_operand(dst, src);
2602 }
2603
2604 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2605 assert(VM_Version::supports_avx512vlbw(), "");
2606 InstructionMark im(this);
2607 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2608 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2609 attributes.set_embedded_opmask_register_specifier(mask);
2610 attributes.set_is_evex_instruction();
2611 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2612 emit_int8(0x6F);
2613 emit_operand(dst, src);
2614 }
2615
2616 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2617 assert(VM_Version::supports_evex(), "");
2618 assert(src != xnoreg, "sanity");
2619 InstructionMark im(this);
2620 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2621 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2622 attributes.set_is_evex_instruction();
2623 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2624 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2625 emit_int8(0x7F);
2626 emit_operand(src, dst);
2627 }
2628
2629 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2630 assert(VM_Version::supports_avx512vlbw(), "");
2631 assert(src != xnoreg, "sanity");
2632 InstructionMark im(this);
2633 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2634 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2635 attributes.reset_is_clear_context();
2636 attributes.set_embedded_opmask_register_specifier(mask);
2637 attributes.set_is_evex_instruction();
2638 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2639 emit_int8(0x7F);
2640 emit_operand(src, dst);
2641 }
2642
2643 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2644 assert(VM_Version::supports_evex(), "");
2645 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2646 attributes.set_is_evex_instruction();
2647 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2648 emit_int16(0x6F, (0xC0 | encode));
2649 }
2650
2651 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2652 assert(VM_Version::supports_evex(), "");
2653 InstructionMark im(this);
2654 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ true);
2655 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2656 attributes.set_is_evex_instruction();
2657 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2658 emit_int8(0x6F);
2659 emit_operand(dst, src);
2660 }
2661
2662 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2663 assert(VM_Version::supports_evex(), "");
2664 assert(src != xnoreg, "sanity");
2665 InstructionMark im(this);
2666 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2667 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2668 attributes.reset_is_clear_context();
2669 attributes.set_is_evex_instruction();
2670 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2671 emit_int8(0x7F);
2672 emit_operand(src, dst);
2673 }
2674
2675 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2676 assert(VM_Version::supports_evex(), "");
2677 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2678 attributes.set_is_evex_instruction();
2679 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2680 emit_int16(0x6F, (0xC0 | encode));
2681 }
2682
2683 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2684 assert(VM_Version::supports_evex(), "");
2685 InstructionMark im(this);
2686 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2687 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2688 attributes.set_is_evex_instruction();
2689 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2690 emit_int8(0x6F);
2691 emit_operand(dst, src);
2692 }
2693
2694 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2695 assert(VM_Version::supports_evex(), "");
2696 assert(src != xnoreg, "sanity");
2697 InstructionMark im(this);
2698 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2699 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2700 attributes.reset_is_clear_context();
2701 attributes.set_is_evex_instruction();
2702 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2703 emit_int8(0x7F);
2704 emit_operand(src, dst);
2705 }
2706
2707 // Uses zero extension on 64bit
2708
2709 void Assembler::movl(Register dst, int32_t imm32) {
2710 int encode = prefix_and_encode(dst->encoding());
2711 emit_int8(0xB8 | encode);
2712 emit_int32(imm32);
2713 }
2714
2715 void Assembler::movl(Register dst, Register src) {
2716 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2717 emit_int16((unsigned char)0x8B, (0xC0 | encode));
2718 }
2719
2720 void Assembler::movl(Register dst, Address src) {
2721 InstructionMark im(this);
2722 prefix(src, dst);
2723 emit_int8((unsigned char)0x8B);
2724 emit_operand(dst, src);
2725 }
2726
2727 void Assembler::movl(Address dst, int32_t imm32) {
2728 InstructionMark im(this);
2729 prefix(dst);
2730 emit_int8((unsigned char)0xC7);
2731 emit_operand(rax, dst, 4);
2732 emit_int32(imm32);
2733 }
2734
2735 void Assembler::movl(Address dst, Register src) {
2736 InstructionMark im(this);
2737 prefix(dst, src);
2738 emit_int8((unsigned char)0x89);
2739 emit_operand(src, dst);
2740 }
2741
2742 // New cpus require to use movsd and movss to avoid partial register stall
2743 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2744 // The selection is done in MacroAssembler::movdbl() and movflt().
2745 void Assembler::movlpd(XMMRegister dst, Address src) {
2746 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2747 InstructionMark im(this);
2748 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2749 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2750 attributes.set_rex_vex_w_reverted();
2751 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2752 emit_int8(0x12);
2753 emit_operand(dst, src);
2754 }
2755
2756 void Assembler::movq(XMMRegister dst, Address src) {
2757 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2758 InstructionMark im(this);
2759 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2760 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2761 attributes.set_rex_vex_w_reverted();
2762 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2763 emit_int8(0x7E);
2764 emit_operand(dst, src);
2765 }
2766
2767 void Assembler::movq(Address dst, XMMRegister src) {
2768 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2769 InstructionMark im(this);
2770 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2771 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2772 attributes.set_rex_vex_w_reverted();
2773 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2774 emit_int8((unsigned char)0xD6);
2775 emit_operand(src, dst);
2776 }
2777
2778 void Assembler::movsbl(Register dst, Address src) { // movsxb
2779 InstructionMark im(this);
2780 prefix(src, dst);
2781 emit_int16(0x0F, (unsigned char)0xBE);
2782 emit_operand(dst, src);
2783 }
2784
2785 void Assembler::movsbl(Register dst, Register src) { // movsxb
2786 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2787 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2788 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
2789 }
2790
2791 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2792 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2793 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2794 attributes.set_rex_vex_w_reverted();
2795 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2796 emit_int16(0x10, (0xC0 | encode));
2797 }
2798
2799 void Assembler::movsd(XMMRegister dst, Address src) {
2800 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2801 InstructionMark im(this);
2802 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2803 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2804 attributes.set_rex_vex_w_reverted();
2805 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2806 emit_int8(0x10);
2807 emit_operand(dst, src);
2808 }
2809
2810 void Assembler::movsd(Address dst, XMMRegister src) {
2811 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2812 InstructionMark im(this);
2813 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2814 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2815 attributes.reset_is_clear_context();
2816 attributes.set_rex_vex_w_reverted();
2817 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2818 emit_int8(0x11);
2819 emit_operand(src, dst);
2820 }
2821
2822 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2823 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2824 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2825 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2826 emit_int16(0x10, (0xC0 | encode));
2827 }
2828
2829 void Assembler::movss(XMMRegister dst, Address src) {
2830 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2831 InstructionMark im(this);
2832 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2833 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2834 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2835 emit_int8(0x10);
2836 emit_operand(dst, src);
2837 }
2838
2839 void Assembler::movss(Address dst, XMMRegister src) {
2840 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2841 InstructionMark im(this);
2842 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2843 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2844 attributes.reset_is_clear_context();
2845 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2846 emit_int8(0x11);
2847 emit_operand(src, dst);
2848 }
2849
2850 void Assembler::movswl(Register dst, Address src) { // movsxw
2851 InstructionMark im(this);
2852 prefix(src, dst);
2853 emit_int16(0x0F, (unsigned char)0xBF);
2854 emit_operand(dst, src);
2855 }
2856
2857 void Assembler::movswl(Register dst, Register src) { // movsxw
2858 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2859 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
2860 }
2861
2862 void Assembler::movw(Address dst, int imm16) {
2863 InstructionMark im(this);
2864
2865 emit_int8(0x66); // switch to 16-bit mode
2866 prefix(dst);
2867 emit_int8((unsigned char)0xC7);
2868 emit_operand(rax, dst, 2);
2869 emit_int16(imm16);
2870 }
2871
2872 void Assembler::movw(Register dst, Address src) {
2873 InstructionMark im(this);
2874 emit_int8(0x66);
2875 prefix(src, dst);
2876 emit_int8((unsigned char)0x8B);
2877 emit_operand(dst, src);
2878 }
2879
2880 void Assembler::movw(Address dst, Register src) {
2881 InstructionMark im(this);
2882 emit_int8(0x66);
2883 prefix(dst, src);
2884 emit_int8((unsigned char)0x89);
2885 emit_operand(src, dst);
2886 }
2887
2888 void Assembler::movzbl(Register dst, Address src) { // movzxb
2889 InstructionMark im(this);
2890 prefix(src, dst);
2891 emit_int16(0x0F, (unsigned char)0xB6);
2892 emit_operand(dst, src);
2893 }
2894
2895 void Assembler::movzbl(Register dst, Register src) { // movzxb
2896 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2897 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2898 emit_int24(0x0F, (unsigned char)0xB6, 0xC0 | encode);
2899 }
2900
2901 void Assembler::movzwl(Register dst, Address src) { // movzxw
2902 InstructionMark im(this);
2903 prefix(src, dst);
2904 emit_int16(0x0F, (unsigned char)0xB7);
2905 emit_operand(dst, src);
2906 }
2907
2908 void Assembler::movzwl(Register dst, Register src) { // movzxw
2909 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2910 emit_int24(0x0F, (unsigned char)0xB7, 0xC0 | encode);
2911 }
2912
2913 void Assembler::mull(Address src) {
2914 InstructionMark im(this);
2915 prefix(src);
2916 emit_int8((unsigned char)0xF7);
2917 emit_operand(rsp, src);
2918 }
2919
2920 void Assembler::mull(Register src) {
2921 int encode = prefix_and_encode(src->encoding());
2922 emit_int16((unsigned char)0xF7, (0xE0 | encode));
2923 }
2924
2925 void Assembler::mulsd(XMMRegister dst, Address src) {
2926 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2927 InstructionMark im(this);
2928 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2929 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2930 attributes.set_rex_vex_w_reverted();
2931 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2932 emit_int8(0x59);
2933 emit_operand(dst, src);
2934 }
2935
2936 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2937 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2938 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2939 attributes.set_rex_vex_w_reverted();
2940 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2941 emit_int16(0x59, (0xC0 | encode));
2942 }
2943
2944 void Assembler::mulss(XMMRegister dst, Address src) {
2945 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2946 InstructionMark im(this);
2947 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2948 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2949 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2950 emit_int8(0x59);
2951 emit_operand(dst, src);
2952 }
2953
2954 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2955 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2956 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2957 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2958 emit_int16(0x59, (0xC0 | encode));
2959 }
2960
2961 void Assembler::negl(Register dst) {
2962 int encode = prefix_and_encode(dst->encoding());
2963 emit_int16((unsigned char)0xF7, (0xD8 | encode));
2964 }
2965
2966 void Assembler::nop(int i) {
2967 #ifdef ASSERT
2968 assert(i > 0, " ");
2969 // The fancy nops aren't currently recognized by debuggers making it a
2970 // pain to disassemble code while debugging. If asserts are on clearly
2971 // speed is not an issue so simply use the single byte traditional nop
2972 // to do alignment.
2973
2974 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2975 return;
2976
2977 #endif // ASSERT
2978
2979 if (UseAddressNop && VM_Version::is_intel()) {
2980 //
2981 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2982 // 1: 0x90
2983 // 2: 0x66 0x90
2984 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2985 // 4: 0x0F 0x1F 0x40 0x00
2986 // 5: 0x0F 0x1F 0x44 0x00 0x00
2987 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2988 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2989 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2990 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2991 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2992 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2993
2994 // The rest coding is Intel specific - don't use consecutive address nops
2995
2996 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2997 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2998 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2999 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3000
3001 while(i >= 15) {
3002 // For Intel don't generate consecutive addess nops (mix with regular nops)
3003 i -= 15;
3004 emit_int24(0x66, 0x66, 0x66);
3005 addr_nop_8();
3006 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3007 }
3008 switch (i) {
3009 case 14:
3010 emit_int8(0x66); // size prefix
3011 case 13:
3012 emit_int8(0x66); // size prefix
3013 case 12:
3014 addr_nop_8();
3015 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3016 break;
3017 case 11:
3018 emit_int8(0x66); // size prefix
3019 case 10:
3020 emit_int8(0x66); // size prefix
3021 case 9:
3022 emit_int8(0x66); // size prefix
3023 case 8:
3024 addr_nop_8();
3025 break;
3026 case 7:
3027 addr_nop_7();
3028 break;
3029 case 6:
3030 emit_int8(0x66); // size prefix
3031 case 5:
3032 addr_nop_5();
3033 break;
3034 case 4:
3035 addr_nop_4();
3036 break;
3037 case 3:
3038 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3039 emit_int8(0x66); // size prefix
3040 case 2:
3041 emit_int8(0x66); // size prefix
3042 case 1:
3043 emit_int8((unsigned char)0x90);
3044 // nop
3045 break;
3046 default:
3047 assert(i == 0, " ");
3048 }
3049 return;
3050 }
3051 if (UseAddressNop && VM_Version::is_amd_family()) {
3052 //
3053 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3054 // 1: 0x90
3055 // 2: 0x66 0x90
3056 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3057 // 4: 0x0F 0x1F 0x40 0x00
3058 // 5: 0x0F 0x1F 0x44 0x00 0x00
3059 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3060 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3061 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3062 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3063 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3064 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3065
3066 // The rest coding is AMD specific - use consecutive address nops
3067
3068 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3069 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3070 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3071 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3072 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3073 // Size prefixes (0x66) are added for larger sizes
3074
3075 while(i >= 22) {
3076 i -= 11;
3077 emit_int24(0x66, 0x66, 0x66);
3078 addr_nop_8();
3079 }
3080 // Generate first nop for size between 21-12
3081 switch (i) {
3082 case 21:
3083 i -= 1;
3084 emit_int8(0x66); // size prefix
3085 case 20:
3086 case 19:
3087 i -= 1;
3088 emit_int8(0x66); // size prefix
3089 case 18:
3090 case 17:
3091 i -= 1;
3092 emit_int8(0x66); // size prefix
3093 case 16:
3094 case 15:
3095 i -= 8;
3096 addr_nop_8();
3097 break;
3098 case 14:
3099 case 13:
3100 i -= 7;
3101 addr_nop_7();
3102 break;
3103 case 12:
3104 i -= 6;
3105 emit_int8(0x66); // size prefix
3106 addr_nop_5();
3107 break;
3108 default:
3109 assert(i < 12, " ");
3110 }
3111
3112 // Generate second nop for size between 11-1
3113 switch (i) {
3114 case 11:
3115 emit_int8(0x66); // size prefix
3116 case 10:
3117 emit_int8(0x66); // size prefix
3118 case 9:
3119 emit_int8(0x66); // size prefix
3120 case 8:
3121 addr_nop_8();
3122 break;
3123 case 7:
3124 addr_nop_7();
3125 break;
3126 case 6:
3127 emit_int8(0x66); // size prefix
3128 case 5:
3129 addr_nop_5();
3130 break;
3131 case 4:
3132 addr_nop_4();
3133 break;
3134 case 3:
3135 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3136 emit_int8(0x66); // size prefix
3137 case 2:
3138 emit_int8(0x66); // size prefix
3139 case 1:
3140 emit_int8((unsigned char)0x90);
3141 // nop
3142 break;
3143 default:
3144 assert(i == 0, " ");
3145 }
3146 return;
3147 }
3148
3149 if (UseAddressNop && VM_Version::is_zx()) {
3150 //
3151 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3152 // 1: 0x90
3153 // 2: 0x66 0x90
3154 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3155 // 4: 0x0F 0x1F 0x40 0x00
3156 // 5: 0x0F 0x1F 0x44 0x00 0x00
3157 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3158 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3159 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3160 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3161 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3162 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3163
3164 // The rest coding is ZX specific - don't use consecutive address nops
3165
3166 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3167 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3168 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3169 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3170
3171 while (i >= 15) {
3172 // For ZX don't generate consecutive addess nops (mix with regular nops)
3173 i -= 15;
3174 emit_int24(0x66, 0x66, 0x66);
3175 addr_nop_8();
3176 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3177 }
3178 switch (i) {
3179 case 14:
3180 emit_int8(0x66); // size prefix
3181 case 13:
3182 emit_int8(0x66); // size prefix
3183 case 12:
3184 addr_nop_8();
3185 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3186 break;
3187 case 11:
3188 emit_int8(0x66); // size prefix
3189 case 10:
3190 emit_int8(0x66); // size prefix
3191 case 9:
3192 emit_int8(0x66); // size prefix
3193 case 8:
3194 addr_nop_8();
3195 break;
3196 case 7:
3197 addr_nop_7();
3198 break;
3199 case 6:
3200 emit_int8(0x66); // size prefix
3201 case 5:
3202 addr_nop_5();
3203 break;
3204 case 4:
3205 addr_nop_4();
3206 break;
3207 case 3:
3208 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3209 emit_int8(0x66); // size prefix
3210 case 2:
3211 emit_int8(0x66); // size prefix
3212 case 1:
3213 emit_int8((unsigned char)0x90);
3214 // nop
3215 break;
3216 default:
3217 assert(i == 0, " ");
3218 }
3219 return;
3220 }
3221
3222 // Using nops with size prefixes "0x66 0x90".
3223 // From AMD Optimization Guide:
3224 // 1: 0x90
3225 // 2: 0x66 0x90
3226 // 3: 0x66 0x66 0x90
3227 // 4: 0x66 0x66 0x66 0x90
3228 // 5: 0x66 0x66 0x90 0x66 0x90
3229 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
3230 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3231 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3232 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3233 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3234 //
3235 while (i > 12) {
3236 i -= 4;
3237 emit_int32(0x66, 0x66, 0x66, (unsigned char)0x90);
3238 }
3239 // 1 - 12 nops
3240 if (i > 8) {
3241 if (i > 9) {
3242 i -= 1;
3243 emit_int8(0x66);
3244 }
3245 i -= 3;
3246 emit_int24(0x66, 0x66, (unsigned char)0x90);
3247 }
3248 // 1 - 8 nops
3249 if (i > 4) {
3250 if (i > 6) {
3251 i -= 1;
3252 emit_int8(0x66);
3253 }
3254 i -= 3;
3255 emit_int24(0x66, 0x66, (unsigned char)0x90);
3256 }
3257 switch (i) {
3258 case 4:
3259 emit_int8(0x66);
3260 case 3:
3261 emit_int8(0x66);
3262 case 2:
3263 emit_int8(0x66);
3264 case 1:
3265 emit_int8((unsigned char)0x90);
3266 break;
3267 default:
3268 assert(i == 0, " ");
3269 }
3270 }
3271
3272 void Assembler::notl(Register dst) {
3273 int encode = prefix_and_encode(dst->encoding());
3274 emit_int16((unsigned char)0xF7, (0xD0 | encode));
3275 }
3276
3277 void Assembler::orl(Address dst, int32_t imm32) {
3278 InstructionMark im(this);
3279 prefix(dst);
3280 emit_arith_operand(0x81, rcx, dst, imm32);
3281 }
3282
3283 void Assembler::orl(Register dst, int32_t imm32) {
3284 prefix(dst);
3285 emit_arith(0x81, 0xC8, dst, imm32);
3286 }
3287
3288 void Assembler::orl(Register dst, Address src) {
3289 InstructionMark im(this);
3290 prefix(src, dst);
3291 emit_int8(0x0B);
3292 emit_operand(dst, src);
3293 }
3294
3295 void Assembler::orl(Register dst, Register src) {
3296 (void) prefix_and_encode(dst->encoding(), src->encoding());
3297 emit_arith(0x0B, 0xC0, dst, src);
3298 }
3299
3300 void Assembler::orl(Address dst, Register src) {
3301 InstructionMark im(this);
3302 prefix(dst, src);
3303 emit_int8(0x09);
3304 emit_operand(src, dst);
3305 }
3306
3307 void Assembler::orb(Address dst, int imm8) {
3308 InstructionMark im(this);
3309 prefix(dst);
3310 emit_int8((unsigned char)0x80);
3311 emit_operand(rcx, dst, 1);
3312 emit_int8(imm8);
3313 }
3314
3315 void Assembler::packuswb(XMMRegister dst, Address src) {
3316 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3317 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3318 InstructionMark im(this);
3319 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3320 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3321 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3322 emit_int8(0x67);
3323 emit_operand(dst, src);
3324 }
3325
3326 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3327 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3328 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3329 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3330 emit_int16(0x67, (0xC0 | encode));
3331 }
3332
3333 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3334 assert(UseAVX > 0, "some form of AVX must be enabled");
3335 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3336 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3337 emit_int16(0x67, (0xC0 | encode));
3338 }
3339
3340 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3341 assert(VM_Version::supports_avx2(), "");
3342 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3343 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3344 emit_int24(0x00, (0xC0 | encode), imm8);
3345 }
3346
3347 void Assembler::vpermq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3348 assert(UseAVX > 2, "requires AVX512F");
3349 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3350 attributes.set_is_evex_instruction();
3351 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3352 emit_int16(0x36, (0xC0 | encode));
3353 }
3354
3355 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3356 assert(VM_Version::supports_avx2(), "");
3357 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3358 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3359 emit_int24(0x46, (0xC0 | encode), imm8);
3360 }
3361
3362 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3363 assert(VM_Version::supports_avx(), "");
3364 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3365 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3366 emit_int24(0x06, (0xC0 | encode), imm8);
3367 }
3368
3369 void Assembler::evpermi2q(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3370 assert(VM_Version::supports_evex(), "");
3371 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3372 attributes.set_is_evex_instruction();
3373 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3374 emit_int16(0x76, (0xC0 | encode));
3375 }
3376
3377
3378 void Assembler::pause() {
3379 emit_int16((unsigned char)0xF3, (unsigned char)0x90);
3380 }
3381
3382 void Assembler::ud2() {
3383 emit_int16(0x0F, 0x0B);
3384 }
3385
3386 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3387 assert(VM_Version::supports_sse4_2(), "");
3388 InstructionMark im(this);
3389 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3390 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3391 emit_int8(0x61);
3392 emit_operand(dst, src);
3393 emit_int8(imm8);
3394 }
3395
3396 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3397 assert(VM_Version::supports_sse4_2(), "");
3398 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3399 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3400 emit_int24(0x61, (0xC0 | encode), imm8);
3401 }
3402
3403 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3404 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3405 assert(VM_Version::supports_sse2(), "");
3406 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3407 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3408 emit_int16(0x74, (0xC0 | encode));
3409 }
3410
3411 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3412 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3413 assert(VM_Version::supports_avx(), "");
3414 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3415 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3416 emit_int16(0x74, (0xC0 | encode));
3417 }
3418
3419 // In this context, kdst is written the mask used to process the equal components
3420 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3421 assert(VM_Version::supports_avx512bw(), "");
3422 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3423 attributes.set_is_evex_instruction();
3424 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3425 emit_int16(0x74, (0xC0 | encode));
3426 }
3427
3428 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3429 assert(VM_Version::supports_avx512vlbw(), "");
3430 InstructionMark im(this);
3431 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3432 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3433 attributes.set_is_evex_instruction();
3434 int dst_enc = kdst->encoding();
3435 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3436 emit_int8(0x64);
3437 emit_operand(as_Register(dst_enc), src);
3438 }
3439
3440 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3441 assert(VM_Version::supports_avx512vlbw(), "");
3442 InstructionMark im(this);
3443 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3444 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3445 attributes.reset_is_clear_context();
3446 attributes.set_embedded_opmask_register_specifier(mask);
3447 attributes.set_is_evex_instruction();
3448 int dst_enc = kdst->encoding();
3449 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3450 emit_int8(0x64);
3451 emit_operand(as_Register(dst_enc), src);
3452 }
3453
3454 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3455 assert(VM_Version::supports_avx512vlbw(), "");
3456 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3457 attributes.set_is_evex_instruction();
3458 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3459 emit_int24(0x3E, (0xC0 | encode), vcc);
3460 }
3461
3462 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3463 assert(VM_Version::supports_avx512vlbw(), "");
3464 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3465 attributes.reset_is_clear_context();
3466 attributes.set_embedded_opmask_register_specifier(mask);
3467 attributes.set_is_evex_instruction();
3468 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3469 emit_int24(0x3E, (0xC0 | encode), vcc);
3470 }
3471
3472 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3473 assert(VM_Version::supports_avx512vlbw(), "");
3474 InstructionMark im(this);
3475 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3476 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3477 attributes.set_is_evex_instruction();
3478 int dst_enc = kdst->encoding();
3479 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3480 emit_int8(0x3E);
3481 emit_operand(as_Register(dst_enc), src);
3482 emit_int8(vcc);
3483 }
3484
3485 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3486 assert(VM_Version::supports_avx512bw(), "");
3487 InstructionMark im(this);
3488 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3489 attributes.set_is_evex_instruction();
3490 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3491 int dst_enc = kdst->encoding();
3492 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3493 emit_int8(0x74);
3494 emit_operand(as_Register(dst_enc), src);
3495 }
3496
3497 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3498 assert(VM_Version::supports_avx512vlbw(), "");
3499 InstructionMark im(this);
3500 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ true);
3501 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3502 attributes.reset_is_clear_context();
3503 attributes.set_embedded_opmask_register_specifier(mask);
3504 attributes.set_is_evex_instruction();
3505 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3506 emit_int8(0x74);
3507 emit_operand(as_Register(kdst->encoding()), src);
3508 }
3509
3510 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3511 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3512 assert(VM_Version::supports_sse2(), "");
3513 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3514 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3515 emit_int16(0x75, (0xC0 | encode));
3516 }
3517
3518 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3519 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3520 assert(VM_Version::supports_avx(), "");
3521 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3522 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3523 emit_int16(0x75, (0xC0 | encode));
3524 }
3525
3526 // In this context, kdst is written the mask used to process the equal components
3527 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3528 assert(VM_Version::supports_avx512bw(), "");
3529 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3530 attributes.set_is_evex_instruction();
3531 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3532 emit_int16(0x75, (0xC0 | encode));
3533 }
3534
3535 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3536 assert(VM_Version::supports_avx512bw(), "");
3537 InstructionMark im(this);
3538 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3539 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3540 attributes.set_is_evex_instruction();
3541 int dst_enc = kdst->encoding();
3542 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3543 emit_int8(0x75);
3544 emit_operand(as_Register(dst_enc), src);
3545 }
3546
3547 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3548 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3549 assert(VM_Version::supports_sse2(), "");
3550 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3551 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3552 emit_int16(0x76, (0xC0 | encode));
3553 }
3554
3555 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3556 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3557 assert(VM_Version::supports_avx(), "");
3558 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3559 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3560 emit_int16(0x76, (0xC0 | encode));
3561 }
3562
3563 // In this context, kdst is written the mask used to process the equal components
3564 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3565 assert(VM_Version::supports_evex(), "");
3566 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3567 attributes.set_is_evex_instruction();
3568 attributes.reset_is_clear_context();
3569 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3570 emit_int16(0x76, (0xC0 | encode));
3571 }
3572
3573 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3574 assert(VM_Version::supports_evex(), "");
3575 InstructionMark im(this);
3576 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3577 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3578 attributes.reset_is_clear_context();
3579 attributes.set_is_evex_instruction();
3580 int dst_enc = kdst->encoding();
3581 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3582 emit_int8(0x76);
3583 emit_operand(as_Register(dst_enc), src);
3584 }
3585
3586 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3587 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3588 assert(VM_Version::supports_sse4_1(), "");
3589 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3590 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3591 emit_int16(0x29, (0xC0 | encode));
3592 }
3593
3594 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3595 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3596 assert(VM_Version::supports_avx(), "");
3597 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3598 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3599 emit_int16(0x29, (0xC0 | encode));
3600 }
3601
3602 // In this context, kdst is written the mask used to process the equal components
3603 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3604 assert(VM_Version::supports_evex(), "");
3605 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3606 attributes.reset_is_clear_context();
3607 attributes.set_is_evex_instruction();
3608 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3609 emit_int16(0x29, (0xC0 | encode));
3610 }
3611
3612 // In this context, kdst is written the mask used to process the equal components
3613 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3614 assert(VM_Version::supports_evex(), "");
3615 InstructionMark im(this);
3616 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3617 attributes.reset_is_clear_context();
3618 attributes.set_is_evex_instruction();
3619 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3620 int dst_enc = kdst->encoding();
3621 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3622 emit_int8(0x29);
3623 emit_operand(as_Register(dst_enc), src);
3624 }
3625
3626 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3627 assert(VM_Version::supports_sse2(), "");
3628 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3629 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3630 emit_int16((unsigned char)0xD7, (0xC0 | encode));
3631 }
3632
3633 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3634 assert(VM_Version::supports_avx2(), "");
3635 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
3636 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3637 emit_int16((unsigned char)0xD7, (0xC0 | encode));
3638 }
3639
3640 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3641 assert(VM_Version::supports_sse4_1(), "");
3642 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3643 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3644 emit_int24(0x16, (0xC0 | encode), imm8);
3645 }
3646
3647 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3648 assert(VM_Version::supports_sse4_1(), "");
3649 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3650 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3651 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3652 emit_int8(0x16);
3653 emit_operand(src, dst);
3654 emit_int8(imm8);
3655 }
3656
3657 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3658 assert(VM_Version::supports_sse4_1(), "");
3659 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3660 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3661 emit_int24(0x16, (0xC0 | encode), imm8);
3662 }
3663
3664 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3665 assert(VM_Version::supports_sse4_1(), "");
3666 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3667 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3668 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3669 emit_int8(0x16);
3670 emit_operand(src, dst);
3671 emit_int8(imm8);
3672 }
3673
3674 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3675 assert(VM_Version::supports_sse2(), "");
3676 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3677 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3678 emit_int24((unsigned char)0xC5, (0xC0 | encode), imm8);
3679 }
3680
3681 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3682 assert(VM_Version::supports_sse4_1(), "");
3683 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3684 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3685 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3686 emit_int8(0x15);
3687 emit_operand(src, dst);
3688 emit_int8(imm8);
3689 }
3690
3691 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3692 assert(VM_Version::supports_sse4_1(), "");
3693 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3694 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3695 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3696 emit_int8(0x14);
3697 emit_operand(src, dst);
3698 emit_int8(imm8);
3699 }
3700
3701 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3702 assert(VM_Version::supports_sse4_1(), "");
3703 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3704 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3705 emit_int24(0x22, (0xC0 | encode), imm8);
3706 }
3707
3708 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3709 assert(VM_Version::supports_sse4_1(), "");
3710 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3711 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3712 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3713 emit_int8(0x22);
3714 emit_operand(dst,src);
3715 emit_int8(imm8);
3716 }
3717
3718 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3719 assert(VM_Version::supports_sse4_1(), "");
3720 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3721 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3722 emit_int24(0x22, (0xC0 | encode), imm8);
3723 }
3724
3725 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3726 assert(VM_Version::supports_sse4_1(), "");
3727 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
3728 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3729 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3730 emit_int8(0x22);
3731 emit_operand(dst, src);
3732 emit_int8(imm8);
3733 }
3734
3735 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3736 assert(VM_Version::supports_sse2(), "");
3737 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3738 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3739 emit_int24((unsigned char)0xC4, (0xC0 | encode), imm8);
3740 }
3741
3742 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3743 assert(VM_Version::supports_sse2(), "");
3744 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3745 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3746 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3747 emit_int8((unsigned char)0xC4);
3748 emit_operand(dst, src);
3749 emit_int8(imm8);
3750 }
3751
3752 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3753 assert(VM_Version::supports_sse4_1(), "");
3754 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3755 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3756 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3757 emit_int8(0x20);
3758 emit_operand(dst, src);
3759 emit_int8(imm8);
3760 }
3761
3762 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3763 assert(VM_Version::supports_sse4_1(), "");
3764 InstructionMark im(this);
3765 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3766 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3767 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3768 emit_int8(0x30);
3769 emit_operand(dst, src);
3770 }
3771
3772 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3773 assert(VM_Version::supports_sse4_1(), "");
3774 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3775 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3776 emit_int16(0x30, (0xC0 | encode));
3777 }
3778
3779 void Assembler::pmovsxbw(XMMRegister dst, XMMRegister src) {
3780 assert(VM_Version::supports_sse4_1(), "");
3781 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3782 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3783 emit_int16(0x20, (0xC0 | encode));
3784 }
3785
3786 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3787 assert(VM_Version::supports_avx(), "");
3788 InstructionMark im(this);
3789 assert(dst != xnoreg, "sanity");
3790 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3791 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3792 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3793 emit_int8(0x30);
3794 emit_operand(dst, src);
3795 }
3796
3797 void Assembler::vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3798 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3799 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3800 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3801 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3802 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3803 emit_int16(0x30, (unsigned char) (0xC0 | encode));
3804 }
3805
3806 void Assembler::vpmovsxbw(XMMRegister dst, XMMRegister src, int vector_len) {
3807 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3808 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3809 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
3810 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3811 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3812 emit_int16(0x20, (0xC0 | encode));
3813 }
3814
3815 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3816 assert(VM_Version::supports_avx512vlbw(), "");
3817 assert(dst != xnoreg, "sanity");
3818 InstructionMark im(this);
3819 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3820 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3821 attributes.set_embedded_opmask_register_specifier(mask);
3822 attributes.set_is_evex_instruction();
3823 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3824 emit_int8(0x30);
3825 emit_operand(dst, src);
3826 }
3827 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3828 assert(VM_Version::supports_avx512vlbw(), "");
3829 assert(src != xnoreg, "sanity");
3830 InstructionMark im(this);
3831 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3832 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3833 attributes.set_is_evex_instruction();
3834 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3835 emit_int8(0x30);
3836 emit_operand(src, dst);
3837 }
3838
3839 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3840 assert(VM_Version::supports_avx512vlbw(), "");
3841 assert(src != xnoreg, "sanity");
3842 InstructionMark im(this);
3843 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3844 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3845 attributes.reset_is_clear_context();
3846 attributes.set_embedded_opmask_register_specifier(mask);
3847 attributes.set_is_evex_instruction();
3848 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3849 emit_int8(0x30);
3850 emit_operand(src, dst);
3851 }
3852
3853 void Assembler::evpmovdb(Address dst, XMMRegister src, int vector_len) {
3854 assert(VM_Version::supports_evex(), "");
3855 assert(src != xnoreg, "sanity");
3856 InstructionMark im(this);
3857 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3858 attributes.set_address_attributes(/* tuple_type */ EVEX_QVM, /* input_size_in_bits */ EVEX_NObit);
3859 attributes.set_is_evex_instruction();
3860 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3861 emit_int8(0x31);
3862 emit_operand(src, dst);
3863 }
3864
3865 void Assembler::vpmovzxwd(XMMRegister dst, XMMRegister src, int vector_len) {
3866 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
3867 vector_len == AVX_256bit? VM_Version::supports_avx2() :
3868 vector_len == AVX_512bit? VM_Version::supports_evex() : 0, " ");
3869 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3870 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3871 emit_int16(0x33, (0xC0 | encode));
3872 }
3873
3874 void Assembler::pmaddwd(XMMRegister dst, XMMRegister src) {
3875 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3876 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3877 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3878 emit_int16((unsigned char)0xF5, (0xC0 | encode));
3879 }
3880
3881 void Assembler::vpmaddwd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3882 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
3883 (vector_len == AVX_256bit ? VM_Version::supports_avx2() :
3884 (vector_len == AVX_512bit ? VM_Version::supports_evex() : 0)), "");
3885 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3886 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3887 emit_int16((unsigned char)0xF5, (0xC0 | encode));
3888 }
3889
3890 void Assembler::evpdpwssd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3891 assert(VM_Version::supports_evex(), "");
3892 assert(VM_Version::supports_avx512_vnni(), "must support vnni");
3893 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3894 attributes.set_is_evex_instruction();
3895 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3896 emit_int16(0x52, (0xC0 | encode));
3897 }
3898
3899 // generic
3900 void Assembler::pop(Register dst) {
3901 int encode = prefix_and_encode(dst->encoding());
3902 emit_int8(0x58 | encode);
3903 }
3904
3905 void Assembler::popcntl(Register dst, Address src) {
3906 assert(VM_Version::supports_popcnt(), "must support");
3907 InstructionMark im(this);
3908 emit_int8((unsigned char)0xF3);
3909 prefix(src, dst);
3910 emit_int16(0x0F, (unsigned char)0xB8);
3911 emit_operand(dst, src);
3912 }
3913
3914 void Assembler::popcntl(Register dst, Register src) {
3915 assert(VM_Version::supports_popcnt(), "must support");
3916 emit_int8((unsigned char)0xF3);
3917 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3918 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
3919 }
3920
3921 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
3922 assert(VM_Version::supports_avx512_vpopcntdq(), "must support vpopcntdq feature");
3923 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3924 attributes.set_is_evex_instruction();
3925 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3926 emit_int16(0x55, (0xC0 | encode));
3927 }
3928
3929 void Assembler::popf() {
3930 emit_int8((unsigned char)0x9D);
3931 }
3932
3933 #ifndef _LP64 // no 32bit push/pop on amd64
3934 void Assembler::popl(Address dst) {
3935 // NOTE: this will adjust stack by 8byte on 64bits
3936 InstructionMark im(this);
3937 prefix(dst);
3938 emit_int8((unsigned char)0x8F);
3939 emit_operand(rax, dst);
3940 }
3941 #endif
3942
3943 void Assembler::prefetchnta(Address src) {
3944 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3945 InstructionMark im(this);
3946 prefix(src);
3947 emit_int16(0x0F, 0x18);
3948 emit_operand(rax, src); // 0, src
3949 }
3950
3951 void Assembler::prefetchr(Address src) {
3952 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3953 InstructionMark im(this);
3954 prefix(src);
3955 emit_int16(0x0F, 0x0D);
3956 emit_operand(rax, src); // 0, src
3957 }
3958
3959 void Assembler::prefetcht0(Address src) {
3960 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3961 InstructionMark im(this);
3962 prefix(src);
3963 emit_int16(0x0F, 0x18);
3964 emit_operand(rcx, src); // 1, src
3965 }
3966
3967 void Assembler::prefetcht1(Address src) {
3968 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3969 InstructionMark im(this);
3970 prefix(src);
3971 emit_int16(0x0F, 0x18);
3972 emit_operand(rdx, src); // 2, src
3973 }
3974
3975 void Assembler::prefetcht2(Address src) {
3976 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3977 InstructionMark im(this);
3978 prefix(src);
3979 emit_int16(0x0F, 0x18);
3980 emit_operand(rbx, src); // 3, src
3981 }
3982
3983 void Assembler::prefetchw(Address src) {
3984 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3985 InstructionMark im(this);
3986 prefix(src);
3987 emit_int16(0x0F, 0x0D);
3988 emit_operand(rcx, src); // 1, src
3989 }
3990
3991 void Assembler::prefix(Prefix p) {
3992 emit_int8(p);
3993 }
3994
3995 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3996 assert(VM_Version::supports_ssse3(), "");
3997 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3998 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3999 emit_int16(0x00, (0xC0 | encode));
4000 }
4001
4002 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4003 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4004 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4005 vector_len == AVX_512bit? VM_Version::supports_avx512bw() : 0, "");
4006 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4007 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4008 emit_int16(0x00, (0xC0 | encode));
4009 }
4010
4011 void Assembler::pshufb(XMMRegister dst, Address src) {
4012 assert(VM_Version::supports_ssse3(), "");
4013 InstructionMark im(this);
4014 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4015 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4016 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4017 emit_int8(0x00);
4018 emit_operand(dst, src);
4019 }
4020
4021 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4022 assert(isByte(mode), "invalid value");
4023 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4024 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4025 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4026 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4027 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
4028 }
4029
4030 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4031 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4032 (vector_len == AVX_256bit? VM_Version::supports_avx2() :
4033 (vector_len == AVX_512bit? VM_Version::supports_evex() : 0)), "");
4034 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4035 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4036 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4037 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
4038 }
4039
4040 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4041 assert(isByte(mode), "invalid value");
4042 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4043 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4044 InstructionMark im(this);
4045 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4046 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4047 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4048 emit_int8(0x70);
4049 emit_operand(dst, src);
4050 emit_int8(mode & 0xFF);
4051 }
4052
4053 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4054 assert(isByte(mode), "invalid value");
4055 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4056 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4057 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4058 emit_int24(0x70, (0xC0 | encode), mode & 0xFF);
4059 }
4060
4061 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4062 assert(isByte(mode), "invalid value");
4063 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4064 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4065 InstructionMark im(this);
4066 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4067 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4068 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4069 emit_int8(0x70);
4070 emit_operand(dst, src);
4071 emit_int8(mode & 0xFF);
4072 }
4073
4074 void Assembler::evshufi64x2(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4075 assert(VM_Version::supports_evex(), "requires EVEX support");
4076 assert(vector_len == Assembler::AVX_256bit || vector_len == Assembler::AVX_512bit, "");
4077 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4078 attributes.set_is_evex_instruction();
4079 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4080 emit_int24(0x43, (0xC0 | encode), imm8 & 0xFF);
4081 }
4082
4083 void Assembler::psrldq(XMMRegister dst, int shift) {
4084 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4085 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4086 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4087 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4088 emit_int24(0x73, (0xC0 | encode), shift);
4089 }
4090
4091 void Assembler::vpsrldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4092 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4093 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4094 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
4095 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4096 int encode = vex_prefix_and_encode(xmm3->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4097 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
4098 }
4099
4100 void Assembler::pslldq(XMMRegister dst, int shift) {
4101 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4102 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4103 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4104 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4105 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4106 emit_int24(0x73, (0xC0 | encode), shift);
4107 }
4108
4109 void Assembler::vpslldq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4110 assert(vector_len == AVX_128bit ? VM_Version::supports_avx() :
4111 vector_len == AVX_256bit ? VM_Version::supports_avx2() :
4112 vector_len == AVX_512bit ? VM_Version::supports_avx512bw() : 0, "");
4113 InstructionAttr attributes(vector_len, /*vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4114 int encode = vex_prefix_and_encode(xmm7->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4115 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
4116 }
4117
4118 void Assembler::ptest(XMMRegister dst, Address src) {
4119 assert(VM_Version::supports_sse4_1(), "");
4120 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4121 InstructionMark im(this);
4122 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4123 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4124 emit_int8(0x17);
4125 emit_operand(dst, src);
4126 }
4127
4128 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4129 assert(VM_Version::supports_sse4_1() || VM_Version::supports_avx(), "");
4130 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4131 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4132 emit_int8(0x17);
4133 emit_int8((0xC0 | encode));
4134 }
4135
4136 void Assembler::vptest(XMMRegister dst, Address src) {
4137 assert(VM_Version::supports_avx(), "");
4138 InstructionMark im(this);
4139 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4140 assert(dst != xnoreg, "sanity");
4141 // swap src<->dst for encoding
4142 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4143 emit_int8(0x17);
4144 emit_operand(dst, src);
4145 }
4146
4147 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4148 assert(VM_Version::supports_avx(), "");
4149 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4150 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4151 emit_int16(0x17, (0xC0 | encode));
4152 }
4153
4154 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4155 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4156 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4157 InstructionMark im(this);
4158 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
4159 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4160 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4161 emit_int8(0x60);
4162 emit_operand(dst, src);
4163 }
4164
4165 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4166 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4167 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ true, /* uses_vl */ true);
4168 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4169 emit_int16(0x60, (0xC0 | encode));
4170 }
4171
4172 void Assembler::punpckldq(XMMRegister dst, Address src) {
4173 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4174 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4175 InstructionMark im(this);
4176 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4177 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4178 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4179 emit_int8(0x62);
4180 emit_operand(dst, src);
4181 }
4182
4183 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4184 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4185 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4186 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4187 emit_int16(0x62, (0xC0 | encode));
4188 }
4189
4190 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4191 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4192 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4193 attributes.set_rex_vex_w_reverted();
4194 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4195 emit_int16(0x6C, (0xC0 | encode));
4196 }
4197
4198 void Assembler::push(int32_t imm32) {
4199 // in 64bits we push 64bits onto the stack but only
4200 // take a 32bit immediate
4201 emit_int8(0x68);
4202 emit_int32(imm32);
4203 }
4204
4205 void Assembler::push(Register src) {
4206 int encode = prefix_and_encode(src->encoding());
4207 emit_int8(0x50 | encode);
4208 }
4209
4210 void Assembler::pushf() {
4211 emit_int8((unsigned char)0x9C);
4212 }
4213
4214 #ifndef _LP64 // no 32bit push/pop on amd64
4215 void Assembler::pushl(Address src) {
4216 // Note this will push 64bit on 64bit
4217 InstructionMark im(this);
4218 prefix(src);
4219 emit_int8((unsigned char)0xFF);
4220 emit_operand(rsi, src);
4221 }
4222 #endif
4223
4224 void Assembler::rcll(Register dst, int imm8) {
4225 assert(isShiftCount(imm8), "illegal shift count");
4226 int encode = prefix_and_encode(dst->encoding());
4227 if (imm8 == 1) {
4228 emit_int16((unsigned char)0xD1, (0xD0 | encode));
4229 } else {
4230 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
4231 }
4232 }
4233
4234 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4235 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4236 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4237 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4238 emit_int16(0x53, (0xC0 | encode));
4239 }
4240
4241 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4242 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4243 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4244 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4245 emit_int16(0x53, (0xC0 | encode));
4246 }
4247
4248 void Assembler::rdtsc() {
4249 emit_int16(0x0F, 0x31);
4250 }
4251
4252 // copies data from [esi] to [edi] using rcx pointer sized words
4253 // generic
4254 void Assembler::rep_mov() {
4255 // REP
4256 // MOVSQ
4257 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xA5);)
4258 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xA5);)
4259 }
4260
4261 // sets rcx bytes with rax, value at [edi]
4262 void Assembler::rep_stosb() {
4263 // REP
4264 // STOSB
4265 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAA);)
4266 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAA);)
4267 }
4268
4269 // sets rcx pointer sized words with rax, value at [edi]
4270 // generic
4271 void Assembler::rep_stos() {
4272 // REP
4273 // LP64:STOSQ, LP32:STOSD
4274 LP64_ONLY(emit_int24((unsigned char)0xF3, REX_W, (unsigned char)0xAB);)
4275 NOT_LP64( emit_int16((unsigned char)0xF3, (unsigned char)0xAB);)
4276 }
4277
4278 // scans rcx pointer sized words at [edi] for occurance of rax,
4279 // generic
4280 void Assembler::repne_scan() { // repne_scan
4281 // SCASQ
4282 LP64_ONLY(emit_int24((unsigned char)0xF2, REX_W, (unsigned char)0xAF);)
4283 NOT_LP64( emit_int16((unsigned char)0xF2, (unsigned char)0xAF);)
4284 }
4285
4286 #ifdef _LP64
4287 // scans rcx 4 byte words at [edi] for occurance of rax,
4288 // generic
4289 void Assembler::repne_scanl() { // repne_scan
4290 // SCASL
4291 emit_int16((unsigned char)0xF2, (unsigned char)0xAF);
4292 }
4293 #endif
4294
4295 void Assembler::ret(int imm16) {
4296 if (imm16 == 0) {
4297 emit_int8((unsigned char)0xC3);
4298 } else {
4299 emit_int8((unsigned char)0xC2);
4300 emit_int16(imm16);
4301 }
4302 }
4303
4304 void Assembler::roll(Register dst, int imm8) {
4305 assert(isShiftCount(imm8 >> 1), "illegal shift count");
4306 int encode = prefix_and_encode(dst->encoding());
4307 if (imm8 == 1) {
4308 emit_int16((unsigned char)0xD1, (0xC0 | encode));
4309 } else {
4310 emit_int24((unsigned char)0xC1, (0xc0 | encode), imm8);
4311 }
4312 }
4313
4314 void Assembler::sahf() {
4315 #ifdef _LP64
4316 // Not supported in 64bit mode
4317 ShouldNotReachHere();
4318 #endif
4319 emit_int8((unsigned char)0x9E);
4320 }
4321
4322 void Assembler::sarl(Register dst, int imm8) {
4323 int encode = prefix_and_encode(dst->encoding());
4324 assert(isShiftCount(imm8), "illegal shift count");
4325 if (imm8 == 1) {
4326 emit_int16((unsigned char)0xD1, (0xF8 | encode));
4327 } else {
4328 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
4329 }
4330 }
4331
4332 void Assembler::sarl(Register dst) {
4333 int encode = prefix_and_encode(dst->encoding());
4334 emit_int16((unsigned char)0xD3, (0xF8 | encode));
4335 }
4336
4337 void Assembler::sbbl(Address dst, int32_t imm32) {
4338 InstructionMark im(this);
4339 prefix(dst);
4340 emit_arith_operand(0x81, rbx, dst, imm32);
4341 }
4342
4343 void Assembler::sbbl(Register dst, int32_t imm32) {
4344 prefix(dst);
4345 emit_arith(0x81, 0xD8, dst, imm32);
4346 }
4347
4348
4349 void Assembler::sbbl(Register dst, Address src) {
4350 InstructionMark im(this);
4351 prefix(src, dst);
4352 emit_int8(0x1B);
4353 emit_operand(dst, src);
4354 }
4355
4356 void Assembler::sbbl(Register dst, Register src) {
4357 (void) prefix_and_encode(dst->encoding(), src->encoding());
4358 emit_arith(0x1B, 0xC0, dst, src);
4359 }
4360
4361 void Assembler::setb(Condition cc, Register dst) {
4362 assert(0 <= cc && cc < 16, "illegal cc");
4363 int encode = prefix_and_encode(dst->encoding(), true);
4364 emit_int24(0x0F, (unsigned char)0x90 | cc, (0xC0 | encode));
4365 }
4366
4367 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4368 assert(VM_Version::supports_ssse3(), "");
4369 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4370 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4371 emit_int24(0x0F, (0xC0 | encode), imm8);
4372 }
4373
4374 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4375 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4376 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4377 0, "");
4378 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4379 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4380 emit_int24(0x0F, (0xC0 | encode), imm8);
4381 }
4382
4383 void Assembler::evalignq(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
4384 assert(VM_Version::supports_evex(), "");
4385 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4386 attributes.set_is_evex_instruction();
4387 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4388 emit_int24(0x3, (0xC0 | encode), imm8);
4389 }
4390
4391 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4392 assert(VM_Version::supports_sse4_1(), "");
4393 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4394 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4395 emit_int24(0x0E, (0xC0 | encode), imm8);
4396 }
4397
4398 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4399 assert(VM_Version::supports_sha(), "");
4400 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4401 emit_int24((unsigned char)0xCC, (0xC0 | encode), (unsigned char)imm8);
4402 }
4403
4404 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4405 assert(VM_Version::supports_sha(), "");
4406 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4407 emit_int16((unsigned char)0xC8, (0xC0 | encode));
4408 }
4409
4410 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4411 assert(VM_Version::supports_sha(), "");
4412 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4413 emit_int16((unsigned char)0xC9, (0xC0 | encode));
4414 }
4415
4416 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4417 assert(VM_Version::supports_sha(), "");
4418 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4419 emit_int16((unsigned char)0xCA, (0xC0 | encode));
4420 }
4421
4422 // xmm0 is implicit additional source to this instruction.
4423 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4424 assert(VM_Version::supports_sha(), "");
4425 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4426 emit_int16((unsigned char)0xCB, (0xC0 | encode));
4427 }
4428
4429 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4430 assert(VM_Version::supports_sha(), "");
4431 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4432 emit_int16((unsigned char)0xCC, (0xC0 | encode));
4433 }
4434
4435 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4436 assert(VM_Version::supports_sha(), "");
4437 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4438 emit_int16((unsigned char)0xCD, (0xC0 | encode));
4439 }
4440
4441
4442 void Assembler::shll(Register dst, int imm8) {
4443 assert(isShiftCount(imm8), "illegal shift count");
4444 int encode = prefix_and_encode(dst->encoding());
4445 if (imm8 == 1 ) {
4446 emit_int16((unsigned char)0xD1, (0xE0 | encode));
4447 } else {
4448 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
4449 }
4450 }
4451
4452 void Assembler::shll(Register dst) {
4453 int encode = prefix_and_encode(dst->encoding());
4454 emit_int16((unsigned char)0xD3, (0xE0 | encode));
4455 }
4456
4457 void Assembler::shrl(Register dst, int imm8) {
4458 assert(isShiftCount(imm8), "illegal shift count");
4459 int encode = prefix_and_encode(dst->encoding());
4460 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
4461 }
4462
4463 void Assembler::shrl(Register dst) {
4464 int encode = prefix_and_encode(dst->encoding());
4465 emit_int16((unsigned char)0xD3, (0xE8 | encode));
4466 }
4467
4468 void Assembler::shldl(Register dst, Register src) {
4469 int encode = prefix_and_encode(src->encoding(), dst->encoding());
4470 emit_int24(0x0F, (unsigned char)0xA5, (0xC0 | encode));
4471 }
4472
4473 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
4474 int encode = prefix_and_encode(src->encoding(), dst->encoding());
4475 emit_int32(0x0F, (unsigned char)0xA4, (0xC0 | encode), imm8);
4476 }
4477
4478 void Assembler::shrdl(Register dst, Register src) {
4479 int encode = prefix_and_encode(src->encoding(), dst->encoding());
4480 emit_int24(0x0F, (unsigned char)0xAD, (0xC0 | encode));
4481 }
4482
4483 void Assembler::shrdl(Register dst, Register src, int8_t imm8) {
4484 int encode = prefix_and_encode(src->encoding(), dst->encoding());
4485 emit_int32(0x0F, (unsigned char)0xAC, (0xC0 | encode), imm8);
4486 }
4487
4488 // copies a single word from [esi] to [edi]
4489 void Assembler::smovl() {
4490 emit_int8((unsigned char)0xA5);
4491 }
4492
4493 void Assembler::roundsd(XMMRegister dst, XMMRegister src, int32_t rmode) {
4494 assert(VM_Version::supports_sse4_1(), "");
4495 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4496 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4497 emit_int24(0x0B, (0xC0 | encode), (unsigned char)rmode);
4498 }
4499
4500 void Assembler::roundsd(XMMRegister dst, Address src, int32_t rmode) {
4501 assert(VM_Version::supports_sse4_1(), "");
4502 InstructionMark im(this);
4503 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4504 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4505 emit_int8(0x0B);
4506 emit_operand(dst, src);
4507 emit_int8((unsigned char)rmode);
4508 }
4509
4510 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4511 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4512 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4513 attributes.set_rex_vex_w_reverted();
4514 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4515 emit_int16(0x51, (0xC0 | encode));
4516 }
4517
4518 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4519 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4520 InstructionMark im(this);
4521 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4522 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4523 attributes.set_rex_vex_w_reverted();
4524 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4525 emit_int8(0x51);
4526 emit_operand(dst, src);
4527 }
4528
4529 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4530 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4531 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4532 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4533 emit_int16(0x51, (0xC0 | encode));
4534 }
4535
4536 void Assembler::std() {
4537 emit_int8((unsigned char)0xFD);
4538 }
4539
4540 void Assembler::sqrtss(XMMRegister dst, Address src) {
4541 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4542 InstructionMark im(this);
4543 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4544 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4545 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4546 emit_int8(0x51);
4547 emit_operand(dst, src);
4548 }
4549
4550 void Assembler::stmxcsr( Address dst) {
4551 if (UseAVX > 0 ) {
4552 assert(VM_Version::supports_avx(), "");
4553 InstructionMark im(this);
4554 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4555 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4556 emit_int8((unsigned char)0xAE);
4557 emit_operand(as_Register(3), dst);
4558 } else {
4559 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4560 InstructionMark im(this);
4561 prefix(dst);
4562 emit_int16(0x0F, (unsigned char)0xAE);
4563 emit_operand(as_Register(3), dst);
4564 }
4565 }
4566
4567 void Assembler::subl(Address dst, int32_t imm32) {
4568 InstructionMark im(this);
4569 prefix(dst);
4570 emit_arith_operand(0x81, rbp, dst, imm32);
4571 }
4572
4573 void Assembler::subl(Address dst, Register src) {
4574 InstructionMark im(this);
4575 prefix(dst, src);
4576 emit_int8(0x29);
4577 emit_operand(src, dst);
4578 }
4579
4580 void Assembler::subl(Register dst, int32_t imm32) {
4581 prefix(dst);
4582 emit_arith(0x81, 0xE8, dst, imm32);
4583 }
4584
4585 // Force generation of a 4 byte immediate value even if it fits into 8bit
4586 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4587 prefix(dst);
4588 emit_arith_imm32(0x81, 0xE8, dst, imm32);
4589 }
4590
4591 void Assembler::subl(Register dst, Address src) {
4592 InstructionMark im(this);
4593 prefix(src, dst);
4594 emit_int8(0x2B);
4595 emit_operand(dst, src);
4596 }
4597
4598 void Assembler::subl(Register dst, Register src) {
4599 (void) prefix_and_encode(dst->encoding(), src->encoding());
4600 emit_arith(0x2B, 0xC0, dst, src);
4601 }
4602
4603 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4604 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4605 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4606 attributes.set_rex_vex_w_reverted();
4607 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4608 emit_int16(0x5C, (0xC0 | encode));
4609 }
4610
4611 void Assembler::subsd(XMMRegister dst, Address src) {
4612 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4613 InstructionMark im(this);
4614 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4615 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4616 attributes.set_rex_vex_w_reverted();
4617 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4618 emit_int8(0x5C);
4619 emit_operand(dst, src);
4620 }
4621
4622 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4623 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4624 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
4625 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4626 emit_int16(0x5C, (0xC0 | encode));
4627 }
4628
4629 void Assembler::subss(XMMRegister dst, Address src) {
4630 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4631 InstructionMark im(this);
4632 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4633 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4634 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4635 emit_int8(0x5C);
4636 emit_operand(dst, src);
4637 }
4638
4639 void Assembler::testb(Register dst, int imm8) {
4640 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4641 (void) prefix_and_encode(dst->encoding(), true);
4642 emit_arith_b(0xF6, 0xC0, dst, imm8);
4643 }
4644
4645 void Assembler::testb(Address dst, int imm8) {
4646 InstructionMark im(this);
4647 prefix(dst);
4648 emit_int8((unsigned char)0xF6);
4649 emit_operand(rax, dst, 1);
4650 emit_int8(imm8);
4651 }
4652
4653 void Assembler::testl(Register dst, int32_t imm32) {
4654 // not using emit_arith because test
4655 // doesn't support sign-extension of
4656 // 8bit operands
4657 int encode = dst->encoding();
4658 if (encode == 0) {
4659 emit_int8((unsigned char)0xA9);
4660 } else {
4661 encode = prefix_and_encode(encode);
4662 emit_int16((unsigned char)0xF7, (0xC0 | encode));
4663 }
4664 emit_int32(imm32);
4665 }
4666
4667 void Assembler::testl(Register dst, Register src) {
4668 (void) prefix_and_encode(dst->encoding(), src->encoding());
4669 emit_arith(0x85, 0xC0, dst, src);
4670 }
4671
4672 void Assembler::testl(Register dst, Address src) {
4673 InstructionMark im(this);
4674 prefix(src, dst);
4675 emit_int8((unsigned char)0x85);
4676 emit_operand(dst, src);
4677 }
4678
4679 void Assembler::tzcntl(Register dst, Register src) {
4680 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4681 emit_int8((unsigned char)0xF3);
4682 int encode = prefix_and_encode(dst->encoding(), src->encoding());
4683 emit_int24(0x0F,
4684 (unsigned char)0xBC,
4685 0xC0 | encode);
4686 }
4687
4688 void Assembler::tzcntq(Register dst, Register src) {
4689 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4690 emit_int8((unsigned char)0xF3);
4691 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4692 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
4693 }
4694
4695 void Assembler::ucomisd(XMMRegister dst, Address src) {
4696 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4697 InstructionMark im(this);
4698 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4699 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4700 attributes.set_rex_vex_w_reverted();
4701 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4702 emit_int8(0x2E);
4703 emit_operand(dst, src);
4704 }
4705
4706 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4707 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4708 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4709 attributes.set_rex_vex_w_reverted();
4710 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4711 emit_int16(0x2E, (0xC0 | encode));
4712 }
4713
4714 void Assembler::ucomiss(XMMRegister dst, Address src) {
4715 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4716 InstructionMark im(this);
4717 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4718 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4719 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4720 emit_int8(0x2E);
4721 emit_operand(dst, src);
4722 }
4723
4724 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4725 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4726 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4727 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4728 emit_int16(0x2E, (0xC0 | encode));
4729 }
4730
4731 void Assembler::xabort(int8_t imm8) {
4732 emit_int24((unsigned char)0xC6, (unsigned char)0xF8, (imm8 & 0xFF));
4733 }
4734
4735 void Assembler::xaddb(Address dst, Register src) {
4736 InstructionMark im(this);
4737 prefix(dst, src, true);
4738 emit_int16(0x0F, (unsigned char)0xC0);
4739 emit_operand(src, dst);
4740 }
4741
4742 void Assembler::xaddw(Address dst, Register src) {
4743 InstructionMark im(this);
4744 emit_int8(0x66);
4745 prefix(dst, src);
4746 emit_int16(0x0F, (unsigned char)0xC1);
4747 emit_operand(src, dst);
4748 }
4749
4750 void Assembler::xaddl(Address dst, Register src) {
4751 InstructionMark im(this);
4752 prefix(dst, src);
4753 emit_int16(0x0F, (unsigned char)0xC1);
4754 emit_operand(src, dst);
4755 }
4756
4757 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4758 InstructionMark im(this);
4759 relocate(rtype);
4760 if (abort.is_bound()) {
4761 address entry = target(abort);
4762 assert(entry != NULL, "abort entry NULL");
4763 intptr_t offset = entry - pc();
4764 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
4765 emit_int32(offset - 6); // 2 opcode + 4 address
4766 } else {
4767 abort.add_patch_at(code(), locator());
4768 emit_int16((unsigned char)0xC7, (unsigned char)0xF8);
4769 emit_int32(0);
4770 }
4771 }
4772
4773 void Assembler::xchgb(Register dst, Address src) { // xchg
4774 InstructionMark im(this);
4775 prefix(src, dst, true);
4776 emit_int8((unsigned char)0x86);
4777 emit_operand(dst, src);
4778 }
4779
4780 void Assembler::xchgw(Register dst, Address src) { // xchg
4781 InstructionMark im(this);
4782 emit_int8(0x66);
4783 prefix(src, dst);
4784 emit_int8((unsigned char)0x87);
4785 emit_operand(dst, src);
4786 }
4787
4788 void Assembler::xchgl(Register dst, Address src) { // xchg
4789 InstructionMark im(this);
4790 prefix(src, dst);
4791 emit_int8((unsigned char)0x87);
4792 emit_operand(dst, src);
4793 }
4794
4795 void Assembler::xchgl(Register dst, Register src) {
4796 int encode = prefix_and_encode(dst->encoding(), src->encoding());
4797 emit_int16((unsigned char)0x87, (0xC0 | encode));
4798 }
4799
4800 void Assembler::xend() {
4801 emit_int24(0x0F, 0x01, (unsigned char)0xD5);
4802 }
4803
4804 void Assembler::xgetbv() {
4805 emit_int24(0x0F, 0x01, (unsigned char)0xD0);
4806 }
4807
4808 void Assembler::xorl(Register dst, int32_t imm32) {
4809 prefix(dst);
4810 emit_arith(0x81, 0xF0, dst, imm32);
4811 }
4812
4813 void Assembler::xorl(Register dst, Address src) {
4814 InstructionMark im(this);
4815 prefix(src, dst);
4816 emit_int8(0x33);
4817 emit_operand(dst, src);
4818 }
4819
4820 void Assembler::xorl(Register dst, Register src) {
4821 (void) prefix_and_encode(dst->encoding(), src->encoding());
4822 emit_arith(0x33, 0xC0, dst, src);
4823 }
4824
4825 void Assembler::xorb(Register dst, Address src) {
4826 InstructionMark im(this);
4827 prefix(src, dst);
4828 emit_int8(0x32);
4829 emit_operand(dst, src);
4830 }
4831
4832 // AVX 3-operands scalar float-point arithmetic instructions
4833
4834 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4835 assert(VM_Version::supports_avx(), "");
4836 InstructionMark im(this);
4837 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4838 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4839 attributes.set_rex_vex_w_reverted();
4840 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4841 emit_int8(0x58);
4842 emit_operand(dst, src);
4843 }
4844
4845 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4846 assert(VM_Version::supports_avx(), "");
4847 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4848 attributes.set_rex_vex_w_reverted();
4849 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4850 emit_int16(0x58, (0xC0 | encode));
4851 }
4852
4853 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4854 assert(VM_Version::supports_avx(), "");
4855 InstructionMark im(this);
4856 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4857 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4858 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4859 emit_int8(0x58);
4860 emit_operand(dst, src);
4861 }
4862
4863 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4864 assert(VM_Version::supports_avx(), "");
4865 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4866 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4867 emit_int16(0x58, (0xC0 | encode));
4868 }
4869
4870 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4871 assert(VM_Version::supports_avx(), "");
4872 InstructionMark im(this);
4873 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4874 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4875 attributes.set_rex_vex_w_reverted();
4876 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4877 emit_int8(0x5E);
4878 emit_operand(dst, src);
4879 }
4880
4881 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4882 assert(VM_Version::supports_avx(), "");
4883 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4884 attributes.set_rex_vex_w_reverted();
4885 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4886 emit_int16(0x5E, (0xC0 | encode));
4887 }
4888
4889 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4890 assert(VM_Version::supports_avx(), "");
4891 InstructionMark im(this);
4892 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4893 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4894 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4895 emit_int8(0x5E);
4896 emit_operand(dst, src);
4897 }
4898
4899 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4900 assert(VM_Version::supports_avx(), "");
4901 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4902 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4903 emit_int16(0x5E, (0xC0 | encode));
4904 }
4905
4906 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4907 assert(VM_Version::supports_fma(), "");
4908 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4909 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4910 emit_int16((unsigned char)0xB9, (0xC0 | encode));
4911 }
4912
4913 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4914 assert(VM_Version::supports_fma(), "");
4915 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4916 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4917 emit_int16((unsigned char)0xB9, (0xC0 | encode));
4918 }
4919
4920 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4921 assert(VM_Version::supports_avx(), "");
4922 InstructionMark im(this);
4923 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4924 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4925 attributes.set_rex_vex_w_reverted();
4926 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4927 emit_int8(0x59);
4928 emit_operand(dst, src);
4929 }
4930
4931 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4932 assert(VM_Version::supports_avx(), "");
4933 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4934 attributes.set_rex_vex_w_reverted();
4935 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4936 emit_int16(0x59, (0xC0 | encode));
4937 }
4938
4939 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4940 assert(VM_Version::supports_avx(), "");
4941 InstructionMark im(this);
4942 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4943 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4944 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4945 emit_int8(0x59);
4946 emit_operand(dst, src);
4947 }
4948
4949 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4950 assert(VM_Version::supports_avx(), "");
4951 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4952 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4953 emit_int16(0x59, (0xC0 | encode));
4954 }
4955
4956 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4957 assert(VM_Version::supports_avx(), "");
4958 InstructionMark im(this);
4959 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4960 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4961 attributes.set_rex_vex_w_reverted();
4962 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4963 emit_int8(0x5C);
4964 emit_operand(dst, src);
4965 }
4966
4967 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4968 assert(VM_Version::supports_avx(), "");
4969 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4970 attributes.set_rex_vex_w_reverted();
4971 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4972 emit_int16(0x5C, (0xC0 | encode));
4973 }
4974
4975 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4976 assert(VM_Version::supports_avx(), "");
4977 InstructionMark im(this);
4978 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4979 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4980 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4981 emit_int8(0x5C);
4982 emit_operand(dst, src);
4983 }
4984
4985 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4986 assert(VM_Version::supports_avx(), "");
4987 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4988 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4989 emit_int16(0x5C, (0xC0 | encode));
4990 }
4991
4992 //====================VECTOR ARITHMETIC=====================================
4993
4994 // Float-point vector arithmetic
4995
4996 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4997 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4998 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
4999 attributes.set_rex_vex_w_reverted();
5000 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5001 emit_int16(0x58, (0xC0 | encode));
5002 }
5003
5004 void Assembler::addpd(XMMRegister dst, Address src) {
5005 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5006 InstructionMark im(this);
5007 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5008 attributes.set_rex_vex_w_reverted();
5009 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5010 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5011 emit_int8(0x58);
5012 emit_operand(dst, src);
5013 }
5014
5015
5016 void Assembler::addps(XMMRegister dst, XMMRegister src) {
5017 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5018 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5019 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5020 emit_int16(0x58, (0xC0 | encode));
5021 }
5022
5023 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5024 assert(VM_Version::supports_avx(), "");
5025 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5026 attributes.set_rex_vex_w_reverted();
5027 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5028 emit_int16(0x58, (0xC0 | encode));
5029 }
5030
5031 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5032 assert(VM_Version::supports_avx(), "");
5033 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5034 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5035 emit_int16(0x58, (0xC0 | encode));
5036 }
5037
5038 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5039 assert(VM_Version::supports_avx(), "");
5040 InstructionMark im(this);
5041 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5042 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5043 attributes.set_rex_vex_w_reverted();
5044 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5045 emit_int8(0x58);
5046 emit_operand(dst, src);
5047 }
5048
5049 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5050 assert(VM_Version::supports_avx(), "");
5051 InstructionMark im(this);
5052 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5053 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5054 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5055 emit_int8(0x58);
5056 emit_operand(dst, src);
5057 }
5058
5059 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
5060 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5061 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5062 attributes.set_rex_vex_w_reverted();
5063 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5064 emit_int16(0x5C, (0xC0 | encode));
5065 }
5066
5067 void Assembler::subps(XMMRegister dst, XMMRegister src) {
5068 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5069 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5070 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5071 emit_int16(0x5C, (0xC0 | encode));
5072 }
5073
5074 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5075 assert(VM_Version::supports_avx(), "");
5076 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5077 attributes.set_rex_vex_w_reverted();
5078 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5079 emit_int16(0x5C, (0xC0 | encode));
5080 }
5081
5082 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5083 assert(VM_Version::supports_avx(), "");
5084 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5085 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5086 emit_int16(0x5C, (0xC0 | encode));
5087 }
5088
5089 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5090 assert(VM_Version::supports_avx(), "");
5091 InstructionMark im(this);
5092 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5093 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5094 attributes.set_rex_vex_w_reverted();
5095 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5096 emit_int8(0x5C);
5097 emit_operand(dst, src);
5098 }
5099
5100 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5101 assert(VM_Version::supports_avx(), "");
5102 InstructionMark im(this);
5103 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5104 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5105 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5106 emit_int8(0x5C);
5107 emit_operand(dst, src);
5108 }
5109
5110 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
5111 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5112 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5113 attributes.set_rex_vex_w_reverted();
5114 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5115 emit_int16(0x59, (0xC0 | encode));
5116 }
5117
5118 void Assembler::mulpd(XMMRegister dst, Address src) {
5119 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5120 InstructionMark im(this);
5121 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5122 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5123 attributes.set_rex_vex_w_reverted();
5124 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5125 emit_int8(0x59);
5126 emit_operand(dst, src);
5127 }
5128
5129 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5130 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5131 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5132 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5133 emit_int16(0x59, (0xC0 | encode));
5134 }
5135
5136 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5137 assert(VM_Version::supports_avx(), "");
5138 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5139 attributes.set_rex_vex_w_reverted();
5140 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5141 emit_int16(0x59, (0xC0 | encode));
5142 }
5143
5144 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5145 assert(VM_Version::supports_avx(), "");
5146 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5147 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5148 emit_int16(0x59, (0xC0 | encode));
5149 }
5150
5151 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5152 assert(VM_Version::supports_avx(), "");
5153 InstructionMark im(this);
5154 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5155 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5156 attributes.set_rex_vex_w_reverted();
5157 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5158 emit_int8(0x59);
5159 emit_operand(dst, src);
5160 }
5161
5162 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5163 assert(VM_Version::supports_avx(), "");
5164 InstructionMark im(this);
5165 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5166 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5167 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5168 emit_int8(0x59);
5169 emit_operand(dst, src);
5170 }
5171
5172 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5173 assert(VM_Version::supports_fma(), "");
5174 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5175 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5176 emit_int16((unsigned char)0xB8, (0xC0 | encode));
5177 }
5178
5179 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5180 assert(VM_Version::supports_fma(), "");
5181 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5182 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5183 emit_int16((unsigned char)0xB8, (0xC0 | encode));
5184 }
5185
5186 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5187 assert(VM_Version::supports_fma(), "");
5188 InstructionMark im(this);
5189 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5190 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5191 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5192 emit_int8((unsigned char)0xB8);
5193 emit_operand(dst, src2);
5194 }
5195
5196 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5197 assert(VM_Version::supports_fma(), "");
5198 InstructionMark im(this);
5199 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5200 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5201 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5202 emit_int8((unsigned char)0xB8);
5203 emit_operand(dst, src2);
5204 }
5205
5206 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5207 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5208 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5209 attributes.set_rex_vex_w_reverted();
5210 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5211 emit_int16(0x5E, (0xC0 | encode));
5212 }
5213
5214 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5215 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5216 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5217 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5218 emit_int16(0x5E, (0xC0 | encode));
5219 }
5220
5221 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5222 assert(VM_Version::supports_avx(), "");
5223 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5224 attributes.set_rex_vex_w_reverted();
5225 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5226 emit_int16(0x5E, (0xC0 | encode));
5227 }
5228
5229 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5230 assert(VM_Version::supports_avx(), "");
5231 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5232 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5233 emit_int16(0x5E, (0xC0 | encode));
5234 }
5235
5236 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5237 assert(VM_Version::supports_avx(), "");
5238 InstructionMark im(this);
5239 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5240 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5241 attributes.set_rex_vex_w_reverted();
5242 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5243 emit_int8(0x5E);
5244 emit_operand(dst, src);
5245 }
5246
5247 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5248 assert(VM_Version::supports_avx(), "");
5249 InstructionMark im(this);
5250 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5251 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5252 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5253 emit_int8(0x5E);
5254 emit_operand(dst, src);
5255 }
5256
5257 void Assembler::vroundpd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
5258 assert(VM_Version::supports_avx(), "");
5259 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5260 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5261 emit_int24(0x09, (0xC0 | encode), (rmode));
5262 }
5263
5264 void Assembler::vroundpd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
5265 assert(VM_Version::supports_avx(), "");
5266 InstructionMark im(this);
5267 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5268 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5269 emit_int8(0x09);
5270 emit_operand(dst, src);
5271 emit_int8((rmode));
5272 }
5273
5274 void Assembler::vrndscalepd(XMMRegister dst, XMMRegister src, int32_t rmode, int vector_len) {
5275 assert(VM_Version::supports_evex(), "requires EVEX support");
5276 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5277 attributes.set_is_evex_instruction();
5278 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5279 emit_int24(0x09, (0xC0 | encode), (rmode));
5280 }
5281
5282 void Assembler::vrndscalepd(XMMRegister dst, Address src, int32_t rmode, int vector_len) {
5283 assert(VM_Version::supports_evex(), "requires EVEX support");
5284 assert(dst != xnoreg, "sanity");
5285 InstructionMark im(this);
5286 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5287 attributes.set_is_evex_instruction();
5288 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5289 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5290 emit_int8(0x09);
5291 emit_operand(dst, src);
5292 emit_int8((rmode));
5293 }
5294
5295
5296 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5297 assert(VM_Version::supports_avx(), "");
5298 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5299 attributes.set_rex_vex_w_reverted();
5300 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5301 emit_int16(0x51, (0xC0 | encode));
5302 }
5303
5304 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5305 assert(VM_Version::supports_avx(), "");
5306 InstructionMark im(this);
5307 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5308 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5309 attributes.set_rex_vex_w_reverted();
5310 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5311 emit_int8(0x51);
5312 emit_operand(dst, src);
5313 }
5314
5315 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
5316 assert(VM_Version::supports_avx(), "");
5317 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5318 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5319 emit_int16(0x51, (0xC0 | encode));
5320 }
5321
5322 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
5323 assert(VM_Version::supports_avx(), "");
5324 InstructionMark im(this);
5325 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5326 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5327 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5328 emit_int8(0x51);
5329 emit_operand(dst, src);
5330 }
5331
5332 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5333 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5334 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5335 attributes.set_rex_vex_w_reverted();
5336 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5337 emit_int16(0x54, (0xC0 | encode));
5338 }
5339
5340 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5341 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5342 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5343 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5344 emit_int16(0x54, (0xC0 | encode));
5345 }
5346
5347 void Assembler::andps(XMMRegister dst, Address src) {
5348 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5349 InstructionMark im(this);
5350 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5351 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5352 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5353 emit_int8(0x54);
5354 emit_operand(dst, src);
5355 }
5356
5357 void Assembler::andpd(XMMRegister dst, Address src) {
5358 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5359 InstructionMark im(this);
5360 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5361 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5362 attributes.set_rex_vex_w_reverted();
5363 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5364 emit_int8(0x54);
5365 emit_operand(dst, src);
5366 }
5367
5368 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5369 assert(VM_Version::supports_avx(), "");
5370 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5371 attributes.set_rex_vex_w_reverted();
5372 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5373 emit_int16(0x54, (0xC0 | encode));
5374 }
5375
5376 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5377 assert(VM_Version::supports_avx(), "");
5378 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5379 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5380 emit_int16(0x54, (0xC0 | encode));
5381 }
5382
5383 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5384 assert(VM_Version::supports_avx(), "");
5385 InstructionMark im(this);
5386 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5387 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5388 attributes.set_rex_vex_w_reverted();
5389 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5390 emit_int8(0x54);
5391 emit_operand(dst, src);
5392 }
5393
5394 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5395 assert(VM_Version::supports_avx(), "");
5396 InstructionMark im(this);
5397 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5398 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5399 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5400 emit_int8(0x54);
5401 emit_operand(dst, src);
5402 }
5403
5404 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5405 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5406 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5407 attributes.set_rex_vex_w_reverted();
5408 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5409 emit_int8(0x15);
5410 emit_int8((0xC0 | encode));
5411 }
5412
5413 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5414 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5415 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5416 attributes.set_rex_vex_w_reverted();
5417 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5418 emit_int16(0x14, (0xC0 | encode));
5419 }
5420
5421 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5422 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5423 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5424 attributes.set_rex_vex_w_reverted();
5425 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5426 emit_int16(0x57, (0xC0 | encode));
5427 }
5428
5429 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5430 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5431 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5432 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5433 emit_int16(0x57, (0xC0 | encode));
5434 }
5435
5436 void Assembler::xorpd(XMMRegister dst, Address src) {
5437 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5438 InstructionMark im(this);
5439 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5440 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5441 attributes.set_rex_vex_w_reverted();
5442 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5443 emit_int8(0x57);
5444 emit_operand(dst, src);
5445 }
5446
5447 void Assembler::xorps(XMMRegister dst, Address src) {
5448 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5449 InstructionMark im(this);
5450 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5451 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5452 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5453 emit_int8(0x57);
5454 emit_operand(dst, src);
5455 }
5456
5457 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5458 assert(VM_Version::supports_avx(), "");
5459 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5460 attributes.set_rex_vex_w_reverted();
5461 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5462 emit_int16(0x57, (0xC0 | encode));
5463 }
5464
5465 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5466 assert(VM_Version::supports_avx(), "");
5467 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5468 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5469 emit_int16(0x57, (0xC0 | encode));
5470 }
5471
5472 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5473 assert(VM_Version::supports_avx(), "");
5474 InstructionMark im(this);
5475 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5476 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5477 attributes.set_rex_vex_w_reverted();
5478 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5479 emit_int8(0x57);
5480 emit_operand(dst, src);
5481 }
5482
5483 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5484 assert(VM_Version::supports_avx(), "");
5485 InstructionMark im(this);
5486 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5487 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5488 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5489 emit_int8(0x57);
5490 emit_operand(dst, src);
5491 }
5492
5493 // Integer vector arithmetic
5494 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5495 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5496 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5497 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
5498 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5499 emit_int16(0x01, (0xC0 | encode));
5500 }
5501
5502 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5503 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5504 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5505 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
5506 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5507 emit_int16(0x02, (0xC0 | encode));
5508 }
5509
5510 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5511 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5512 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5513 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5514 emit_int16((unsigned char)0xFC, (0xC0 | encode));
5515 }
5516
5517 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5518 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5519 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5520 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5521 emit_int16((unsigned char)0xFD, (0xC0 | encode));
5522 }
5523
5524 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5525 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5526 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5527 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5528 emit_int16((unsigned char)0xFE, (0xC0 | encode));
5529 }
5530
5531 void Assembler::paddd(XMMRegister dst, Address src) {
5532 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5533 InstructionMark im(this);
5534 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5535 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5536 emit_int8((unsigned char)0xFE);
5537 emit_operand(dst, src);
5538 }
5539
5540 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5541 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5542 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5543 attributes.set_rex_vex_w_reverted();
5544 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5545 emit_int16((unsigned char)0xD4, (0xC0 | encode));
5546 }
5547
5548 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5549 assert(VM_Version::supports_sse3(), "");
5550 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
5551 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5552 emit_int16(0x01, (0xC0 | encode));
5553 }
5554
5555 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5556 assert(VM_Version::supports_sse3(), "");
5557 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
5558 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5559 emit_int16(0x02, (0xC0 | encode));
5560 }
5561
5562 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5563 assert(UseAVX > 0, "requires some form of AVX");
5564 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5565 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5566 emit_int16((unsigned char)0xFC, (0xC0 | encode));
5567 }
5568
5569 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5570 assert(UseAVX > 0, "requires some form of AVX");
5571 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5572 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5573 emit_int16((unsigned char)0xFD, (0xC0 | encode));
5574 }
5575
5576 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5577 assert(UseAVX > 0, "requires some form of AVX");
5578 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5579 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5580 emit_int16((unsigned char)0xFE, (0xC0 | encode));
5581 }
5582
5583 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5584 assert(UseAVX > 0, "requires some form of AVX");
5585 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5586 attributes.set_rex_vex_w_reverted();
5587 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5588 emit_int16((unsigned char)0xD4, (0xC0 | encode));
5589 }
5590
5591 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5592 assert(UseAVX > 0, "requires some form of AVX");
5593 InstructionMark im(this);
5594 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5595 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5596 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5597 emit_int8((unsigned char)0xFC);
5598 emit_operand(dst, src);
5599 }
5600
5601 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5602 assert(UseAVX > 0, "requires some form of AVX");
5603 InstructionMark im(this);
5604 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5605 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5606 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5607 emit_int8((unsigned char)0xFD);
5608 emit_operand(dst, src);
5609 }
5610
5611 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5612 assert(UseAVX > 0, "requires some form of AVX");
5613 InstructionMark im(this);
5614 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5615 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5616 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5617 emit_int8((unsigned char)0xFE);
5618 emit_operand(dst, src);
5619 }
5620
5621 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5622 assert(UseAVX > 0, "requires some form of AVX");
5623 InstructionMark im(this);
5624 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5625 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5626 attributes.set_rex_vex_w_reverted();
5627 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5628 emit_int8((unsigned char)0xD4);
5629 emit_operand(dst, src);
5630 }
5631
5632 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5633 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5634 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5635 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5636 emit_int16((unsigned char)0xF8, (0xC0 | encode));
5637 }
5638
5639 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5640 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5641 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5642 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5643 emit_int16((unsigned char)0xF9, (0xC0 | encode));
5644 }
5645
5646 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5647 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5648 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5649 emit_int16((unsigned char)0xFA, (0xC0 | encode));
5650 }
5651
5652 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5653 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5654 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5655 attributes.set_rex_vex_w_reverted();
5656 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5657 emit_int8((unsigned char)0xFB);
5658 emit_int8((0xC0 | encode));
5659 }
5660
5661 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5662 assert(UseAVX > 0, "requires some form of AVX");
5663 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5664 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5665 emit_int16((unsigned char)0xF8, (0xC0 | encode));
5666 }
5667
5668 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5669 assert(UseAVX > 0, "requires some form of AVX");
5670 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5671 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5672 emit_int16((unsigned char)0xF9, (0xC0 | encode));
5673 }
5674
5675 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5676 assert(UseAVX > 0, "requires some form of AVX");
5677 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5678 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5679 emit_int16((unsigned char)0xFA, (0xC0 | encode));
5680 }
5681
5682 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5683 assert(UseAVX > 0, "requires some form of AVX");
5684 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5685 attributes.set_rex_vex_w_reverted();
5686 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5687 emit_int16((unsigned char)0xFB, (0xC0 | encode));
5688 }
5689
5690 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5691 assert(UseAVX > 0, "requires some form of AVX");
5692 InstructionMark im(this);
5693 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5694 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5695 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5696 emit_int8((unsigned char)0xF8);
5697 emit_operand(dst, src);
5698 }
5699
5700 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5701 assert(UseAVX > 0, "requires some form of AVX");
5702 InstructionMark im(this);
5703 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5704 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5705 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5706 emit_int8((unsigned char)0xF9);
5707 emit_operand(dst, src);
5708 }
5709
5710 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5711 assert(UseAVX > 0, "requires some form of AVX");
5712 InstructionMark im(this);
5713 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5714 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5715 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5716 emit_int8((unsigned char)0xFA);
5717 emit_operand(dst, src);
5718 }
5719
5720 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5721 assert(UseAVX > 0, "requires some form of AVX");
5722 InstructionMark im(this);
5723 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5724 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5725 attributes.set_rex_vex_w_reverted();
5726 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5727 emit_int8((unsigned char)0xFB);
5728 emit_operand(dst, src);
5729 }
5730
5731 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5732 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5733 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5734 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5735 emit_int16((unsigned char)0xD5, (0xC0 | encode));
5736 }
5737
5738 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5739 assert(VM_Version::supports_sse4_1(), "");
5740 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5741 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5742 emit_int16(0x40, (0xC0 | encode));
5743 }
5744
5745 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5746 assert(UseAVX > 0, "requires some form of AVX");
5747 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5748 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5749 emit_int16((unsigned char)0xD5, (0xC0 | encode));
5750 }
5751
5752 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5753 assert(UseAVX > 0, "requires some form of AVX");
5754 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5755 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5756 emit_int16(0x40, (0xC0 | encode));
5757 }
5758
5759 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5760 assert(UseAVX > 2, "requires some form of EVEX");
5761 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5762 attributes.set_is_evex_instruction();
5763 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5764 emit_int16(0x40, (0xC0 | encode));
5765 }
5766
5767 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5768 assert(UseAVX > 0, "requires some form of AVX");
5769 InstructionMark im(this);
5770 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5771 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5772 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5773 emit_int8((unsigned char)0xD5);
5774 emit_operand(dst, src);
5775 }
5776
5777 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5778 assert(UseAVX > 0, "requires some form of AVX");
5779 InstructionMark im(this);
5780 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5781 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5782 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5783 emit_int8(0x40);
5784 emit_operand(dst, src);
5785 }
5786
5787 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5788 assert(UseAVX > 2, "requires some form of EVEX");
5789 InstructionMark im(this);
5790 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ true);
5791 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5792 attributes.set_is_evex_instruction();
5793 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5794 emit_int8(0x40);
5795 emit_operand(dst, src);
5796 }
5797
5798 // Shift packed integers left by specified number of bits.
5799 void Assembler::psllw(XMMRegister dst, int shift) {
5800 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5801 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5802 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5803 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5804 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
5805 }
5806
5807 void Assembler::pslld(XMMRegister dst, int shift) {
5808 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5809 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5810 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5811 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5812 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
5813 }
5814
5815 void Assembler::psllq(XMMRegister dst, int shift) {
5816 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5817 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5818 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5819 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5820 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5821 }
5822
5823 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5824 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5825 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5826 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5827 emit_int16((unsigned char)0xF1, (0xC0 | encode));
5828 }
5829
5830 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5831 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5832 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5833 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5834 emit_int16((unsigned char)0xF2, (0xC0 | encode));
5835 }
5836
5837 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5838 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5839 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5840 attributes.set_rex_vex_w_reverted();
5841 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5842 emit_int16((unsigned char)0xF3, (0xC0 | encode));
5843 }
5844
5845 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5846 assert(UseAVX > 0, "requires some form of AVX");
5847 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5848 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5849 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5850 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
5851 }
5852
5853 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5854 assert(UseAVX > 0, "requires some form of AVX");
5855 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5856 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5857 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5858 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5859 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
5860 }
5861
5862 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5863 assert(UseAVX > 0, "requires some form of AVX");
5864 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5865 attributes.set_rex_vex_w_reverted();
5866 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5867 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5868 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5869 }
5870
5871 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5872 assert(UseAVX > 0, "requires some form of AVX");
5873 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5874 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5875 emit_int16((unsigned char)0xF1, (0xC0 | encode));
5876 }
5877
5878 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5879 assert(UseAVX > 0, "requires some form of AVX");
5880 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5881 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5882 emit_int16((unsigned char)0xF2, (0xC0 | encode));
5883 }
5884
5885 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5886 assert(UseAVX > 0, "requires some form of AVX");
5887 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5888 attributes.set_rex_vex_w_reverted();
5889 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5890 emit_int16((unsigned char)0xF3, (0xC0 | encode));
5891 }
5892
5893 // Shift packed integers logically right by specified number of bits.
5894 void Assembler::psrlw(XMMRegister dst, int shift) {
5895 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5896 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5897 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5898 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5899 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
5900 }
5901
5902 void Assembler::psrld(XMMRegister dst, int shift) {
5903 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5904 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5905 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5906 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5907 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
5908 }
5909
5910 void Assembler::psrlq(XMMRegister dst, int shift) {
5911 // Do not confuse it with psrldq SSE2 instruction which
5912 // shifts 128 bit value in xmm register by number of bytes.
5913 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5914 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5915 attributes.set_rex_vex_w_reverted();
5916 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5917 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5918 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5919 }
5920
5921 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5922 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5923 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5924 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5925 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5926 }
5927
5928 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5929 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5930 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5931 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5932 emit_int16((unsigned char)0xD2, (0xC0 | encode));
5933 }
5934
5935 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5936 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5937 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5938 attributes.set_rex_vex_w_reverted();
5939 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5940 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5941 }
5942
5943 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5944 assert(UseAVX > 0, "requires some form of AVX");
5945 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5946 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5947 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5948 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
5949 }
5950
5951 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5952 assert(UseAVX > 0, "requires some form of AVX");
5953 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5954 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5955 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5956 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
5957 }
5958
5959 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5960 assert(UseAVX > 0, "requires some form of AVX");
5961 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5962 attributes.set_rex_vex_w_reverted();
5963 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5964 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5965 emit_int24(0x73, (0xC0 | encode), shift & 0xFF);
5966 }
5967
5968 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5969 assert(UseAVX > 0, "requires some form of AVX");
5970 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5971 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5972 emit_int16((unsigned char)0xD1, (0xC0 | encode));
5973 }
5974
5975 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5976 assert(UseAVX > 0, "requires some form of AVX");
5977 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5978 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5979 emit_int16((unsigned char)0xD2, (0xC0 | encode));
5980 }
5981
5982 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5983 assert(UseAVX > 0, "requires some form of AVX");
5984 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5985 attributes.set_rex_vex_w_reverted();
5986 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5987 emit_int16((unsigned char)0xD3, (0xC0 | encode));
5988 }
5989
5990 void Assembler::evpsrlvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5991 assert(VM_Version::supports_avx512bw(), "");
5992 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
5993 attributes.set_is_evex_instruction();
5994 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5995 emit_int16(0x10, (0xC0 | encode));
5996 }
5997
5998 void Assembler::evpsllvw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5999 assert(VM_Version::supports_avx512bw(), "");
6000 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6001 attributes.set_is_evex_instruction();
6002 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6003 emit_int16(0x12, (0xC0 | encode));
6004 }
6005
6006 // Shift packed integers arithmetically right by specified number of bits.
6007 void Assembler::psraw(XMMRegister dst, int shift) {
6008 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6009 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6010 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6011 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6012 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
6013 }
6014
6015 void Assembler::psrad(XMMRegister dst, int shift) {
6016 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6017 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6018 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6019 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6020 emit_int8(0x72);
6021 emit_int8((0xC0 | encode));
6022 emit_int8(shift & 0xFF);
6023 }
6024
6025 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
6026 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6027 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6028 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6029 emit_int16((unsigned char)0xE1, (0xC0 | encode));
6030 }
6031
6032 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
6033 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6034 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6035 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6036 emit_int16((unsigned char)0xE2, (0xC0 | encode));
6037 }
6038
6039 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6040 assert(UseAVX > 0, "requires some form of AVX");
6041 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6042 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6043 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6044 emit_int24(0x71, (0xC0 | encode), shift & 0xFF);
6045 }
6046
6047 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6048 assert(UseAVX > 0, "requires some form of AVX");
6049 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6050 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6051 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6052 emit_int24(0x72, (0xC0 | encode), shift & 0xFF);
6053 }
6054
6055 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6056 assert(UseAVX > 0, "requires some form of AVX");
6057 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6058 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6059 emit_int16((unsigned char)0xE1, (0xC0 | encode));
6060 }
6061
6062 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6063 assert(UseAVX > 0, "requires some form of AVX");
6064 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6065 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6066 emit_int16((unsigned char)0xE2, (0xC0 | encode));
6067 }
6068
6069 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6070 assert(UseAVX > 2, "requires AVX512");
6071 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
6072 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6073 attributes.set_is_evex_instruction();
6074 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6075 emit_int24((unsigned char)0x72, (0xC0 | encode), shift & 0xFF);
6076 }
6077
6078 void Assembler::evpsraq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6079 assert(UseAVX > 2, "requires AVX512");
6080 assert ((VM_Version::supports_avx512vl() || vector_len == 2), "requires AVX512vl");
6081 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6082 attributes.set_is_evex_instruction();
6083 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6084 emit_int16((unsigned char)0xE2, (0xC0 | encode));
6085 }
6086
6087 // logical operations packed integers
6088 void Assembler::pand(XMMRegister dst, XMMRegister src) {
6089 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6090 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6091 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6092 emit_int16((unsigned char)0xDB, (0xC0 | encode));
6093 }
6094
6095 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6096 assert(UseAVX > 0, "requires some form of AVX");
6097 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6098 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6099 emit_int16((unsigned char)0xDB, (0xC0 | encode));
6100 }
6101
6102 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6103 assert(UseAVX > 0, "requires some form of AVX");
6104 InstructionMark im(this);
6105 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6106 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6107 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6108 emit_int8((unsigned char)0xDB);
6109 emit_operand(dst, src);
6110 }
6111
6112 void Assembler::vpandq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6113 assert(VM_Version::supports_evex(), "");
6114 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6115 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6116 emit_int16((unsigned char)0xDB, (0xC0 | encode));
6117 }
6118
6119 void Assembler::vpshldvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6120 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
6121 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6122 attributes.set_is_evex_instruction();
6123 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6124 emit_int8(0x71);
6125 emit_int8((0xC0 | encode));
6126 }
6127
6128 void Assembler::vpshrdvd(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6129 assert(VM_Version::supports_avx512_vbmi2(), "requires vbmi2");
6130 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6131 attributes.set_is_evex_instruction();
6132 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6133 emit_int16(0x73, (0xC0 | encode));
6134 }
6135
6136 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6137 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6138 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6139 attributes.set_rex_vex_w_reverted();
6140 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6141 emit_int16((unsigned char)0xDF, (0xC0 | encode));
6142 }
6143
6144 void Assembler::vpandn(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6145 assert(UseAVX > 0, "requires some form of AVX");
6146 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6147 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6148 emit_int16((unsigned char)0xDF, (0xC0 | encode));
6149 }
6150
6151
6152 void Assembler::por(XMMRegister dst, XMMRegister src) {
6153 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6154 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6155 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6156 emit_int16((unsigned char)0xEB, (0xC0 | encode));
6157 }
6158
6159 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6160 assert(UseAVX > 0, "requires some form of AVX");
6161 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6162 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6163 emit_int16((unsigned char)0xEB, (0xC0 | encode));
6164 }
6165
6166 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6167 assert(UseAVX > 0, "requires some form of AVX");
6168 InstructionMark im(this);
6169 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6170 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6171 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6172 emit_int8((unsigned char)0xEB);
6173 emit_operand(dst, src);
6174 }
6175
6176 void Assembler::vporq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6177 assert(VM_Version::supports_evex(), "");
6178 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6179 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6180 emit_int16((unsigned char)0xEB, (0xC0 | encode));
6181 }
6182
6183
6184 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6185 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6186 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6187 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6188 emit_int16((unsigned char)0xEF, (0xC0 | encode));
6189 }
6190
6191 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6192 assert(UseAVX > 0, "requires some form of AVX");
6193 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6194 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6195 emit_int16((unsigned char)0xEF, (0xC0 | encode));
6196 }
6197
6198 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6199 assert(UseAVX > 0, "requires some form of AVX");
6200 InstructionMark im(this);
6201 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6202 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6203 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6204 emit_int8((unsigned char)0xEF);
6205 emit_operand(dst, src);
6206 }
6207
6208 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6209 assert(VM_Version::supports_evex(), "requires EVEX support");
6210 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6211 attributes.set_is_evex_instruction();
6212 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6213 emit_int8((unsigned char)0xEF);
6214 emit_int8((0xC0 | encode));
6215 }
6216
6217 void Assembler::evpxorq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6218 assert(VM_Version::supports_evex(), "requires EVEX support");
6219 assert(dst != xnoreg, "sanity");
6220 InstructionMark im(this);
6221 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6222 attributes.set_is_evex_instruction();
6223 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6224 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6225 emit_int8((unsigned char)0xEF);
6226 emit_operand(dst, src);
6227 }
6228
6229 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
6230 assert(VM_Version::supports_evex(), "requires EVEX support");
6231 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
6232 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6233 attributes.set_is_evex_instruction();
6234 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6235 emit_int8(0x25);
6236 emit_int8((unsigned char)(0xC0 | encode));
6237 emit_int8(imm8);
6238 }
6239
6240 void Assembler::vpternlogd(XMMRegister dst, int imm8, XMMRegister src2, Address src3, int vector_len) {
6241 assert(VM_Version::supports_evex(), "requires EVEX support");
6242 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
6243 assert(dst != xnoreg, "sanity");
6244 InstructionMark im(this);
6245 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6246 attributes.set_is_evex_instruction();
6247 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
6248 vex_prefix(src3, src2->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6249 emit_int8(0x25);
6250 emit_operand(dst, src3);
6251 emit_int8(imm8);
6252 }
6253
6254 void Assembler::vpternlogq(XMMRegister dst, int imm8, XMMRegister src2, XMMRegister src3, int vector_len) {
6255 assert(VM_Version::supports_evex(), "requires EVEX support");
6256 assert(vector_len == Assembler::AVX_512bit || VM_Version::supports_avx512vl(), "requires VL support");
6257 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6258 attributes.set_is_evex_instruction();
6259 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src3->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6260 emit_int8(0x25);
6261 emit_int8((unsigned char)(0xC0 | encode));
6262 emit_int8(imm8);
6263 }
6264
6265 // vinserti forms
6266
6267 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6268 assert(VM_Version::supports_avx2(), "");
6269 assert(imm8 <= 0x01, "imm8: %u", imm8);
6270 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6271 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6272 // last byte:
6273 // 0x00 - insert into lower 128 bits
6274 // 0x01 - insert into upper 128 bits
6275 emit_int24(0x38, (0xC0 | encode), imm8 & 0x01);
6276 }
6277
6278 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6279 assert(VM_Version::supports_avx2(), "");
6280 assert(dst != xnoreg, "sanity");
6281 assert(imm8 <= 0x01, "imm8: %u", imm8);
6282 InstructionMark im(this);
6283 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6284 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6285 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6286 emit_int8(0x38);
6287 emit_operand(dst, src);
6288 // 0x00 - insert into lower 128 bits
6289 // 0x01 - insert into upper 128 bits
6290 emit_int8(imm8 & 0x01);
6291 }
6292
6293 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6294 assert(VM_Version::supports_evex(), "");
6295 assert(imm8 <= 0x03, "imm8: %u", imm8);
6296 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6297 attributes.set_is_evex_instruction();
6298 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6299 // imm8:
6300 // 0x00 - insert into q0 128 bits (0..127)
6301 // 0x01 - insert into q1 128 bits (128..255)
6302 // 0x02 - insert into q2 128 bits (256..383)
6303 // 0x03 - insert into q3 128 bits (384..511)
6304 emit_int24(0x38, (0xC0 | encode), imm8 & 0x03);
6305 }
6306
6307 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6308 assert(VM_Version::supports_avx(), "");
6309 assert(dst != xnoreg, "sanity");
6310 assert(imm8 <= 0x03, "imm8: %u", imm8);
6311 InstructionMark im(this);
6312 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6313 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6314 attributes.set_is_evex_instruction();
6315 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6316 emit_int8(0x18);
6317 emit_operand(dst, src);
6318 // 0x00 - insert into q0 128 bits (0..127)
6319 // 0x01 - insert into q1 128 bits (128..255)
6320 // 0x02 - insert into q2 128 bits (256..383)
6321 // 0x03 - insert into q3 128 bits (384..511)
6322 emit_int8(imm8 & 0x03);
6323 }
6324
6325 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6326 assert(VM_Version::supports_evex(), "");
6327 assert(imm8 <= 0x01, "imm8: %u", imm8);
6328 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6329 attributes.set_is_evex_instruction();
6330 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6331 //imm8:
6332 // 0x00 - insert into lower 256 bits
6333 // 0x01 - insert into upper 256 bits
6334 emit_int24(0x3A, (0xC0 | encode), imm8 & 0x01);
6335 }
6336
6337
6338 // vinsertf forms
6339
6340 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6341 assert(VM_Version::supports_avx(), "");
6342 assert(imm8 <= 0x01, "imm8: %u", imm8);
6343 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6344 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6345 // imm8:
6346 // 0x00 - insert into lower 128 bits
6347 // 0x01 - insert into upper 128 bits
6348 emit_int24(0x18, (0xC0 | encode), imm8 & 0x01);
6349 }
6350
6351 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6352 assert(VM_Version::supports_avx(), "");
6353 assert(dst != xnoreg, "sanity");
6354 assert(imm8 <= 0x01, "imm8: %u", imm8);
6355 InstructionMark im(this);
6356 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6357 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6358 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6359 emit_int8(0x18);
6360 emit_operand(dst, src);
6361 // 0x00 - insert into lower 128 bits
6362 // 0x01 - insert into upper 128 bits
6363 emit_int8(imm8 & 0x01);
6364 }
6365
6366 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6367 assert(VM_Version::supports_avx2(), "");
6368 assert(imm8 <= 0x03, "imm8: %u", imm8);
6369 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6370 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6371 // imm8:
6372 // 0x00 - insert into q0 128 bits (0..127)
6373 // 0x01 - insert into q1 128 bits (128..255)
6374 // 0x02 - insert into q0 128 bits (256..383)
6375 // 0x03 - insert into q1 128 bits (384..512)
6376 emit_int24(0x18, (0xC0 | encode), imm8 & 0x03);
6377 }
6378
6379 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6380 assert(VM_Version::supports_avx(), "");
6381 assert(dst != xnoreg, "sanity");
6382 assert(imm8 <= 0x03, "imm8: %u", imm8);
6383 InstructionMark im(this);
6384 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6385 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6386 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6387 emit_int8(0x18);
6388 emit_operand(dst, src);
6389 // 0x00 - insert into q0 128 bits (0..127)
6390 // 0x01 - insert into q1 128 bits (128..255)
6391 // 0x02 - insert into q0 128 bits (256..383)
6392 // 0x03 - insert into q1 128 bits (384..512)
6393 emit_int8(imm8 & 0x03);
6394 }
6395
6396 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6397 assert(VM_Version::supports_evex(), "");
6398 assert(imm8 <= 0x01, "imm8: %u", imm8);
6399 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6400 attributes.set_is_evex_instruction();
6401 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6402 // imm8:
6403 // 0x00 - insert into lower 256 bits
6404 // 0x01 - insert into upper 256 bits
6405 emit_int24(0x1A, (0xC0 | encode), imm8 & 0x01);
6406 }
6407
6408 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6409 assert(VM_Version::supports_evex(), "");
6410 assert(dst != xnoreg, "sanity");
6411 assert(imm8 <= 0x01, "imm8: %u", imm8);
6412 InstructionMark im(this);
6413 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6414 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6415 attributes.set_is_evex_instruction();
6416 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6417 emit_int8(0x1A);
6418 emit_operand(dst, src);
6419 // 0x00 - insert into lower 256 bits
6420 // 0x01 - insert into upper 256 bits
6421 emit_int8(imm8 & 0x01);
6422 }
6423
6424
6425 // vextracti forms
6426
6427 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6428 assert(VM_Version::supports_avx2(), "");
6429 assert(imm8 <= 0x01, "imm8: %u", imm8);
6430 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6431 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6432 // imm8:
6433 // 0x00 - extract from lower 128 bits
6434 // 0x01 - extract from upper 128 bits
6435 emit_int24(0x39, (0xC0 | encode), imm8 & 0x01);
6436 }
6437
6438 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6439 assert(VM_Version::supports_avx2(), "");
6440 assert(src != xnoreg, "sanity");
6441 assert(imm8 <= 0x01, "imm8: %u", imm8);
6442 InstructionMark im(this);
6443 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6444 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6445 attributes.reset_is_clear_context();
6446 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6447 emit_int8(0x39);
6448 emit_operand(src, dst);
6449 // 0x00 - extract from lower 128 bits
6450 // 0x01 - extract from upper 128 bits
6451 emit_int8(imm8 & 0x01);
6452 }
6453
6454 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6455 assert(VM_Version::supports_evex(), "");
6456 assert(imm8 <= 0x03, "imm8: %u", imm8);
6457 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6458 attributes.set_is_evex_instruction();
6459 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6460 // imm8:
6461 // 0x00 - extract from bits 127:0
6462 // 0x01 - extract from bits 255:128
6463 // 0x02 - extract from bits 383:256
6464 // 0x03 - extract from bits 511:384
6465 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
6466 }
6467
6468 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6469 assert(VM_Version::supports_evex(), "");
6470 assert(src != xnoreg, "sanity");
6471 assert(imm8 <= 0x03, "imm8: %u", imm8);
6472 InstructionMark im(this);
6473 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6474 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6475 attributes.reset_is_clear_context();
6476 attributes.set_is_evex_instruction();
6477 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6478 emit_int8(0x39);
6479 emit_operand(src, dst);
6480 // 0x00 - extract from bits 127:0
6481 // 0x01 - extract from bits 255:128
6482 // 0x02 - extract from bits 383:256
6483 // 0x03 - extract from bits 511:384
6484 emit_int8(imm8 & 0x03);
6485 }
6486
6487 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6488 assert(VM_Version::supports_avx512dq(), "");
6489 assert(imm8 <= 0x03, "imm8: %u", imm8);
6490 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6491 attributes.set_is_evex_instruction();
6492 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6493 // imm8:
6494 // 0x00 - extract from bits 127:0
6495 // 0x01 - extract from bits 255:128
6496 // 0x02 - extract from bits 383:256
6497 // 0x03 - extract from bits 511:384
6498 emit_int24(0x39, (0xC0 | encode), imm8 & 0x03);
6499 }
6500
6501 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6502 assert(VM_Version::supports_evex(), "");
6503 assert(imm8 <= 0x01, "imm8: %u", imm8);
6504 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6505 attributes.set_is_evex_instruction();
6506 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6507 // imm8:
6508 // 0x00 - extract from lower 256 bits
6509 // 0x01 - extract from upper 256 bits
6510 emit_int24(0x3B, (0xC0 | encode), imm8 & 0x01);
6511 }
6512
6513 void Assembler::vextracti64x4(Address dst, XMMRegister src, uint8_t imm8) {
6514 assert(VM_Version::supports_evex(), "");
6515 assert(src != xnoreg, "sanity");
6516 assert(imm8 <= 0x01, "imm8: %u", imm8);
6517 InstructionMark im(this);
6518 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6519 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6520 attributes.reset_is_clear_context();
6521 attributes.set_is_evex_instruction();
6522 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6523 emit_int8(0x38);
6524 emit_operand(src, dst);
6525 // 0x00 - extract from lower 256 bits
6526 // 0x01 - extract from upper 256 bits
6527 emit_int8(imm8 & 0x01);
6528 }
6529 // vextractf forms
6530
6531 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6532 assert(VM_Version::supports_avx(), "");
6533 assert(imm8 <= 0x01, "imm8: %u", imm8);
6534 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6535 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6536 // imm8:
6537 // 0x00 - extract from lower 128 bits
6538 // 0x01 - extract from upper 128 bits
6539 emit_int24(0x19, (0xC0 | encode), imm8 & 0x01);
6540 }
6541
6542 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6543 assert(VM_Version::supports_avx(), "");
6544 assert(src != xnoreg, "sanity");
6545 assert(imm8 <= 0x01, "imm8: %u", imm8);
6546 InstructionMark im(this);
6547 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6548 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6549 attributes.reset_is_clear_context();
6550 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6551 emit_int8(0x19);
6552 emit_operand(src, dst);
6553 // 0x00 - extract from lower 128 bits
6554 // 0x01 - extract from upper 128 bits
6555 emit_int8(imm8 & 0x01);
6556 }
6557
6558 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6559 assert(VM_Version::supports_evex(), "");
6560 assert(imm8 <= 0x03, "imm8: %u", imm8);
6561 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6562 attributes.set_is_evex_instruction();
6563 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6564 // imm8:
6565 // 0x00 - extract from bits 127:0
6566 // 0x01 - extract from bits 255:128
6567 // 0x02 - extract from bits 383:256
6568 // 0x03 - extract from bits 511:384
6569 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
6570 }
6571
6572 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6573 assert(VM_Version::supports_evex(), "");
6574 assert(src != xnoreg, "sanity");
6575 assert(imm8 <= 0x03, "imm8: %u", imm8);
6576 InstructionMark im(this);
6577 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6578 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6579 attributes.reset_is_clear_context();
6580 attributes.set_is_evex_instruction();
6581 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6582 emit_int8(0x19);
6583 emit_operand(src, dst);
6584 // 0x00 - extract from bits 127:0
6585 // 0x01 - extract from bits 255:128
6586 // 0x02 - extract from bits 383:256
6587 // 0x03 - extract from bits 511:384
6588 emit_int8(imm8 & 0x03);
6589 }
6590
6591 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6592 assert(VM_Version::supports_avx512dq(), "");
6593 assert(imm8 <= 0x03, "imm8: %u", imm8);
6594 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6595 attributes.set_is_evex_instruction();
6596 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6597 // imm8:
6598 // 0x00 - extract from bits 127:0
6599 // 0x01 - extract from bits 255:128
6600 // 0x02 - extract from bits 383:256
6601 // 0x03 - extract from bits 511:384
6602 emit_int24(0x19, (0xC0 | encode), imm8 & 0x03);
6603 }
6604
6605 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6606 assert(VM_Version::supports_evex(), "");
6607 assert(imm8 <= 0x01, "imm8: %u", imm8);
6608 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6609 attributes.set_is_evex_instruction();
6610 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6611 // imm8:
6612 // 0x00 - extract from lower 256 bits
6613 // 0x01 - extract from upper 256 bits
6614 emit_int24(0x1B, (0xC0 | encode), imm8 & 0x01);
6615 }
6616
6617 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6618 assert(VM_Version::supports_evex(), "");
6619 assert(src != xnoreg, "sanity");
6620 assert(imm8 <= 0x01, "imm8: %u", imm8);
6621 InstructionMark im(this);
6622 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6623 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
6624 attributes.reset_is_clear_context();
6625 attributes.set_is_evex_instruction();
6626 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6627 emit_int8(0x1B);
6628 emit_operand(src, dst);
6629 // 0x00 - extract from lower 256 bits
6630 // 0x01 - extract from upper 256 bits
6631 emit_int8(imm8 & 0x01);
6632 }
6633
6634 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6635 void Assembler::vpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6636 assert(VM_Version::supports_avx2(), "");
6637 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6638 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6639 emit_int16(0x78, (0xC0 | encode));
6640 }
6641
6642 void Assembler::vpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6643 assert(VM_Version::supports_avx2(), "");
6644 assert(dst != xnoreg, "sanity");
6645 InstructionMark im(this);
6646 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6647 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6648 // swap src<->dst for encoding
6649 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6650 emit_int8(0x78);
6651 emit_operand(dst, src);
6652 }
6653
6654 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6655 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6656 assert(VM_Version::supports_avx2(), "");
6657 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6658 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6659 emit_int16(0x79, (0xC0 | encode));
6660 }
6661
6662 void Assembler::vpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6663 assert(VM_Version::supports_avx2(), "");
6664 assert(dst != xnoreg, "sanity");
6665 InstructionMark im(this);
6666 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6667 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6668 // swap src<->dst for encoding
6669 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6670 emit_int8(0x79);
6671 emit_operand(dst, src);
6672 }
6673
6674 // xmm/mem sourced byte/word/dword/qword replicate
6675
6676 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6677 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6678 assert(UseAVX >= 2, "");
6679 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6680 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6681 emit_int16(0x58, (0xC0 | encode));
6682 }
6683
6684 void Assembler::vpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6685 assert(VM_Version::supports_avx2(), "");
6686 assert(dst != xnoreg, "sanity");
6687 InstructionMark im(this);
6688 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6689 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6690 // swap src<->dst for encoding
6691 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6692 emit_int8(0x58);
6693 emit_operand(dst, src);
6694 }
6695
6696 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6697 void Assembler::vpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6698 assert(VM_Version::supports_avx2(), "");
6699 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6700 attributes.set_rex_vex_w_reverted();
6701 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6702 emit_int16(0x59, (0xC0 | encode));
6703 }
6704
6705 void Assembler::vpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6706 assert(VM_Version::supports_avx2(), "");
6707 assert(dst != xnoreg, "sanity");
6708 InstructionMark im(this);
6709 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6710 attributes.set_rex_vex_w_reverted();
6711 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6712 // swap src<->dst for encoding
6713 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6714 emit_int8(0x59);
6715 emit_operand(dst, src);
6716 }
6717
6718 void Assembler::evbroadcasti32x4(XMMRegister dst, Address src, int vector_len) {
6719 assert(vector_len != Assembler::AVX_128bit, "");
6720 assert(VM_Version::supports_avx512dq(), "");
6721 assert(dst != xnoreg, "sanity");
6722 InstructionMark im(this);
6723 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6724 attributes.set_rex_vex_w_reverted();
6725 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
6726 // swap src<->dst for encoding
6727 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6728 emit_int8(0x5A);
6729 emit_operand(dst, src);
6730 }
6731
6732 void Assembler::evbroadcasti64x2(XMMRegister dst, XMMRegister src, int vector_len) {
6733 assert(vector_len != Assembler::AVX_128bit, "");
6734 assert(VM_Version::supports_avx512dq(), "");
6735 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6736 attributes.set_rex_vex_w_reverted();
6737 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6738 emit_int16(0x5A, (0xC0 | encode));
6739 }
6740
6741 void Assembler::evbroadcasti64x2(XMMRegister dst, Address src, int vector_len) {
6742 assert(vector_len != Assembler::AVX_128bit, "");
6743 assert(VM_Version::supports_avx512dq(), "");
6744 assert(dst != xnoreg, "sanity");
6745 InstructionMark im(this);
6746 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6747 attributes.set_rex_vex_w_reverted();
6748 attributes.set_address_attributes(/* tuple_type */ EVEX_T2, /* input_size_in_bits */ EVEX_64bit);
6749 // swap src<->dst for encoding
6750 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6751 emit_int8(0x5A);
6752 emit_operand(dst, src);
6753 }
6754
6755 // scalar single/double precision replicate
6756
6757 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6758 void Assembler::vbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6759 assert(VM_Version::supports_avx2(), "");
6760 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6761 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6762 emit_int16(0x18, (0xC0 | encode));
6763 }
6764
6765 void Assembler::vbroadcastss(XMMRegister dst, Address src, int vector_len) {
6766 assert(VM_Version::supports_avx(), "");
6767 assert(dst != xnoreg, "sanity");
6768 InstructionMark im(this);
6769 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6770 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6771 // swap src<->dst for encoding
6772 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6773 emit_int8(0x18);
6774 emit_operand(dst, src);
6775 }
6776
6777 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6778 void Assembler::vbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6779 assert(VM_Version::supports_avx2(), "");
6780 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
6781 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6782 attributes.set_rex_vex_w_reverted();
6783 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6784 emit_int16(0x19, (0xC0 | encode));
6785 }
6786
6787 void Assembler::vbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6788 assert(VM_Version::supports_avx(), "");
6789 assert(vector_len == AVX_256bit || vector_len == AVX_512bit, "");
6790 assert(dst != xnoreg, "sanity");
6791 InstructionMark im(this);
6792 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6793 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6794 attributes.set_rex_vex_w_reverted();
6795 // swap src<->dst for encoding
6796 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6797 emit_int8(0x19);
6798 emit_operand(dst, src);
6799 }
6800
6801
6802 // gpr source broadcast forms
6803
6804 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6805 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6806 assert(VM_Version::supports_avx512bw(), "");
6807 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6808 attributes.set_is_evex_instruction();
6809 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6810 emit_int16(0x7A, (0xC0 | encode));
6811 }
6812
6813 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6814 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6815 assert(VM_Version::supports_avx512bw(), "");
6816 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6817 attributes.set_is_evex_instruction();
6818 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6819 emit_int16(0x7B, (0xC0 | encode));
6820 }
6821
6822 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6823 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6824 assert(VM_Version::supports_evex(), "");
6825 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6826 attributes.set_is_evex_instruction();
6827 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6828 emit_int16(0x7C, (0xC0 | encode));
6829 }
6830
6831 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6832 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6833 assert(VM_Version::supports_evex(), "");
6834 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6835 attributes.set_is_evex_instruction();
6836 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6837 emit_int16(0x7C, (0xC0 | encode));
6838 }
6839 void Assembler::evpgatherdd(XMMRegister dst, KRegister mask, Address src, int vector_len) {
6840 assert(VM_Version::supports_evex(), "");
6841 assert(dst != xnoreg, "sanity");
6842 InstructionMark im(this);
6843 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6844 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6845 attributes.reset_is_clear_context();
6846 attributes.set_embedded_opmask_register_specifier(mask);
6847 attributes.set_is_evex_instruction();
6848 // swap src<->dst for encoding
6849 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6850 emit_int8((unsigned char)0x90);
6851 emit_operand(dst, src);
6852 }
6853 // Carry-Less Multiplication Quadword
6854 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6855 assert(VM_Version::supports_clmul(), "");
6856 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6857 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6858 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
6859 }
6860
6861 // Carry-Less Multiplication Quadword
6862 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6863 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6864 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
6865 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6866 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
6867 }
6868
6869 void Assembler::evpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask, int vector_len) {
6870 assert(VM_Version::supports_avx512_vpclmulqdq(), "Requires vector carryless multiplication support");
6871 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
6872 attributes.set_is_evex_instruction();
6873 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6874 emit_int24(0x44, (0xC0 | encode), (unsigned char)mask);
6875 }
6876
6877 void Assembler::vzeroupper_uncached() {
6878 if (VM_Version::supports_vzeroupper()) {
6879 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6880 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6881 emit_int8(0x77);
6882 }
6883 }
6884
6885 #ifndef _LP64
6886 // 32bit only pieces of the assembler
6887
6888 void Assembler::emms() {
6889 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
6890 emit_int16(0x0F, 0x77);
6891 }
6892
6893 void Assembler::vzeroupper() {
6894 vzeroupper_uncached();
6895 }
6896
6897 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6898 // NO PREFIX AS NEVER 64BIT
6899 InstructionMark im(this);
6900 emit_int16((unsigned char)0x81, (0xF8 | src1->encoding()));
6901 emit_data(imm32, rspec, 0);
6902 }
6903
6904 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6905 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6906 InstructionMark im(this);
6907 emit_int8((unsigned char)0x81);
6908 emit_operand(rdi, src1);
6909 emit_data(imm32, rspec, 0);
6910 }
6911
6912 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6913 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6914 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
6915 void Assembler::cmpxchg8(Address adr) {
6916 InstructionMark im(this);
6917 emit_int16(0x0F, (unsigned char)0xC7);
6918 emit_operand(rcx, adr);
6919 }
6920
6921 void Assembler::decl(Register dst) {
6922 // Don't use it directly. Use MacroAssembler::decrementl() instead.
6923 emit_int8(0x48 | dst->encoding());
6924 }
6925
6926 // 64bit doesn't use the x87
6927
6928 void Assembler::emit_operand32(Register reg, Address adr) {
6929 assert(reg->encoding() < 8, "no extended registers");
6930 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
6931 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
6932 adr._rspec);
6933 }
6934
6935 void Assembler::emit_farith(int b1, int b2, int i) {
6936 assert(isByte(b1) && isByte(b2), "wrong opcode");
6937 assert(0 <= i && i < 8, "illegal stack offset");
6938 emit_int16(b1, b2 + i);
6939 }
6940
6941 void Assembler::fabs() {
6942 emit_int16((unsigned char)0xD9, (unsigned char)0xE1);
6943 }
6944
6945 void Assembler::fadd(int i) {
6946 emit_farith(0xD8, 0xC0, i);
6947 }
6948
6949 void Assembler::fadd_d(Address src) {
6950 InstructionMark im(this);
6951 emit_int8((unsigned char)0xDC);
6952 emit_operand32(rax, src);
6953 }
6954
6955 void Assembler::fadd_s(Address src) {
6956 InstructionMark im(this);
6957 emit_int8((unsigned char)0xD8);
6958 emit_operand32(rax, src);
6959 }
6960
6961 void Assembler::fadda(int i) {
6962 emit_farith(0xDC, 0xC0, i);
6963 }
6964
6965 void Assembler::faddp(int i) {
6966 emit_farith(0xDE, 0xC0, i);
6967 }
6968
6969 void Assembler::fchs() {
6970 emit_int16((unsigned char)0xD9, (unsigned char)0xE0);
6971 }
6972
6973 void Assembler::fcom(int i) {
6974 emit_farith(0xD8, 0xD0, i);
6975 }
6976
6977 void Assembler::fcomp(int i) {
6978 emit_farith(0xD8, 0xD8, i);
6979 }
6980
6981 void Assembler::fcomp_d(Address src) {
6982 InstructionMark im(this);
6983 emit_int8((unsigned char)0xDC);
6984 emit_operand32(rbx, src);
6985 }
6986
6987 void Assembler::fcomp_s(Address src) {
6988 InstructionMark im(this);
6989 emit_int8((unsigned char)0xD8);
6990 emit_operand32(rbx, src);
6991 }
6992
6993 void Assembler::fcompp() {
6994 emit_int16((unsigned char)0xDE, (unsigned char)0xD9);
6995 }
6996
6997 void Assembler::fcos() {
6998 emit_int16((unsigned char)0xD9, (unsigned char)0xFF);
6999 }
7000
7001 void Assembler::fdecstp() {
7002 emit_int16((unsigned char)0xD9, (unsigned char)0xF6);
7003 }
7004
7005 void Assembler::fdiv(int i) {
7006 emit_farith(0xD8, 0xF0, i);
7007 }
7008
7009 void Assembler::fdiv_d(Address src) {
7010 InstructionMark im(this);
7011 emit_int8((unsigned char)0xDC);
7012 emit_operand32(rsi, src);
7013 }
7014
7015 void Assembler::fdiv_s(Address src) {
7016 InstructionMark im(this);
7017 emit_int8((unsigned char)0xD8);
7018 emit_operand32(rsi, src);
7019 }
7020
7021 void Assembler::fdiva(int i) {
7022 emit_farith(0xDC, 0xF8, i);
7023 }
7024
7025 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
7026 // is erroneous for some of the floating-point instructions below.
7027
7028 void Assembler::fdivp(int i) {
7029 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
7030 }
7031
7032 void Assembler::fdivr(int i) {
7033 emit_farith(0xD8, 0xF8, i);
7034 }
7035
7036 void Assembler::fdivr_d(Address src) {
7037 InstructionMark im(this);
7038 emit_int8((unsigned char)0xDC);
7039 emit_operand32(rdi, src);
7040 }
7041
7042 void Assembler::fdivr_s(Address src) {
7043 InstructionMark im(this);
7044 emit_int8((unsigned char)0xD8);
7045 emit_operand32(rdi, src);
7046 }
7047
7048 void Assembler::fdivra(int i) {
7049 emit_farith(0xDC, 0xF0, i);
7050 }
7051
7052 void Assembler::fdivrp(int i) {
7053 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
7054 }
7055
7056 void Assembler::ffree(int i) {
7057 emit_farith(0xDD, 0xC0, i);
7058 }
7059
7060 void Assembler::fild_d(Address adr) {
7061 InstructionMark im(this);
7062 emit_int8((unsigned char)0xDF);
7063 emit_operand32(rbp, adr);
7064 }
7065
7066 void Assembler::fild_s(Address adr) {
7067 InstructionMark im(this);
7068 emit_int8((unsigned char)0xDB);
7069 emit_operand32(rax, adr);
7070 }
7071
7072 void Assembler::fincstp() {
7073 emit_int16((unsigned char)0xD9, (unsigned char)0xF7);
7074 }
7075
7076 void Assembler::finit() {
7077 emit_int24((unsigned char)0x9B, (unsigned char)0xDB, (unsigned char)0xE3);
7078 }
7079
7080 void Assembler::fist_s(Address adr) {
7081 InstructionMark im(this);
7082 emit_int8((unsigned char)0xDB);
7083 emit_operand32(rdx, adr);
7084 }
7085
7086 void Assembler::fistp_d(Address adr) {
7087 InstructionMark im(this);
7088 emit_int8((unsigned char)0xDF);
7089 emit_operand32(rdi, adr);
7090 }
7091
7092 void Assembler::fistp_s(Address adr) {
7093 InstructionMark im(this);
7094 emit_int8((unsigned char)0xDB);
7095 emit_operand32(rbx, adr);
7096 }
7097
7098 void Assembler::fld1() {
7099 emit_int16((unsigned char)0xD9, (unsigned char)0xE8);
7100 }
7101
7102 void Assembler::fld_d(Address adr) {
7103 InstructionMark im(this);
7104 emit_int8((unsigned char)0xDD);
7105 emit_operand32(rax, adr);
7106 }
7107
7108 void Assembler::fld_s(Address adr) {
7109 InstructionMark im(this);
7110 emit_int8((unsigned char)0xD9);
7111 emit_operand32(rax, adr);
7112 }
7113
7114
7115 void Assembler::fld_s(int index) {
7116 emit_farith(0xD9, 0xC0, index);
7117 }
7118
7119 void Assembler::fld_x(Address adr) {
7120 InstructionMark im(this);
7121 emit_int8((unsigned char)0xDB);
7122 emit_operand32(rbp, adr);
7123 }
7124
7125 void Assembler::fldcw(Address src) {
7126 InstructionMark im(this);
7127 emit_int8((unsigned char)0xD9);
7128 emit_operand32(rbp, src);
7129 }
7130
7131 void Assembler::fldenv(Address src) {
7132 InstructionMark im(this);
7133 emit_int8((unsigned char)0xD9);
7134 emit_operand32(rsp, src);
7135 }
7136
7137 void Assembler::fldlg2() {
7138 emit_int16((unsigned char)0xD9, (unsigned char)0xEC);
7139 }
7140
7141 void Assembler::fldln2() {
7142 emit_int16((unsigned char)0xD9, (unsigned char)0xED);
7143 }
7144
7145 void Assembler::fldz() {
7146 emit_int16((unsigned char)0xD9, (unsigned char)0xEE);
7147 }
7148
7149 void Assembler::flog() {
7150 fldln2();
7151 fxch();
7152 fyl2x();
7153 }
7154
7155 void Assembler::flog10() {
7156 fldlg2();
7157 fxch();
7158 fyl2x();
7159 }
7160
7161 void Assembler::fmul(int i) {
7162 emit_farith(0xD8, 0xC8, i);
7163 }
7164
7165 void Assembler::fmul_d(Address src) {
7166 InstructionMark im(this);
7167 emit_int8((unsigned char)0xDC);
7168 emit_operand32(rcx, src);
7169 }
7170
7171 void Assembler::fmul_s(Address src) {
7172 InstructionMark im(this);
7173 emit_int8((unsigned char)0xD8);
7174 emit_operand32(rcx, src);
7175 }
7176
7177 void Assembler::fmula(int i) {
7178 emit_farith(0xDC, 0xC8, i);
7179 }
7180
7181 void Assembler::fmulp(int i) {
7182 emit_farith(0xDE, 0xC8, i);
7183 }
7184
7185 void Assembler::fnsave(Address dst) {
7186 InstructionMark im(this);
7187 emit_int8((unsigned char)0xDD);
7188 emit_operand32(rsi, dst);
7189 }
7190
7191 void Assembler::fnstcw(Address src) {
7192 InstructionMark im(this);
7193 emit_int16((unsigned char)0x9B, (unsigned char)0xD9);
7194 emit_operand32(rdi, src);
7195 }
7196
7197 void Assembler::fnstsw_ax() {
7198 emit_int16((unsigned char)0xDF, (unsigned char)0xE0);
7199 }
7200
7201 void Assembler::fprem() {
7202 emit_int16((unsigned char)0xD9, (unsigned char)0xF8);
7203 }
7204
7205 void Assembler::fprem1() {
7206 emit_int16((unsigned char)0xD9, (unsigned char)0xF5);
7207 }
7208
7209 void Assembler::frstor(Address src) {
7210 InstructionMark im(this);
7211 emit_int8((unsigned char)0xDD);
7212 emit_operand32(rsp, src);
7213 }
7214
7215 void Assembler::fsin() {
7216 emit_int16((unsigned char)0xD9, (unsigned char)0xFE);
7217 }
7218
7219 void Assembler::fsqrt() {
7220 emit_int16((unsigned char)0xD9, (unsigned char)0xFA);
7221 }
7222
7223 void Assembler::fst_d(Address adr) {
7224 InstructionMark im(this);
7225 emit_int8((unsigned char)0xDD);
7226 emit_operand32(rdx, adr);
7227 }
7228
7229 void Assembler::fst_s(Address adr) {
7230 InstructionMark im(this);
7231 emit_int8((unsigned char)0xD9);
7232 emit_operand32(rdx, adr);
7233 }
7234
7235 void Assembler::fstp_d(Address adr) {
7236 InstructionMark im(this);
7237 emit_int8((unsigned char)0xDD);
7238 emit_operand32(rbx, adr);
7239 }
7240
7241 void Assembler::fstp_d(int index) {
7242 emit_farith(0xDD, 0xD8, index);
7243 }
7244
7245 void Assembler::fstp_s(Address adr) {
7246 InstructionMark im(this);
7247 emit_int8((unsigned char)0xD9);
7248 emit_operand32(rbx, adr);
7249 }
7250
7251 void Assembler::fstp_x(Address adr) {
7252 InstructionMark im(this);
7253 emit_int8((unsigned char)0xDB);
7254 emit_operand32(rdi, adr);
7255 }
7256
7257 void Assembler::fsub(int i) {
7258 emit_farith(0xD8, 0xE0, i);
7259 }
7260
7261 void Assembler::fsub_d(Address src) {
7262 InstructionMark im(this);
7263 emit_int8((unsigned char)0xDC);
7264 emit_operand32(rsp, src);
7265 }
7266
7267 void Assembler::fsub_s(Address src) {
7268 InstructionMark im(this);
7269 emit_int8((unsigned char)0xD8);
7270 emit_operand32(rsp, src);
7271 }
7272
7273 void Assembler::fsuba(int i) {
7274 emit_farith(0xDC, 0xE8, i);
7275 }
7276
7277 void Assembler::fsubp(int i) {
7278 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7279 }
7280
7281 void Assembler::fsubr(int i) {
7282 emit_farith(0xD8, 0xE8, i);
7283 }
7284
7285 void Assembler::fsubr_d(Address src) {
7286 InstructionMark im(this);
7287 emit_int8((unsigned char)0xDC);
7288 emit_operand32(rbp, src);
7289 }
7290
7291 void Assembler::fsubr_s(Address src) {
7292 InstructionMark im(this);
7293 emit_int8((unsigned char)0xD8);
7294 emit_operand32(rbp, src);
7295 }
7296
7297 void Assembler::fsubra(int i) {
7298 emit_farith(0xDC, 0xE0, i);
7299 }
7300
7301 void Assembler::fsubrp(int i) {
7302 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7303 }
7304
7305 void Assembler::ftan() {
7306 emit_int32((unsigned char)0xD9, (unsigned char)0xF2, (unsigned char)0xDD, (unsigned char)0xD8);
7307 }
7308
7309 void Assembler::ftst() {
7310 emit_int16((unsigned char)0xD9, (unsigned char)0xE4);
7311 }
7312
7313 void Assembler::fucomi(int i) {
7314 // make sure the instruction is supported (introduced for P6, together with cmov)
7315 guarantee(VM_Version::supports_cmov(), "illegal instruction");
7316 emit_farith(0xDB, 0xE8, i);
7317 }
7318
7319 void Assembler::fucomip(int i) {
7320 // make sure the instruction is supported (introduced for P6, together with cmov)
7321 guarantee(VM_Version::supports_cmov(), "illegal instruction");
7322 emit_farith(0xDF, 0xE8, i);
7323 }
7324
7325 void Assembler::fwait() {
7326 emit_int8((unsigned char)0x9B);
7327 }
7328
7329 void Assembler::fxch(int i) {
7330 emit_farith(0xD9, 0xC8, i);
7331 }
7332
7333 void Assembler::fyl2x() {
7334 emit_int16((unsigned char)0xD9, (unsigned char)0xF1);
7335 }
7336
7337 void Assembler::frndint() {
7338 emit_int16((unsigned char)0xD9, (unsigned char)0xFC);
7339 }
7340
7341 void Assembler::f2xm1() {
7342 emit_int16((unsigned char)0xD9, (unsigned char)0xF0);
7343 }
7344
7345 void Assembler::fldl2e() {
7346 emit_int16((unsigned char)0xD9, (unsigned char)0xEA);
7347 }
7348 #endif // !_LP64
7349
7350 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7351 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7352 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7353 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
7354
7355 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7356 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7357 if (pre > 0) {
7358 emit_int8(simd_pre[pre]);
7359 }
7360 if (rex_w) {
7361 prefixq(adr, xreg);
7362 } else {
7363 prefix(adr, xreg);
7364 }
7365 if (opc > 0) {
7366 emit_int8(0x0F);
7367 int opc2 = simd_opc[opc];
7368 if (opc2 > 0) {
7369 emit_int8(opc2);
7370 }
7371 }
7372 }
7373
7374 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7375 if (pre > 0) {
7376 emit_int8(simd_pre[pre]);
7377 }
7378 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7379 if (opc > 0) {
7380 emit_int8(0x0F);
7381 int opc2 = simd_opc[opc];
7382 if (opc2 > 0) {
7383 emit_int8(opc2);
7384 }
7385 }
7386 return encode;
7387 }
7388
7389
7390 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7391 int vector_len = _attributes->get_vector_len();
7392 bool vex_w = _attributes->is_rex_vex_w();
7393 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7394 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7395 byte1 = (~byte1) & 0xE0;
7396 byte1 |= opc;
7397
7398 int byte2 = ((~nds_enc) & 0xf) << 3;
7399 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7400
7401 emit_int24((unsigned char)VEX_3bytes, byte1, byte2);
7402 } else {
7403 int byte1 = vex_r ? VEX_R : 0;
7404 byte1 = (~byte1) & 0x80;
7405 byte1 |= ((~nds_enc) & 0xf) << 3;
7406 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7407 emit_int16((unsigned char)VEX_2bytes, byte1);
7408 }
7409 }
7410
7411 // This is a 4 byte encoding
7412 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7413 // EVEX 0x62 prefix
7414 // byte1 = EVEX_4bytes;
7415
7416 bool vex_w = _attributes->is_rex_vex_w();
7417 int evex_encoding = (vex_w ? VEX_W : 0);
7418 // EVEX.b is not currently used for broadcast of single element or data rounding modes
7419 _attributes->set_evex_encoding(evex_encoding);
7420
7421 // P0: byte 2, initialized to RXBR`00mm
7422 // instead of not'd
7423 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7424 byte2 = (~byte2) & 0xF0;
7425 // confine opc opcode extensions in mm bits to lower two bits
7426 // of form {0F, 0F_38, 0F_3A}
7427 byte2 |= opc;
7428
7429 // P1: byte 3 as Wvvvv1pp
7430 int byte3 = ((~nds_enc) & 0xf) << 3;
7431 // p[10] is always 1
7432 byte3 |= EVEX_F;
7433 byte3 |= (vex_w & 1) << 7;
7434 // confine pre opcode extensions in pp bits to lower two bits
7435 // of form {66, F3, F2}
7436 byte3 |= pre;
7437
7438 // P2: byte 4 as zL'Lbv'aaa
7439 // kregs are implemented in the low 3 bits as aaa
7440 int byte4 = (_attributes->is_no_reg_mask()) ?
7441 0 :
7442 _attributes->get_embedded_opmask_register_specifier();
7443 // EVEX.v` for extending EVEX.vvvv or VIDX
7444 byte4 |= (evex_v ? 0: EVEX_V);
7445 // third EXEC.b for broadcast actions
7446 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7447 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7448 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7449 // last is EVEX.z for zero/merge actions
7450 if (_attributes->is_no_reg_mask() == false) {
7451 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7452 }
7453
7454 emit_int32(EVEX_4bytes, byte2, byte3, byte4);
7455 }
7456
7457 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7458 bool vex_r = (xreg_enc & 8) == 8;
7459 bool vex_b = adr.base_needs_rex();
7460 bool vex_x;
7461 if (adr.isxmmindex()) {
7462 vex_x = adr.xmmindex_needs_rex();
7463 } else {
7464 vex_x = adr.index_needs_rex();
7465 }
7466 set_attributes(attributes);
7467 attributes->set_current_assembler(this);
7468
7469 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
7470 // is allowed in legacy mode and has resources which will fit in it.
7471 // Pure EVEX instructions will have is_evex_instruction set in their definition.
7472 if (!attributes->is_legacy_mode()) {
7473 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
7474 if ((attributes->get_vector_len() != AVX_512bit) && (nds_enc < 16) && (xreg_enc < 16)) {
7475 attributes->set_is_legacy_mode();
7476 }
7477 }
7478 }
7479
7480 if (UseAVX > 2) {
7481 assert(((!attributes->uses_vl()) ||
7482 (attributes->get_vector_len() == AVX_512bit) ||
7483 (!_legacy_mode_vl) ||
7484 (attributes->is_legacy_mode())),"XMM register should be 0-15");
7485 assert(((nds_enc < 16 && xreg_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
7486 }
7487
7488 clear_managed();
7489 if (UseAVX > 2 && !attributes->is_legacy_mode())
7490 {
7491 bool evex_r = (xreg_enc >= 16);
7492 bool evex_v;
7493 // EVEX.V' is set to true when VSIB is used as we may need to use higher order XMM registers (16-31)
7494 if (adr.isxmmindex()) {
7495 evex_v = ((adr._xmmindex->encoding() > 15) ? true : false);
7496 } else {
7497 evex_v = (nds_enc >= 16);
7498 }
7499 attributes->set_is_evex_instruction();
7500 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7501 } else {
7502 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7503 attributes->set_rex_vex_w(false);
7504 }
7505 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7506 }
7507 }
7508
7509 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7510 bool vex_r = (dst_enc & 8) == 8;
7511 bool vex_b = (src_enc & 8) == 8;
7512 bool vex_x = false;
7513 set_attributes(attributes);
7514 attributes->set_current_assembler(this);
7515
7516 // For EVEX instruction (which is not marked as pure EVEX instruction) check and see if this instruction
7517 // is allowed in legacy mode and has resources which will fit in it.
7518 // Pure EVEX instructions will have is_evex_instruction set in their definition.
7519 if (!attributes->is_legacy_mode()) {
7520 if (UseAVX > 2 && !attributes->is_evex_instruction() && !is_managed()) {
7521 if ((!attributes->uses_vl() || (attributes->get_vector_len() != AVX_512bit)) &&
7522 (dst_enc < 16) && (nds_enc < 16) && (src_enc < 16)) {
7523 attributes->set_is_legacy_mode();
7524 }
7525 }
7526 }
7527
7528 if (UseAVX > 2) {
7529 // All the scalar fp instructions (with uses_vl as false) can have legacy_mode as false
7530 // Instruction with uses_vl true are vector instructions
7531 // All the vector instructions with AVX_512bit length can have legacy_mode as false
7532 // All the vector instructions with < AVX_512bit length can have legacy_mode as false if AVX512vl() is supported
7533 // Rest all should have legacy_mode set as true
7534 assert(((!attributes->uses_vl()) ||
7535 (attributes->get_vector_len() == AVX_512bit) ||
7536 (!_legacy_mode_vl) ||
7537 (attributes->is_legacy_mode())),"XMM register should be 0-15");
7538 // Instruction with legacy_mode true should have dst, nds and src < 15
7539 assert(((dst_enc < 16 && nds_enc < 16 && src_enc < 16) || (!attributes->is_legacy_mode())),"XMM register should be 0-15");
7540 }
7541
7542 clear_managed();
7543 if (UseAVX > 2 && !attributes->is_legacy_mode())
7544 {
7545 bool evex_r = (dst_enc >= 16);
7546 bool evex_v = (nds_enc >= 16);
7547 // can use vex_x as bank extender on rm encoding
7548 vex_x = (src_enc >= 16);
7549 attributes->set_is_evex_instruction();
7550 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7551 } else {
7552 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7553 attributes->set_rex_vex_w(false);
7554 }
7555 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7556 }
7557
7558 // return modrm byte components for operands
7559 return (((dst_enc & 7) << 3) | (src_enc & 7));
7560 }
7561
7562
7563 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7564 VexOpcode opc, InstructionAttr *attributes) {
7565 if (UseAVX > 0) {
7566 int xreg_enc = xreg->encoding();
7567 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7568 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7569 } else {
7570 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7571 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7572 }
7573 }
7574
7575 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7576 VexOpcode opc, InstructionAttr *attributes) {
7577 int dst_enc = dst->encoding();
7578 int src_enc = src->encoding();
7579 if (UseAVX > 0) {
7580 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7581 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7582 } else {
7583 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7584 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7585 }
7586 }
7587
7588 void Assembler::vmaxss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7589 assert(VM_Version::supports_avx(), "");
7590 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7591 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7592 emit_int16(0x5F, (0xC0 | encode));
7593 }
7594
7595 void Assembler::vmaxsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7596 assert(VM_Version::supports_avx(), "");
7597 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7598 attributes.set_rex_vex_w_reverted();
7599 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7600 emit_int16(0x5F, (0xC0 | encode));
7601 }
7602
7603 void Assembler::vminss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7604 assert(VM_Version::supports_avx(), "");
7605 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7606 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7607 emit_int16(0x5D, (0xC0 | encode));
7608 }
7609
7610 void Assembler::vminsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
7611 assert(VM_Version::supports_avx(), "");
7612 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7613 attributes.set_rex_vex_w_reverted();
7614 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7615 emit_int16(0x5D, (0xC0 | encode));
7616 }
7617
7618 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7619 assert(VM_Version::supports_avx(), "");
7620 assert(vector_len <= AVX_256bit, "");
7621 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7622 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7623 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
7624 }
7625
7626 void Assembler::blendvpb(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7627 assert(VM_Version::supports_avx(), "");
7628 assert(vector_len <= AVX_256bit, "");
7629 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7630 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7631 int src2_enc = src2->encoding();
7632 emit_int24(0x4C, (0xC0 | encode), (0xF0 & src2_enc << 4));
7633 }
7634
7635 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7636 assert(VM_Version::supports_avx(), "");
7637 assert(vector_len <= AVX_256bit, "");
7638 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7639 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7640 int src2_enc = src2->encoding();
7641 emit_int24(0x4B, (0xC0 | encode), (0xF0 & src2_enc << 4));
7642 }
7643
7644 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7645 assert(VM_Version::supports_avx(), "");
7646 assert(vector_len <= AVX_256bit, "");
7647 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7648 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7649 emit_int24((unsigned char)0xC2, (0xC0 | encode), (0xF & cop));
7650 }
7651
7652 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7653 assert(VM_Version::supports_avx(), "");
7654 assert(vector_len <= AVX_256bit, "");
7655 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7656 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7657 int src2_enc = src2->encoding();
7658 emit_int24(0x4A, (0xC0 | encode), (0xF0 & src2_enc << 4));
7659 }
7660
7661 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7662 assert(VM_Version::supports_avx2(), "");
7663 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7664 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7665 emit_int24(0x02, (0xC0 | encode), (unsigned char)imm8);
7666 }
7667
7668 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7669 assert(VM_Version::supports_bmi2(), "");
7670 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7671 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7672 emit_int16((unsigned char)0xF7, (0xC0 | encode));
7673 }
7674
7675 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7676 assert(VM_Version::supports_bmi2(), "");
7677 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
7678 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7679 emit_int16((unsigned char)0xF7, (0xC0 | encode));
7680 }
7681
7682 #ifndef _LP64
7683
7684 void Assembler::incl(Register dst) {
7685 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7686 emit_int8(0x40 | dst->encoding());
7687 }
7688
7689 void Assembler::lea(Register dst, Address src) {
7690 leal(dst, src);
7691 }
7692
7693 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7694 InstructionMark im(this);
7695 emit_int8((unsigned char)0xC7);
7696 emit_operand(rax, dst);
7697 emit_data((int)imm32, rspec, 0);
7698 }
7699
7700 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7701 InstructionMark im(this);
7702 int encode = prefix_and_encode(dst->encoding());
7703 emit_int8((0xB8 | encode));
7704 emit_data((int)imm32, rspec, 0);
7705 }
7706
7707 void Assembler::popa() { // 32bit
7708 emit_int8(0x61);
7709 }
7710
7711 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7712 InstructionMark im(this);
7713 emit_int8(0x68);
7714 emit_data(imm32, rspec, 0);
7715 }
7716
7717 void Assembler::pusha() { // 32bit
7718 emit_int8(0x60);
7719 }
7720
7721 void Assembler::set_byte_if_not_zero(Register dst) {
7722 emit_int24(0x0F, (unsigned char)0x95, (0xE0 | dst->encoding()));
7723 }
7724
7725 #else // LP64
7726
7727 void Assembler::set_byte_if_not_zero(Register dst) {
7728 int enc = prefix_and_encode(dst->encoding(), true);
7729 emit_int24(0x0F, (unsigned char)0x95, (0xE0 | enc));
7730 }
7731
7732 // 64bit only pieces of the assembler
7733 // This should only be used by 64bit instructions that can use rip-relative
7734 // it cannot be used by instructions that want an immediate value.
7735
7736 bool Assembler::reachable(AddressLiteral adr) {
7737 int64_t disp;
7738 relocInfo::relocType relocType = adr.reloc();
7739
7740 // None will force a 64bit literal to the code stream. Likely a placeholder
7741 // for something that will be patched later and we need to certain it will
7742 // always be reachable.
7743 if (relocType == relocInfo::none) {
7744 return false;
7745 }
7746 if (relocType == relocInfo::internal_word_type) {
7747 // This should be rip relative and easily reachable.
7748 return true;
7749 }
7750 if (relocType == relocInfo::virtual_call_type ||
7751 relocType == relocInfo::opt_virtual_call_type ||
7752 relocType == relocInfo::static_call_type ||
7753 relocType == relocInfo::static_stub_type ) {
7754 // This should be rip relative within the code cache and easily
7755 // reachable until we get huge code caches. (At which point
7756 // ic code is going to have issues).
7757 return true;
7758 }
7759 if (relocType != relocInfo::external_word_type &&
7760 relocType != relocInfo::poll_return_type && // these are really external_word but need special
7761 relocType != relocInfo::poll_type && // relocs to identify them
7762 relocType != relocInfo::runtime_call_type ) {
7763 return false;
7764 }
7765
7766 // Stress the correction code
7767 if (ForceUnreachable) {
7768 // Must be runtimecall reloc, see if it is in the codecache
7769 // Flipping stuff in the codecache to be unreachable causes issues
7770 // with things like inline caches where the additional instructions
7771 // are not handled.
7772 if (CodeCache::find_blob(adr._target) == NULL) {
7773 return false;
7774 }
7775 }
7776 // For external_word_type/runtime_call_type if it is reachable from where we
7777 // are now (possibly a temp buffer) and where we might end up
7778 // anywhere in the codeCache then we are always reachable.
7779 // This would have to change if we ever save/restore shared code
7780 // to be more pessimistic.
7781 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7782 if (!is_simm32(disp)) return false;
7783 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7784 if (!is_simm32(disp)) return false;
7785
7786 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7787
7788 // Because rip relative is a disp + address_of_next_instruction and we
7789 // don't know the value of address_of_next_instruction we apply a fudge factor
7790 // to make sure we will be ok no matter the size of the instruction we get placed into.
7791 // We don't have to fudge the checks above here because they are already worst case.
7792
7793 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7794 // + 4 because better safe than sorry.
7795 const int fudge = 12 + 4;
7796 if (disp < 0) {
7797 disp -= fudge;
7798 } else {
7799 disp += fudge;
7800 }
7801 return is_simm32(disp);
7802 }
7803
7804 void Assembler::emit_data64(jlong data,
7805 relocInfo::relocType rtype,
7806 int format) {
7807 if (rtype == relocInfo::none) {
7808 emit_int64(data);
7809 } else {
7810 emit_data64(data, Relocation::spec_simple(rtype), format);
7811 }
7812 }
7813
7814 void Assembler::emit_data64(jlong data,
7815 RelocationHolder const& rspec,
7816 int format) {
7817 assert(imm_operand == 0, "default format must be immediate in this file");
7818 assert(imm_operand == format, "must be immediate");
7819 assert(inst_mark() != NULL, "must be inside InstructionMark");
7820 // Do not use AbstractAssembler::relocate, which is not intended for
7821 // embedded words. Instead, relocate to the enclosing instruction.
7822 code_section()->relocate(inst_mark(), rspec, format);
7823 #ifdef ASSERT
7824 check_relocation(rspec, format);
7825 #endif
7826 emit_int64(data);
7827 }
7828
7829 void Assembler::prefix(Register reg) {
7830 if (reg->encoding() >= 8) {
7831 prefix(REX_B);
7832 }
7833 }
7834
7835 void Assembler::prefix(Register dst, Register src, Prefix p) {
7836 if (src->encoding() >= 8) {
7837 p = (Prefix)(p | REX_B);
7838 }
7839 if (dst->encoding() >= 8) {
7840 p = (Prefix)(p | REX_R);
7841 }
7842 if (p != Prefix_EMPTY) {
7843 // do not generate an empty prefix
7844 prefix(p);
7845 }
7846 }
7847
7848 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7849 if (adr.base_needs_rex()) {
7850 if (adr.index_needs_rex()) {
7851 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7852 } else {
7853 prefix(REX_B);
7854 }
7855 } else {
7856 if (adr.index_needs_rex()) {
7857 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7858 }
7859 }
7860 if (dst->encoding() >= 8) {
7861 p = (Prefix)(p | REX_R);
7862 }
7863 if (p != Prefix_EMPTY) {
7864 // do not generate an empty prefix
7865 prefix(p);
7866 }
7867 }
7868
7869 void Assembler::prefix(Address adr) {
7870 if (adr.base_needs_rex()) {
7871 if (adr.index_needs_rex()) {
7872 prefix(REX_XB);
7873 } else {
7874 prefix(REX_B);
7875 }
7876 } else {
7877 if (adr.index_needs_rex()) {
7878 prefix(REX_X);
7879 }
7880 }
7881 }
7882
7883 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7884 if (reg->encoding() < 8) {
7885 if (adr.base_needs_rex()) {
7886 if (adr.index_needs_rex()) {
7887 prefix(REX_XB);
7888 } else {
7889 prefix(REX_B);
7890 }
7891 } else {
7892 if (adr.index_needs_rex()) {
7893 prefix(REX_X);
7894 } else if (byteinst && reg->encoding() >= 4) {
7895 prefix(REX);
7896 }
7897 }
7898 } else {
7899 if (adr.base_needs_rex()) {
7900 if (adr.index_needs_rex()) {
7901 prefix(REX_RXB);
7902 } else {
7903 prefix(REX_RB);
7904 }
7905 } else {
7906 if (adr.index_needs_rex()) {
7907 prefix(REX_RX);
7908 } else {
7909 prefix(REX_R);
7910 }
7911 }
7912 }
7913 }
7914
7915 void Assembler::prefix(Address adr, XMMRegister reg) {
7916 if (reg->encoding() < 8) {
7917 if (adr.base_needs_rex()) {
7918 if (adr.index_needs_rex()) {
7919 prefix(REX_XB);
7920 } else {
7921 prefix(REX_B);
7922 }
7923 } else {
7924 if (adr.index_needs_rex()) {
7925 prefix(REX_X);
7926 }
7927 }
7928 } else {
7929 if (adr.base_needs_rex()) {
7930 if (adr.index_needs_rex()) {
7931 prefix(REX_RXB);
7932 } else {
7933 prefix(REX_RB);
7934 }
7935 } else {
7936 if (adr.index_needs_rex()) {
7937 prefix(REX_RX);
7938 } else {
7939 prefix(REX_R);
7940 }
7941 }
7942 }
7943 }
7944
7945 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7946 if (reg_enc >= 8) {
7947 prefix(REX_B);
7948 reg_enc -= 8;
7949 } else if (byteinst && reg_enc >= 4) {
7950 prefix(REX);
7951 }
7952 return reg_enc;
7953 }
7954
7955 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7956 if (dst_enc < 8) {
7957 if (src_enc >= 8) {
7958 prefix(REX_B);
7959 src_enc -= 8;
7960 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7961 prefix(REX);
7962 }
7963 } else {
7964 if (src_enc < 8) {
7965 prefix(REX_R);
7966 } else {
7967 prefix(REX_RB);
7968 src_enc -= 8;
7969 }
7970 dst_enc -= 8;
7971 }
7972 return dst_enc << 3 | src_enc;
7973 }
7974
7975 int8_t Assembler::get_prefixq(Address adr) {
7976 int8_t prfx = get_prefixq(adr, rax);
7977 assert(REX_W <= prfx && prfx <= REX_WXB, "must be");
7978 return prfx;
7979 }
7980
7981 int8_t Assembler::get_prefixq(Address adr, Register src) {
7982 int8_t prfx = (int8_t)(REX_W +
7983 ((int)adr.base_needs_rex()) +
7984 ((int)adr.index_needs_rex() << 1) +
7985 ((int)(src->encoding() >= 8) << 2));
7986 #ifdef ASSERT
7987 if (src->encoding() < 8) {
7988 if (adr.base_needs_rex()) {
7989 if (adr.index_needs_rex()) {
7990 assert(prfx == REX_WXB, "must be");
7991 } else {
7992 assert(prfx == REX_WB, "must be");
7993 }
7994 } else {
7995 if (adr.index_needs_rex()) {
7996 assert(prfx == REX_WX, "must be");
7997 } else {
7998 assert(prfx == REX_W, "must be");
7999 }
8000 }
8001 } else {
8002 if (adr.base_needs_rex()) {
8003 if (adr.index_needs_rex()) {
8004 assert(prfx == REX_WRXB, "must be");
8005 } else {
8006 assert(prfx == REX_WRB, "must be");
8007 }
8008 } else {
8009 if (adr.index_needs_rex()) {
8010 assert(prfx == REX_WRX, "must be");
8011 } else {
8012 assert(prfx == REX_WR, "must be");
8013 }
8014 }
8015 }
8016 #endif
8017 return prfx;
8018 }
8019
8020 void Assembler::prefixq(Address adr) {
8021 emit_int8(get_prefixq(adr));
8022 }
8023
8024 void Assembler::prefixq(Address adr, Register src) {
8025 emit_int8(get_prefixq(adr, src));
8026 }
8027
8028 void Assembler::prefixq(Address adr, XMMRegister src) {
8029 if (src->encoding() < 8) {
8030 if (adr.base_needs_rex()) {
8031 if (adr.index_needs_rex()) {
8032 prefix(REX_WXB);
8033 } else {
8034 prefix(REX_WB);
8035 }
8036 } else {
8037 if (adr.index_needs_rex()) {
8038 prefix(REX_WX);
8039 } else {
8040 prefix(REX_W);
8041 }
8042 }
8043 } else {
8044 if (adr.base_needs_rex()) {
8045 if (adr.index_needs_rex()) {
8046 prefix(REX_WRXB);
8047 } else {
8048 prefix(REX_WRB);
8049 }
8050 } else {
8051 if (adr.index_needs_rex()) {
8052 prefix(REX_WRX);
8053 } else {
8054 prefix(REX_WR);
8055 }
8056 }
8057 }
8058 }
8059
8060 int Assembler::prefixq_and_encode(int reg_enc) {
8061 if (reg_enc < 8) {
8062 prefix(REX_W);
8063 } else {
8064 prefix(REX_WB);
8065 reg_enc -= 8;
8066 }
8067 return reg_enc;
8068 }
8069
8070 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
8071 if (dst_enc < 8) {
8072 if (src_enc < 8) {
8073 prefix(REX_W);
8074 } else {
8075 prefix(REX_WB);
8076 src_enc -= 8;
8077 }
8078 } else {
8079 if (src_enc < 8) {
8080 prefix(REX_WR);
8081 } else {
8082 prefix(REX_WRB);
8083 src_enc -= 8;
8084 }
8085 dst_enc -= 8;
8086 }
8087 return dst_enc << 3 | src_enc;
8088 }
8089
8090 void Assembler::adcq(Register dst, int32_t imm32) {
8091 (void) prefixq_and_encode(dst->encoding());
8092 emit_arith(0x81, 0xD0, dst, imm32);
8093 }
8094
8095 void Assembler::adcq(Register dst, Address src) {
8096 InstructionMark im(this);
8097 emit_int16(get_prefixq(src, dst), 0x13);
8098 emit_operand(dst, src);
8099 }
8100
8101 void Assembler::adcq(Register dst, Register src) {
8102 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8103 emit_arith(0x13, 0xC0, dst, src);
8104 }
8105
8106 void Assembler::addq(Address dst, int32_t imm32) {
8107 InstructionMark im(this);
8108 prefixq(dst);
8109 emit_arith_operand(0x81, rax, dst, imm32);
8110 }
8111
8112 void Assembler::addq(Address dst, Register src) {
8113 InstructionMark im(this);
8114 emit_int16(get_prefixq(dst, src), 0x01);
8115 emit_operand(src, dst);
8116 }
8117
8118 void Assembler::addq(Register dst, int32_t imm32) {
8119 (void) prefixq_and_encode(dst->encoding());
8120 emit_arith(0x81, 0xC0, dst, imm32);
8121 }
8122
8123 void Assembler::addq(Register dst, Address src) {
8124 InstructionMark im(this);
8125 emit_int16(get_prefixq(src, dst), 0x03);
8126 emit_operand(dst, src);
8127 }
8128
8129 void Assembler::addq(Register dst, Register src) {
8130 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8131 emit_arith(0x03, 0xC0, dst, src);
8132 }
8133
8134 void Assembler::adcxq(Register dst, Register src) {
8135 //assert(VM_Version::supports_adx(), "adx instructions not supported");
8136 emit_int8(0x66);
8137 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8138 emit_int32(0x0F,
8139 0x38,
8140 (unsigned char)0xF6,
8141 (0xC0 | encode));
8142 }
8143
8144 void Assembler::adoxq(Register dst, Register src) {
8145 //assert(VM_Version::supports_adx(), "adx instructions not supported");
8146 emit_int8((unsigned char)0xF3);
8147 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8148 emit_int32(0x0F,
8149 0x38,
8150 (unsigned char)0xF6,
8151 (0xC0 | encode));
8152 }
8153
8154 void Assembler::andq(Address dst, int32_t imm32) {
8155 InstructionMark im(this);
8156 emit_int16(get_prefixq(dst), (unsigned char)0x81);
8157 emit_operand(rsp, dst, 4);
8158 emit_int32(imm32);
8159 }
8160
8161 void Assembler::andq(Register dst, int32_t imm32) {
8162 (void) prefixq_and_encode(dst->encoding());
8163 emit_arith(0x81, 0xE0, dst, imm32);
8164 }
8165
8166 void Assembler::andq(Register dst, Address src) {
8167 InstructionMark im(this);
8168 emit_int16(get_prefixq(src, dst), 0x23);
8169 emit_operand(dst, src);
8170 }
8171
8172 void Assembler::andq(Register dst, Register src) {
8173 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8174 emit_arith(0x23, 0xC0, dst, src);
8175 }
8176
8177 void Assembler::andnq(Register dst, Register src1, Register src2) {
8178 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8179 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8180 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8181 emit_int16((unsigned char)0xF2, (0xC0 | encode));
8182 }
8183
8184 void Assembler::andnq(Register dst, Register src1, Address src2) {
8185 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8186 InstructionMark im(this);
8187 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8188 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8189 emit_int8((unsigned char)0xF2);
8190 emit_operand(dst, src2);
8191 }
8192
8193 void Assembler::bsfq(Register dst, Register src) {
8194 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8195 emit_int24(0x0F, (unsigned char)0xBC, (0xC0 | encode));
8196 }
8197
8198 void Assembler::bsrq(Register dst, Register src) {
8199 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8200 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
8201 }
8202
8203 void Assembler::bswapq(Register reg) {
8204 int encode = prefixq_and_encode(reg->encoding());
8205 emit_int16(0x0F, (0xC8 | encode));
8206 }
8207
8208 void Assembler::blsiq(Register dst, Register src) {
8209 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8210 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8211 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8212 emit_int16((unsigned char)0xF3, (0xC0 | encode));
8213 }
8214
8215 void Assembler::blsiq(Register dst, Address src) {
8216 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8217 InstructionMark im(this);
8218 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8219 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8220 emit_int8((unsigned char)0xF3);
8221 emit_operand(rbx, src);
8222 }
8223
8224 void Assembler::blsmskq(Register dst, Register src) {
8225 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8226 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8227 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8228 emit_int16((unsigned char)0xF3, (0xC0 | encode));
8229 }
8230
8231 void Assembler::blsmskq(Register dst, Address src) {
8232 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8233 InstructionMark im(this);
8234 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8235 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8236 emit_int8((unsigned char)0xF3);
8237 emit_operand(rdx, src);
8238 }
8239
8240 void Assembler::blsrq(Register dst, Register src) {
8241 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8242 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8243 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8244 emit_int16((unsigned char)0xF3, (0xC0 | encode));
8245 }
8246
8247 void Assembler::blsrq(Register dst, Address src) {
8248 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8249 InstructionMark im(this);
8250 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8251 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8252 emit_int8((unsigned char)0xF3);
8253 emit_operand(rcx, src);
8254 }
8255
8256 void Assembler::cdqq() {
8257 emit_int16(REX_W, (unsigned char)0x99);
8258 }
8259
8260 void Assembler::clflush(Address adr) {
8261 assert(VM_Version::supports_clflush(), "should do");
8262 prefix(adr);
8263 emit_int16(0x0F, (unsigned char)0xAE);
8264 emit_operand(rdi, adr);
8265 }
8266
8267 void Assembler::clflushopt(Address adr) {
8268 assert(VM_Version::supports_clflushopt(), "should do!");
8269 // adr should be base reg only with no index or offset
8270 assert(adr.index() == noreg, "index should be noreg");
8271 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
8272 assert(adr.disp() == 0, "displacement should be 0");
8273 // instruction prefix is 0x66
8274 emit_int8(0x66);
8275 prefix(adr);
8276 // opcode family is 0x0F 0xAE
8277 emit_int16(0x0F, (unsigned char)0xAE);
8278 // extended opcode byte is 7 == rdi
8279 emit_operand(rdi, adr);
8280 }
8281
8282 void Assembler::clwb(Address adr) {
8283 assert(VM_Version::supports_clwb(), "should do!");
8284 // adr should be base reg only with no index or offset
8285 assert(adr.index() == noreg, "index should be noreg");
8286 assert(adr.scale() == Address::no_scale, "scale should be no_scale");
8287 assert(adr.disp() == 0, "displacement should be 0");
8288 // instruction prefix is 0x66
8289 emit_int8(0x66);
8290 prefix(adr);
8291 // opcode family is 0x0f 0xAE
8292 emit_int16(0x0F, (unsigned char)0xAE);
8293 // extended opcode byte is 6 == rsi
8294 emit_operand(rsi, adr);
8295 }
8296
8297 void Assembler::cmovq(Condition cc, Register dst, Register src) {
8298 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8299 emit_int24(0x0F, (0x40 | cc), (0xC0 | encode));
8300 }
8301
8302 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8303 InstructionMark im(this);
8304 emit_int24(get_prefixq(src, dst), 0x0F, (0x40 | cc));
8305 emit_operand(dst, src);
8306 }
8307
8308 void Assembler::cmpq(Address dst, int32_t imm32) {
8309 InstructionMark im(this);
8310 emit_int16(get_prefixq(dst), (unsigned char)0x81);
8311 emit_operand(rdi, dst, 4);
8312 emit_int32(imm32);
8313 }
8314
8315 void Assembler::cmpq(Register dst, int32_t imm32) {
8316 (void) prefixq_and_encode(dst->encoding());
8317 emit_arith(0x81, 0xF8, dst, imm32);
8318 }
8319
8320 void Assembler::cmpq(Address dst, Register src) {
8321 InstructionMark im(this);
8322 emit_int16(get_prefixq(dst, src), 0x3B);
8323 emit_operand(src, dst);
8324 }
8325
8326 void Assembler::cmpq(Register dst, Register src) {
8327 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8328 emit_arith(0x3B, 0xC0, dst, src);
8329 }
8330
8331 void Assembler::cmpq(Register dst, Address src) {
8332 InstructionMark im(this);
8333 emit_int16(get_prefixq(src, dst), 0x3B);
8334 emit_operand(dst, src);
8335 }
8336
8337 void Assembler::cmpxchgq(Register reg, Address adr) {
8338 InstructionMark im(this);
8339 emit_int24(get_prefixq(adr, reg), 0x0F, (unsigned char)0xB1);
8340 emit_operand(reg, adr);
8341 }
8342
8343 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8344 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8345 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8346 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8347 emit_int16(0x2A, (0xC0 | encode));
8348 }
8349
8350 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8351 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8352 InstructionMark im(this);
8353 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8354 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8355 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8356 emit_int8(0x2A);
8357 emit_operand(dst, src);
8358 }
8359
8360 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8361 NOT_LP64(assert(VM_Version::supports_sse(), ""));
8362 InstructionMark im(this);
8363 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8364 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8365 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8366 emit_int8(0x2A);
8367 emit_operand(dst, src);
8368 }
8369
8370 void Assembler::cvttsd2siq(Register dst, Address src) {
8371 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8372 // F2 REX.W 0F 2C /r
8373 // CVTTSD2SI r64, xmm1/m64
8374 InstructionMark im(this);
8375 emit_int32((unsigned char)0xF2, REX_W, 0x0F, 0x2C);
8376 emit_operand(dst, src);
8377 }
8378
8379 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8380 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8381 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8382 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8383 emit_int16(0x2C, (0xC0 | encode));
8384 }
8385
8386 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8387 NOT_LP64(assert(VM_Version::supports_sse(), ""));
8388 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8389 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8390 emit_int16(0x2C, (0xC0 | encode));
8391 }
8392
8393 void Assembler::decl(Register dst) {
8394 // Don't use it directly. Use MacroAssembler::decrementl() instead.
8395 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8396 int encode = prefix_and_encode(dst->encoding());
8397 emit_int16((unsigned char)0xFF, (0xC8 | encode));
8398 }
8399
8400 void Assembler::decq(Register dst) {
8401 // Don't use it directly. Use MacroAssembler::decrementq() instead.
8402 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8403 int encode = prefixq_and_encode(dst->encoding());
8404 emit_int16((unsigned char)0xFF, 0xC8 | encode);
8405 }
8406
8407 void Assembler::decq(Address dst) {
8408 // Don't use it directly. Use MacroAssembler::decrementq() instead.
8409 InstructionMark im(this);
8410 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
8411 emit_operand(rcx, dst);
8412 }
8413
8414 void Assembler::fxrstor(Address src) {
8415 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
8416 emit_operand(as_Register(1), src);
8417 }
8418
8419 void Assembler::xrstor(Address src) {
8420 emit_int24(get_prefixq(src), 0x0F, (unsigned char)0xAE);
8421 emit_operand(as_Register(5), src);
8422 }
8423
8424 void Assembler::fxsave(Address dst) {
8425 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
8426 emit_operand(as_Register(0), dst);
8427 }
8428
8429 void Assembler::xsave(Address dst) {
8430 emit_int24(get_prefixq(dst), 0x0F, (unsigned char)0xAE);
8431 emit_operand(as_Register(4), dst);
8432 }
8433
8434 void Assembler::idivq(Register src) {
8435 int encode = prefixq_and_encode(src->encoding());
8436 emit_int16((unsigned char)0xF7, (0xF8 | encode));
8437 }
8438
8439 void Assembler::imulq(Register dst, Register src) {
8440 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8441 emit_int24(0x0F, (unsigned char)0xAF, (0xC0 | encode));
8442 }
8443
8444 void Assembler::imulq(Register dst, Register src, int value) {
8445 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8446 if (is8bit(value)) {
8447 emit_int24(0x6B, (0xC0 | encode), (value & 0xFF));
8448 } else {
8449 emit_int16(0x69, (0xC0 | encode));
8450 emit_int32(value);
8451 }
8452 }
8453
8454 void Assembler::imulq(Register dst, Address src) {
8455 InstructionMark im(this);
8456 emit_int24(get_prefixq(src, dst), 0x0F, (unsigned char)0xAF);
8457 emit_operand(dst, src);
8458 }
8459
8460 void Assembler::incl(Register dst) {
8461 // Don't use it directly. Use MacroAssembler::incrementl() instead.
8462 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8463 int encode = prefix_and_encode(dst->encoding());
8464 emit_int16((unsigned char)0xFF, (0xC0 | encode));
8465 }
8466
8467 void Assembler::incq(Register dst) {
8468 // Don't use it directly. Use MacroAssembler::incrementq() instead.
8469 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8470 int encode = prefixq_and_encode(dst->encoding());
8471 emit_int16((unsigned char)0xFF, (0xC0 | encode));
8472 }
8473
8474 void Assembler::incq(Address dst) {
8475 // Don't use it directly. Use MacroAssembler::incrementq() instead.
8476 InstructionMark im(this);
8477 emit_int16(get_prefixq(dst), (unsigned char)0xFF);
8478 emit_operand(rax, dst);
8479 }
8480
8481 void Assembler::lea(Register dst, Address src) {
8482 leaq(dst, src);
8483 }
8484
8485 void Assembler::leaq(Register dst, Address src) {
8486 InstructionMark im(this);
8487 emit_int16(get_prefixq(src, dst), (unsigned char)0x8D);
8488 emit_operand(dst, src);
8489 }
8490
8491 void Assembler::mov64(Register dst, int64_t imm64) {
8492 InstructionMark im(this);
8493 int encode = prefixq_and_encode(dst->encoding());
8494 emit_int8(0xB8 | encode);
8495 emit_int64(imm64);
8496 }
8497
8498 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8499 InstructionMark im(this);
8500 int encode = prefixq_and_encode(dst->encoding());
8501 emit_int8(0xB8 | encode);
8502 emit_data64(imm64, rspec);
8503 }
8504
8505 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8506 InstructionMark im(this);
8507 int encode = prefix_and_encode(dst->encoding());
8508 emit_int8(0xB8 | encode);
8509 emit_data((int)imm32, rspec, narrow_oop_operand);
8510 }
8511
8512 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
8513 InstructionMark im(this);
8514 prefix(dst);
8515 emit_int8((unsigned char)0xC7);
8516 emit_operand(rax, dst, 4);
8517 emit_data((int)imm32, rspec, narrow_oop_operand);
8518 }
8519
8520 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8521 InstructionMark im(this);
8522 int encode = prefix_and_encode(src1->encoding());
8523 emit_int16((unsigned char)0x81, (0xF8 | encode));
8524 emit_data((int)imm32, rspec, narrow_oop_operand);
8525 }
8526
8527 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8528 InstructionMark im(this);
8529 prefix(src1);
8530 emit_int8((unsigned char)0x81);
8531 emit_operand(rax, src1, 4);
8532 emit_data((int)imm32, rspec, narrow_oop_operand);
8533 }
8534
8535 void Assembler::lzcntq(Register dst, Register src) {
8536 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8537 emit_int8((unsigned char)0xF3);
8538 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8539 emit_int24(0x0F, (unsigned char)0xBD, (0xC0 | encode));
8540 }
8541
8542 void Assembler::movdq(XMMRegister dst, Register src) {
8543 // table D-1 says MMX/SSE2
8544 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8545 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8546 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8547 emit_int16(0x6E, (0xC0 | encode));
8548 }
8549
8550 void Assembler::movdq(Register dst, XMMRegister src) {
8551 // table D-1 says MMX/SSE2
8552 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8553 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8554 // swap src/dst to get correct prefix
8555 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8556 emit_int16(0x7E,
8557 (0xC0 | encode));
8558 }
8559
8560 void Assembler::movq(Register dst, Register src) {
8561 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8562 emit_int16((unsigned char)0x8B,
8563 (0xC0 | encode));
8564 }
8565
8566 void Assembler::movq(Register dst, Address src) {
8567 InstructionMark im(this);
8568 emit_int16(get_prefixq(src, dst), (unsigned char)0x8B);
8569 emit_operand(dst, src);
8570 }
8571
8572 void Assembler::movq(Address dst, Register src) {
8573 InstructionMark im(this);
8574 emit_int16(get_prefixq(dst, src), (unsigned char)0x89);
8575 emit_operand(src, dst);
8576 }
8577
8578 void Assembler::movsbq(Register dst, Address src) {
8579 InstructionMark im(this);
8580 emit_int24(get_prefixq(src, dst),
8581 0x0F,
8582 (unsigned char)0xBE);
8583 emit_operand(dst, src);
8584 }
8585
8586 void Assembler::movsbq(Register dst, Register src) {
8587 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8588 emit_int24(0x0F, (unsigned char)0xBE, (0xC0 | encode));
8589 }
8590
8591 void Assembler::movslq(Register dst, int32_t imm32) {
8592 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
8593 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
8594 // as a result we shouldn't use until tested at runtime...
8595 ShouldNotReachHere();
8596 InstructionMark im(this);
8597 int encode = prefixq_and_encode(dst->encoding());
8598 emit_int8(0xC7 | encode);
8599 emit_int32(imm32);
8600 }
8601
8602 void Assembler::movslq(Address dst, int32_t imm32) {
8603 assert(is_simm32(imm32), "lost bits");
8604 InstructionMark im(this);
8605 emit_int16(get_prefixq(dst), (unsigned char)0xC7);
8606 emit_operand(rax, dst, 4);
8607 emit_int32(imm32);
8608 }
8609
8610 void Assembler::movslq(Register dst, Address src) {
8611 InstructionMark im(this);
8612 emit_int16(get_prefixq(src, dst), 0x63);
8613 emit_operand(dst, src);
8614 }
8615
8616 void Assembler::movslq(Register dst, Register src) {
8617 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8618 emit_int16(0x63, (0xC0 | encode));
8619 }
8620
8621 void Assembler::movswq(Register dst, Address src) {
8622 InstructionMark im(this);
8623 emit_int24(get_prefixq(src, dst),
8624 0x0F,
8625 (unsigned char)0xBF);
8626 emit_operand(dst, src);
8627 }
8628
8629 void Assembler::movswq(Register dst, Register src) {
8630 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8631 emit_int24(0x0F, (unsigned char)0xBF, (0xC0 | encode));
8632 }
8633
8634 void Assembler::movzbq(Register dst, Address src) {
8635 InstructionMark im(this);
8636 emit_int24(get_prefixq(src, dst),
8637 0x0F,
8638 (unsigned char)0xB6);
8639 emit_operand(dst, src);
8640 }
8641
8642 void Assembler::movzbq(Register dst, Register src) {
8643 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8644 emit_int24(0x0F, (unsigned char)0xB6, (0xC0 | encode));
8645 }
8646
8647 void Assembler::movzwq(Register dst, Address src) {
8648 InstructionMark im(this);
8649 emit_int24(get_prefixq(src, dst),
8650 0x0F,
8651 (unsigned char)0xB7);
8652 emit_operand(dst, src);
8653 }
8654
8655 void Assembler::movzwq(Register dst, Register src) {
8656 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8657 emit_int24(0x0F, (unsigned char)0xB7, (0xC0 | encode));
8658 }
8659
8660 void Assembler::mulq(Address src) {
8661 InstructionMark im(this);
8662 emit_int16(get_prefixq(src), (unsigned char)0xF7);
8663 emit_operand(rsp, src);
8664 }
8665
8666 void Assembler::mulq(Register src) {
8667 int encode = prefixq_and_encode(src->encoding());
8668 emit_int16((unsigned char)0xF7, (0xE0 | encode));
8669 }
8670
8671 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8672 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8673 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8674 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8675 emit_int16((unsigned char)0xF6, (0xC0 | encode));
8676 }
8677
8678 void Assembler::negq(Register dst) {
8679 int encode = prefixq_and_encode(dst->encoding());
8680 emit_int16((unsigned char)0xF7, (0xD8 | encode));
8681 }
8682
8683 void Assembler::notq(Register dst) {
8684 int encode = prefixq_and_encode(dst->encoding());
8685 emit_int16((unsigned char)0xF7, (0xD0 | encode));
8686 }
8687
8688 void Assembler::btsq(Address dst, int imm8) {
8689 assert(isByte(imm8), "not a byte");
8690 InstructionMark im(this);
8691 emit_int24(get_prefixq(dst),
8692 0x0F,
8693 (unsigned char)0xBA);
8694 emit_operand(rbp /* 5 */, dst, 1);
8695 emit_int8(imm8);
8696 }
8697
8698 void Assembler::btrq(Address dst, int imm8) {
8699 assert(isByte(imm8), "not a byte");
8700 InstructionMark im(this);
8701 emit_int24(get_prefixq(dst),
8702 0x0F,
8703 (unsigned char)0xBA);
8704 emit_operand(rsi /* 6 */, dst, 1);
8705 emit_int8(imm8);
8706 }
8707
8708 void Assembler::orq(Address dst, int32_t imm32) {
8709 InstructionMark im(this);
8710 emit_int16(get_prefixq(dst), (unsigned char)0x81);
8711 emit_operand(rcx, dst, 4);
8712 emit_int32(imm32);
8713 }
8714
8715 void Assembler::orq(Register dst, int32_t imm32) {
8716 (void) prefixq_and_encode(dst->encoding());
8717 emit_arith(0x81, 0xC8, dst, imm32);
8718 }
8719
8720 void Assembler::orq(Register dst, Address src) {
8721 InstructionMark im(this);
8722 emit_int16(get_prefixq(src, dst), 0x0B);
8723 emit_operand(dst, src);
8724 }
8725
8726 void Assembler::orq(Register dst, Register src) {
8727 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8728 emit_arith(0x0B, 0xC0, dst, src);
8729 }
8730
8731 void Assembler::popcntq(Register dst, Address src) {
8732 assert(VM_Version::supports_popcnt(), "must support");
8733 InstructionMark im(this);
8734 emit_int32((unsigned char)0xF3,
8735 get_prefixq(src, dst),
8736 0x0F,
8737 (unsigned char)0xB8);
8738 emit_operand(dst, src);
8739 }
8740
8741 void Assembler::popcntq(Register dst, Register src) {
8742 assert(VM_Version::supports_popcnt(), "must support");
8743 emit_int8((unsigned char)0xF3);
8744 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8745 emit_int24(0x0F, (unsigned char)0xB8, (0xC0 | encode));
8746 }
8747
8748 void Assembler::popq(Address dst) {
8749 InstructionMark im(this);
8750 emit_int16(get_prefixq(dst), (unsigned char)0x8F);
8751 emit_operand(rax, dst);
8752 }
8753
8754 // Precomputable: popa, pusha, vzeroupper
8755
8756 // The result of these routines are invariant from one invocation to another
8757 // invocation for the duration of a run. Caching the result on bootstrap
8758 // and copying it out on subsequent invocations can thus be beneficial
8759 static bool precomputed = false;
8760
8761 static u_char* popa_code = NULL;
8762 static int popa_len = 0;
8763
8764 static u_char* pusha_code = NULL;
8765 static int pusha_len = 0;
8766
8767 static u_char* vzup_code = NULL;
8768 static int vzup_len = 0;
8769
8770 void Assembler::precompute_instructions() {
8771 assert(!Universe::is_fully_initialized(), "must still be single threaded");
8772 guarantee(!precomputed, "only once");
8773 precomputed = true;
8774 ResourceMark rm;
8775
8776 // Make a temporary buffer big enough for the routines we're capturing
8777 int size = 256;
8778 char* tmp_code = NEW_RESOURCE_ARRAY(char, size);
8779 CodeBuffer buffer((address)tmp_code, size);
8780 MacroAssembler masm(&buffer);
8781
8782 address begin_popa = masm.code_section()->end();
8783 masm.popa_uncached();
8784 address end_popa = masm.code_section()->end();
8785 masm.pusha_uncached();
8786 address end_pusha = masm.code_section()->end();
8787 masm.vzeroupper_uncached();
8788 address end_vzup = masm.code_section()->end();
8789
8790 // Save the instructions to permanent buffers.
8791 popa_len = (int)(end_popa - begin_popa);
8792 popa_code = NEW_C_HEAP_ARRAY(u_char, popa_len, mtInternal);
8793 memcpy(popa_code, begin_popa, popa_len);
8794
8795 pusha_len = (int)(end_pusha - end_popa);
8796 pusha_code = NEW_C_HEAP_ARRAY(u_char, pusha_len, mtInternal);
8797 memcpy(pusha_code, end_popa, pusha_len);
8798
8799 vzup_len = (int)(end_vzup - end_pusha);
8800 if (vzup_len > 0) {
8801 vzup_code = NEW_C_HEAP_ARRAY(u_char, vzup_len, mtInternal);
8802 memcpy(vzup_code, end_pusha, vzup_len);
8803 } else {
8804 vzup_code = pusha_code; // dummy
8805 }
8806
8807 assert(masm.code()->total_oop_size() == 0 &&
8808 masm.code()->total_metadata_size() == 0 &&
8809 masm.code()->total_relocation_size() == 0,
8810 "pre-computed code can't reference oops, metadata or contain relocations");
8811 }
8812
8813 static void emit_copy(CodeSection* code_section, u_char* src, int src_len) {
8814 assert(src != NULL, "code to copy must have been pre-computed");
8815 assert(code_section->limit() - code_section->end() > src_len, "code buffer not large enough");
8816 address end = code_section->end();
8817 memcpy(end, src, src_len);
8818 code_section->set_end(end + src_len);
8819 }
8820
8821 void Assembler::popa() { // 64bit
8822 emit_copy(code_section(), popa_code, popa_len);
8823 }
8824
8825 void Assembler::popa_uncached() { // 64bit
8826 movq(r15, Address(rsp, 0));
8827 movq(r14, Address(rsp, wordSize));
8828 movq(r13, Address(rsp, 2 * wordSize));
8829 movq(r12, Address(rsp, 3 * wordSize));
8830 movq(r11, Address(rsp, 4 * wordSize));
8831 movq(r10, Address(rsp, 5 * wordSize));
8832 movq(r9, Address(rsp, 6 * wordSize));
8833 movq(r8, Address(rsp, 7 * wordSize));
8834 movq(rdi, Address(rsp, 8 * wordSize));
8835 movq(rsi, Address(rsp, 9 * wordSize));
8836 movq(rbp, Address(rsp, 10 * wordSize));
8837 // Skip rsp as it is restored automatically to the value
8838 // before the corresponding pusha when popa is done.
8839 movq(rbx, Address(rsp, 12 * wordSize));
8840 movq(rdx, Address(rsp, 13 * wordSize));
8841 movq(rcx, Address(rsp, 14 * wordSize));
8842 movq(rax, Address(rsp, 15 * wordSize));
8843
8844 addq(rsp, 16 * wordSize);
8845 }
8846
8847 // Does not actually store the value of rsp on the stack.
8848 // The slot for rsp just contains an arbitrary value.
8849 void Assembler::pusha() { // 64bit
8850 emit_copy(code_section(), pusha_code, pusha_len);
8851 }
8852
8853 // Does not actually store the value of rsp on the stack.
8854 // The slot for rsp just contains an arbitrary value.
8855 void Assembler::pusha_uncached() { // 64bit
8856 subq(rsp, 16 * wordSize);
8857
8858 movq(Address(rsp, 15 * wordSize), rax);
8859 movq(Address(rsp, 14 * wordSize), rcx);
8860 movq(Address(rsp, 13 * wordSize), rdx);
8861 movq(Address(rsp, 12 * wordSize), rbx);
8862 // Skip rsp as the value is normally not used. There are a few places where
8863 // the original value of rsp needs to be known but that can be computed
8864 // from the value of rsp immediately after pusha (rsp + 16 * wordSize).
8865 movq(Address(rsp, 10 * wordSize), rbp);
8866 movq(Address(rsp, 9 * wordSize), rsi);
8867 movq(Address(rsp, 8 * wordSize), rdi);
8868 movq(Address(rsp, 7 * wordSize), r8);
8869 movq(Address(rsp, 6 * wordSize), r9);
8870 movq(Address(rsp, 5 * wordSize), r10);
8871 movq(Address(rsp, 4 * wordSize), r11);
8872 movq(Address(rsp, 3 * wordSize), r12);
8873 movq(Address(rsp, 2 * wordSize), r13);
8874 movq(Address(rsp, wordSize), r14);
8875 movq(Address(rsp, 0), r15);
8876 }
8877
8878 void Assembler::vzeroupper() {
8879 emit_copy(code_section(), vzup_code, vzup_len);
8880 }
8881
8882 void Assembler::pushq(Address src) {
8883 InstructionMark im(this);
8884 emit_int16(get_prefixq(src), (unsigned char)0xFF);
8885 emit_operand(rsi, src);
8886 }
8887
8888 void Assembler::rclq(Register dst, int imm8) {
8889 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8890 int encode = prefixq_and_encode(dst->encoding());
8891 if (imm8 == 1) {
8892 emit_int16((unsigned char)0xD1, (0xD0 | encode));
8893 } else {
8894 emit_int24((unsigned char)0xC1, (0xD0 | encode), imm8);
8895 }
8896 }
8897
8898 void Assembler::rcrq(Register dst, int imm8) {
8899 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8900 int encode = prefixq_and_encode(dst->encoding());
8901 if (imm8 == 1) {
8902 emit_int16((unsigned char)0xD1, (0xD8 | encode));
8903 } else {
8904 emit_int24((unsigned char)0xC1, (0xD8 | encode), imm8);
8905 }
8906 }
8907
8908 void Assembler::rorq(Register dst, int imm8) {
8909 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8910 int encode = prefixq_and_encode(dst->encoding());
8911 if (imm8 == 1) {
8912 emit_int16((unsigned char)0xD1, (0xC8 | encode));
8913 } else {
8914 emit_int24((unsigned char)0xC1, (0xc8 | encode), imm8);
8915 }
8916 }
8917
8918 void Assembler::rorxq(Register dst, Register src, int imm8) {
8919 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8920 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8921 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8922 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
8923 }
8924
8925 void Assembler::rorxd(Register dst, Register src, int imm8) {
8926 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8927 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8928 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8929 emit_int24((unsigned char)0xF0, (0xC0 | encode), imm8);
8930 }
8931
8932 void Assembler::sarq(Register dst, int imm8) {
8933 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8934 int encode = prefixq_and_encode(dst->encoding());
8935 if (imm8 == 1) {
8936 emit_int16((unsigned char)0xD1, (0xF8 | encode));
8937 } else {
8938 emit_int24((unsigned char)0xC1, (0xF8 | encode), imm8);
8939 }
8940 }
8941
8942 void Assembler::sarq(Register dst) {
8943 int encode = prefixq_and_encode(dst->encoding());
8944 emit_int16((unsigned char)0xD3, (0xF8 | encode));
8945 }
8946
8947 void Assembler::sbbq(Address dst, int32_t imm32) {
8948 InstructionMark im(this);
8949 prefixq(dst);
8950 emit_arith_operand(0x81, rbx, dst, imm32);
8951 }
8952
8953 void Assembler::sbbq(Register dst, int32_t imm32) {
8954 (void) prefixq_and_encode(dst->encoding());
8955 emit_arith(0x81, 0xD8, dst, imm32);
8956 }
8957
8958 void Assembler::sbbq(Register dst, Address src) {
8959 InstructionMark im(this);
8960 emit_int16(get_prefixq(src, dst), 0x1B);
8961 emit_operand(dst, src);
8962 }
8963
8964 void Assembler::sbbq(Register dst, Register src) {
8965 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8966 emit_arith(0x1B, 0xC0, dst, src);
8967 }
8968
8969 void Assembler::shlq(Register dst, int imm8) {
8970 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8971 int encode = prefixq_and_encode(dst->encoding());
8972 if (imm8 == 1) {
8973 emit_int16((unsigned char)0xD1, (0xE0 | encode));
8974 } else {
8975 emit_int24((unsigned char)0xC1, (0xE0 | encode), imm8);
8976 }
8977 }
8978
8979 void Assembler::shlq(Register dst) {
8980 int encode = prefixq_and_encode(dst->encoding());
8981 emit_int16((unsigned char)0xD3, (0xE0 | encode));
8982 }
8983
8984 void Assembler::shrq(Register dst, int imm8) {
8985 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8986 int encode = prefixq_and_encode(dst->encoding());
8987 emit_int24((unsigned char)0xC1, (0xE8 | encode), imm8);
8988 }
8989
8990 void Assembler::shrq(Register dst) {
8991 int encode = prefixq_and_encode(dst->encoding());
8992 emit_int16((unsigned char)0xD3, 0xE8 | encode);
8993 }
8994
8995 void Assembler::subq(Address dst, int32_t imm32) {
8996 InstructionMark im(this);
8997 prefixq(dst);
8998 emit_arith_operand(0x81, rbp, dst, imm32);
8999 }
9000
9001 void Assembler::subq(Address dst, Register src) {
9002 InstructionMark im(this);
9003 emit_int16(get_prefixq(dst, src), 0x29);
9004 emit_operand(src, dst);
9005 }
9006
9007 void Assembler::subq(Register dst, int32_t imm32) {
9008 (void) prefixq_and_encode(dst->encoding());
9009 emit_arith(0x81, 0xE8, dst, imm32);
9010 }
9011
9012 // Force generation of a 4 byte immediate value even if it fits into 8bit
9013 void Assembler::subq_imm32(Register dst, int32_t imm32) {
9014 (void) prefixq_and_encode(dst->encoding());
9015 emit_arith_imm32(0x81, 0xE8, dst, imm32);
9016 }
9017
9018 void Assembler::subq(Register dst, Address src) {
9019 InstructionMark im(this);
9020 emit_int16(get_prefixq(src, dst), 0x2B);
9021 emit_operand(dst, src);
9022 }
9023
9024 void Assembler::subq(Register dst, Register src) {
9025 (void) prefixq_and_encode(dst->encoding(), src->encoding());
9026 emit_arith(0x2B, 0xC0, dst, src);
9027 }
9028
9029 void Assembler::testq(Register dst, int32_t imm32) {
9030 // not using emit_arith because test
9031 // doesn't support sign-extension of
9032 // 8bit operands
9033 int encode = dst->encoding();
9034 if (encode == 0) {
9035 emit_int16(REX_W, (unsigned char)0xA9);
9036 } else {
9037 encode = prefixq_and_encode(encode);
9038 emit_int16((unsigned char)0xF7, (0xC0 | encode));
9039 }
9040 emit_int32(imm32);
9041 }
9042
9043 void Assembler::testq(Register dst, Register src) {
9044 (void) prefixq_and_encode(dst->encoding(), src->encoding());
9045 emit_arith(0x85, 0xC0, dst, src);
9046 }
9047
9048 void Assembler::testq(Register dst, Address src) {
9049 InstructionMark im(this);
9050 emit_int16(get_prefixq(src, dst), (unsigned char)0x85);
9051 emit_operand(dst, src);
9052 }
9053
9054 void Assembler::xaddq(Address dst, Register src) {
9055 InstructionMark im(this);
9056 emit_int24(get_prefixq(dst, src), 0x0F, (unsigned char)0xC1);
9057 emit_operand(src, dst);
9058 }
9059
9060 void Assembler::xchgq(Register dst, Address src) {
9061 InstructionMark im(this);
9062 emit_int16(get_prefixq(src, dst), (unsigned char)0x87);
9063 emit_operand(dst, src);
9064 }
9065
9066 void Assembler::xchgq(Register dst, Register src) {
9067 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
9068 emit_int16((unsigned char)0x87, (0xc0 | encode));
9069 }
9070
9071 void Assembler::xorq(Register dst, Register src) {
9072 (void) prefixq_and_encode(dst->encoding(), src->encoding());
9073 emit_arith(0x33, 0xC0, dst, src);
9074 }
9075
9076 void Assembler::xorq(Register dst, Address src) {
9077 InstructionMark im(this);
9078 emit_int16(get_prefixq(src, dst), 0x33);
9079 emit_operand(dst, src);
9080 }
9081
9082 #endif // !LP64