1 /*
2 * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableModRefBS.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/biasedLocking.hpp"
34 #include "runtime/interfaceSupport.hpp"
35 #include "runtime/objectMonitor.hpp"
36 #include "runtime/os.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "runtime/stubRoutines.hpp"
39 #include "utilities/macros.hpp"
40 #if INCLUDE_ALL_GCS
41 #include "gc/g1/g1BarrierSet.hpp"
42 #include "gc/g1/g1CollectedHeap.inline.hpp"
43 #include "gc/g1/heapRegion.hpp"
44 #endif // INCLUDE_ALL_GCS
45
46 #ifdef PRODUCT
47 #define BLOCK_COMMENT(str) /* nothing */
48 #define STOP(error) stop(error)
49 #else
50 #define BLOCK_COMMENT(str) block_comment(str)
51 #define STOP(error) block_comment(error); stop(error)
52 #endif
53
54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
55 // Implementation of AddressLiteral
56
57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
59 // -----------------Table 4.5 -------------------- //
60 16, 32, 64, // EVEX_FV(0)
61 4, 4, 4, // EVEX_FV(1) - with Evex.b
62 16, 32, 64, // EVEX_FV(2) - with Evex.w
63 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
64 8, 16, 32, // EVEX_HV(0)
65 4, 4, 4, // EVEX_HV(1) - with Evex.b
66 // -----------------Table 4.6 -------------------- //
67 16, 32, 64, // EVEX_FVM(0)
68 1, 1, 1, // EVEX_T1S(0)
69 2, 2, 2, // EVEX_T1S(1)
70 4, 4, 4, // EVEX_T1S(2)
71 8, 8, 8, // EVEX_T1S(3)
72 4, 4, 4, // EVEX_T1F(0)
73 8, 8, 8, // EVEX_T1F(1)
74 8, 8, 8, // EVEX_T2(0)
75 0, 16, 16, // EVEX_T2(1)
76 0, 16, 16, // EVEX_T4(0)
77 0, 0, 32, // EVEX_T4(1)
78 0, 0, 32, // EVEX_T8(0)
79 8, 16, 32, // EVEX_HVM(0)
80 4, 8, 16, // EVEX_QVM(0)
81 2, 4, 8, // EVEX_OVM(0)
82 16, 16, 16, // EVEX_M128(0)
83 8, 32, 64, // EVEX_DUP(0)
84 0, 0, 0 // EVEX_NTUP
85 };
86
87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
88 _is_lval = false;
89 _target = target;
90 switch (rtype) {
91 case relocInfo::oop_type:
92 case relocInfo::metadata_type:
93 // Oops are a special case. Normally they would be their own section
94 // but in cases like icBuffer they are literals in the code stream that
95 // we don't have a section for. We use none so that we get a literal address
96 // which is always patchable.
97 break;
98 case relocInfo::external_word_type:
99 _rspec = external_word_Relocation::spec(target);
100 break;
101 case relocInfo::internal_word_type:
102 _rspec = internal_word_Relocation::spec(target);
103 break;
104 case relocInfo::opt_virtual_call_type:
105 _rspec = opt_virtual_call_Relocation::spec();
106 break;
107 case relocInfo::static_call_type:
108 _rspec = static_call_Relocation::spec();
109 break;
110 case relocInfo::runtime_call_type:
111 _rspec = runtime_call_Relocation::spec();
112 break;
113 case relocInfo::poll_type:
114 case relocInfo::poll_return_type:
115 _rspec = Relocation::spec_simple(rtype);
116 break;
117 case relocInfo::none:
118 break;
119 default:
120 ShouldNotReachHere();
121 break;
122 }
123 }
124
125 // Implementation of Address
126
127 #ifdef _LP64
128
129 Address Address::make_array(ArrayAddress adr) {
130 // Not implementable on 64bit machines
131 // Should have been handled higher up the call chain.
132 ShouldNotReachHere();
133 return Address();
134 }
135
136 // exceedingly dangerous constructor
137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
138 _base = noreg;
139 _index = noreg;
140 _scale = no_scale;
141 _disp = disp;
142 switch (rtype) {
143 case relocInfo::external_word_type:
144 _rspec = external_word_Relocation::spec(loc);
145 break;
146 case relocInfo::internal_word_type:
147 _rspec = internal_word_Relocation::spec(loc);
148 break;
149 case relocInfo::runtime_call_type:
150 // HMM
151 _rspec = runtime_call_Relocation::spec();
152 break;
153 case relocInfo::poll_type:
154 case relocInfo::poll_return_type:
155 _rspec = Relocation::spec_simple(rtype);
156 break;
157 case relocInfo::none:
158 break;
159 default:
160 ShouldNotReachHere();
161 }
162 }
163 #else // LP64
164
165 Address Address::make_array(ArrayAddress adr) {
166 AddressLiteral base = adr.base();
167 Address index = adr.index();
168 assert(index._disp == 0, "must not have disp"); // maybe it can?
169 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
170 array._rspec = base._rspec;
171 return array;
172 }
173
174 // exceedingly dangerous constructor
175 Address::Address(address loc, RelocationHolder spec) {
176 _base = noreg;
177 _index = noreg;
178 _scale = no_scale;
179 _disp = (intptr_t) loc;
180 _rspec = spec;
181 }
182
183 #endif // _LP64
184
185
186
187 // Convert the raw encoding form into the form expected by the constructor for
188 // Address. An index of 4 (rsp) corresponds to having no index, so convert
189 // that to noreg for the Address constructor.
190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
191 RelocationHolder rspec;
192 if (disp_reloc != relocInfo::none) {
193 rspec = Relocation::spec_simple(disp_reloc);
194 }
195 bool valid_index = index != rsp->encoding();
196 if (valid_index) {
197 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
198 madr._rspec = rspec;
199 return madr;
200 } else {
201 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
202 madr._rspec = rspec;
203 return madr;
204 }
205 }
206
207 // Implementation of Assembler
208
209 int AbstractAssembler::code_fill_byte() {
210 return (u_char)'\xF4'; // hlt
211 }
212
213 // make this go away someday
214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
215 if (rtype == relocInfo::none)
216 emit_int32(data);
217 else
218 emit_data(data, Relocation::spec_simple(rtype), format);
219 }
220
221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
222 assert(imm_operand == 0, "default format must be immediate in this file");
223 assert(inst_mark() != NULL, "must be inside InstructionMark");
224 if (rspec.type() != relocInfo::none) {
225 #ifdef ASSERT
226 check_relocation(rspec, format);
227 #endif
228 // Do not use AbstractAssembler::relocate, which is not intended for
229 // embedded words. Instead, relocate to the enclosing instruction.
230
231 // hack. call32 is too wide for mask so use disp32
232 if (format == call32_operand)
233 code_section()->relocate(inst_mark(), rspec, disp32_operand);
234 else
235 code_section()->relocate(inst_mark(), rspec, format);
236 }
237 emit_int32(data);
238 }
239
240 static int encode(Register r) {
241 int enc = r->encoding();
242 if (enc >= 8) {
243 enc -= 8;
244 }
245 return enc;
246 }
247
248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
249 assert(dst->has_byte_register(), "must have byte register");
250 assert(isByte(op1) && isByte(op2), "wrong opcode");
251 assert(isByte(imm8), "not a byte");
252 assert((op1 & 0x01) == 0, "should be 8bit operation");
253 emit_int8(op1);
254 emit_int8(op2 | encode(dst));
255 emit_int8(imm8);
256 }
257
258
259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
260 assert(isByte(op1) && isByte(op2), "wrong opcode");
261 assert((op1 & 0x01) == 1, "should be 32bit operation");
262 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
263 if (is8bit(imm32)) {
264 emit_int8(op1 | 0x02); // set sign bit
265 emit_int8(op2 | encode(dst));
266 emit_int8(imm32 & 0xFF);
267 } else {
268 emit_int8(op1);
269 emit_int8(op2 | encode(dst));
270 emit_int32(imm32);
271 }
272 }
273
274 // Force generation of a 4 byte immediate value even if it fits into 8bit
275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
276 assert(isByte(op1) && isByte(op2), "wrong opcode");
277 assert((op1 & 0x01) == 1, "should be 32bit operation");
278 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
279 emit_int8(op1);
280 emit_int8(op2 | encode(dst));
281 emit_int32(imm32);
282 }
283
284 // immediate-to-memory forms
285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
286 assert((op1 & 0x01) == 1, "should be 32bit operation");
287 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
288 if (is8bit(imm32)) {
289 emit_int8(op1 | 0x02); // set sign bit
290 emit_operand(rm, adr, 1);
291 emit_int8(imm32 & 0xFF);
292 } else {
293 emit_int8(op1);
294 emit_operand(rm, adr, 4);
295 emit_int32(imm32);
296 }
297 }
298
299
300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
301 assert(isByte(op1) && isByte(op2), "wrong opcode");
302 emit_int8(op1);
303 emit_int8(op2 | encode(dst) << 3 | encode(src));
304 }
305
306
307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
308 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
309 int mod_idx = 0;
310 // We will test if the displacement fits the compressed format and if so
311 // apply the compression to the displacment iff the result is8bit.
312 if (VM_Version::supports_evex() && is_evex_inst) {
313 switch (cur_tuple_type) {
314 case EVEX_FV:
315 if ((cur_encoding & VEX_W) == VEX_W) {
316 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
317 } else {
318 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
319 }
320 break;
321
322 case EVEX_HV:
323 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
324 break;
325
326 case EVEX_FVM:
327 break;
328
329 case EVEX_T1S:
330 switch (in_size_in_bits) {
331 case EVEX_8bit:
332 break;
333
334 case EVEX_16bit:
335 mod_idx = 1;
336 break;
337
338 case EVEX_32bit:
339 mod_idx = 2;
340 break;
341
342 case EVEX_64bit:
343 mod_idx = 3;
344 break;
345 }
346 break;
347
348 case EVEX_T1F:
349 case EVEX_T2:
350 case EVEX_T4:
351 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
352 break;
353
354 case EVEX_T8:
355 break;
356
357 case EVEX_HVM:
358 break;
359
360 case EVEX_QVM:
361 break;
362
363 case EVEX_OVM:
364 break;
365
366 case EVEX_M128:
367 break;
368
369 case EVEX_DUP:
370 break;
371
372 default:
373 assert(0, "no valid evex tuple_table entry");
374 break;
375 }
376
377 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
378 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
379 if ((disp % disp_factor) == 0) {
380 int new_disp = disp / disp_factor;
381 if ((-0x80 <= new_disp && new_disp < 0x80)) {
382 disp = new_disp;
383 }
384 } else {
385 return false;
386 }
387 }
388 }
389 return (-0x80 <= disp && disp < 0x80);
390 }
391
392
393 bool Assembler::emit_compressed_disp_byte(int &disp) {
394 int mod_idx = 0;
395 // We will test if the displacement fits the compressed format and if so
396 // apply the compression to the displacment iff the result is8bit.
397 if (VM_Version::supports_evex() && _attributes && _attributes->is_evex_instruction()) {
398 int evex_encoding = _attributes->get_evex_encoding();
399 int tuple_type = _attributes->get_tuple_type();
400 switch (tuple_type) {
401 case EVEX_FV:
402 if ((evex_encoding & VEX_W) == VEX_W) {
403 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
404 } else {
405 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
406 }
407 break;
408
409 case EVEX_HV:
410 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
411 break;
412
413 case EVEX_FVM:
414 break;
415
416 case EVEX_T1S:
417 switch (_attributes->get_input_size()) {
418 case EVEX_8bit:
419 break;
420
421 case EVEX_16bit:
422 mod_idx = 1;
423 break;
424
425 case EVEX_32bit:
426 mod_idx = 2;
427 break;
428
429 case EVEX_64bit:
430 mod_idx = 3;
431 break;
432 }
433 break;
434
435 case EVEX_T1F:
436 case EVEX_T2:
437 case EVEX_T4:
438 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
439 break;
440
441 case EVEX_T8:
442 break;
443
444 case EVEX_HVM:
445 break;
446
447 case EVEX_QVM:
448 break;
449
450 case EVEX_OVM:
451 break;
452
453 case EVEX_M128:
454 break;
455
456 case EVEX_DUP:
457 break;
458
459 default:
460 assert(0, "no valid evex tuple_table entry");
461 break;
462 }
463
464 int vector_len = _attributes->get_vector_len();
465 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
466 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
467 if ((disp % disp_factor) == 0) {
468 int new_disp = disp / disp_factor;
469 if (is8bit(new_disp)) {
470 disp = new_disp;
471 }
472 } else {
473 return false;
474 }
475 }
476 }
477 return is8bit(disp);
478 }
479
480
481 void Assembler::emit_operand(Register reg, Register base, Register index,
482 Address::ScaleFactor scale, int disp,
483 RelocationHolder const& rspec,
484 int rip_relative_correction) {
485 relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
486
487 // Encode the registers as needed in the fields they are used in
488
489 int regenc = encode(reg) << 3;
490 int indexenc = index->is_valid() ? encode(index) << 3 : 0;
491 int baseenc = base->is_valid() ? encode(base) : 0;
492
493 if (base->is_valid()) {
494 if (index->is_valid()) {
495 assert(scale != Address::no_scale, "inconsistent address");
496 // [base + index*scale + disp]
497 if (disp == 0 && rtype == relocInfo::none &&
498 base != rbp LP64_ONLY(&& base != r13)) {
499 // [base + index*scale]
500 // [00 reg 100][ss index base]
501 assert(index != rsp, "illegal addressing mode");
502 emit_int8(0x04 | regenc);
503 emit_int8(scale << 6 | indexenc | baseenc);
504 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
505 // [base + index*scale + imm8]
506 // [01 reg 100][ss index base] imm8
507 assert(index != rsp, "illegal addressing mode");
508 emit_int8(0x44 | regenc);
509 emit_int8(scale << 6 | indexenc | baseenc);
510 emit_int8(disp & 0xFF);
511 } else {
512 // [base + index*scale + disp32]
513 // [10 reg 100][ss index base] disp32
514 assert(index != rsp, "illegal addressing mode");
515 emit_int8(0x84 | regenc);
516 emit_int8(scale << 6 | indexenc | baseenc);
517 emit_data(disp, rspec, disp32_operand);
518 }
519 } else if (base == rsp LP64_ONLY(|| base == r12)) {
520 // [rsp + disp]
521 if (disp == 0 && rtype == relocInfo::none) {
522 // [rsp]
523 // [00 reg 100][00 100 100]
524 emit_int8(0x04 | regenc);
525 emit_int8(0x24);
526 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
527 // [rsp + imm8]
528 // [01 reg 100][00 100 100] disp8
529 emit_int8(0x44 | regenc);
530 emit_int8(0x24);
531 emit_int8(disp & 0xFF);
532 } else {
533 // [rsp + imm32]
534 // [10 reg 100][00 100 100] disp32
535 emit_int8(0x84 | regenc);
536 emit_int8(0x24);
537 emit_data(disp, rspec, disp32_operand);
538 }
539 } else {
540 // [base + disp]
541 assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
542 if (disp == 0 && rtype == relocInfo::none &&
543 base != rbp LP64_ONLY(&& base != r13)) {
544 // [base]
545 // [00 reg base]
546 emit_int8(0x00 | regenc | baseenc);
547 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
548 // [base + disp8]
549 // [01 reg base] disp8
550 emit_int8(0x40 | regenc | baseenc);
551 emit_int8(disp & 0xFF);
552 } else {
553 // [base + disp32]
554 // [10 reg base] disp32
555 emit_int8(0x80 | regenc | baseenc);
556 emit_data(disp, rspec, disp32_operand);
557 }
558 }
559 } else {
560 if (index->is_valid()) {
561 assert(scale != Address::no_scale, "inconsistent address");
562 // [index*scale + disp]
563 // [00 reg 100][ss index 101] disp32
564 assert(index != rsp, "illegal addressing mode");
565 emit_int8(0x04 | regenc);
566 emit_int8(scale << 6 | indexenc | 0x05);
567 emit_data(disp, rspec, disp32_operand);
568 } else if (rtype != relocInfo::none ) {
569 // [disp] (64bit) RIP-RELATIVE (32bit) abs
570 // [00 000 101] disp32
571
572 emit_int8(0x05 | regenc);
573 // Note that the RIP-rel. correction applies to the generated
574 // disp field, but _not_ to the target address in the rspec.
575
576 // disp was created by converting the target address minus the pc
577 // at the start of the instruction. That needs more correction here.
578 // intptr_t disp = target - next_ip;
579 assert(inst_mark() != NULL, "must be inside InstructionMark");
580 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
581 int64_t adjusted = disp;
582 // Do rip-rel adjustment for 64bit
583 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
584 assert(is_simm32(adjusted),
585 "must be 32bit offset (RIP relative address)");
586 emit_data((int32_t) adjusted, rspec, disp32_operand);
587
588 } else {
589 // 32bit never did this, did everything as the rip-rel/disp code above
590 // [disp] ABSOLUTE
591 // [00 reg 100][00 100 101] disp32
592 emit_int8(0x04 | regenc);
593 emit_int8(0x25);
594 emit_data(disp, rspec, disp32_operand);
595 }
596 }
597 }
598
599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
600 Address::ScaleFactor scale, int disp,
601 RelocationHolder const& rspec) {
602 if (UseAVX > 2) {
603 int xreg_enc = reg->encoding();
604 if (xreg_enc > 15) {
605 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
606 emit_operand((Register)new_reg, base, index, scale, disp, rspec);
607 return;
608 }
609 }
610 emit_operand((Register)reg, base, index, scale, disp, rspec);
611 }
612
613 // Secret local extension to Assembler::WhichOperand:
614 #define end_pc_operand (_WhichOperand_limit)
615
616 address Assembler::locate_operand(address inst, WhichOperand which) {
617 // Decode the given instruction, and return the address of
618 // an embedded 32-bit operand word.
619
620 // If "which" is disp32_operand, selects the displacement portion
621 // of an effective address specifier.
622 // If "which" is imm64_operand, selects the trailing immediate constant.
623 // If "which" is call32_operand, selects the displacement of a call or jump.
624 // Caller is responsible for ensuring that there is such an operand,
625 // and that it is 32/64 bits wide.
626
627 // If "which" is end_pc_operand, find the end of the instruction.
628
629 address ip = inst;
630 bool is_64bit = false;
631
632 debug_only(bool has_disp32 = false);
633 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
634
635 again_after_prefix:
636 switch (0xFF & *ip++) {
637
638 // These convenience macros generate groups of "case" labels for the switch.
639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
641 case (x)+4: case (x)+5: case (x)+6: case (x)+7
642 #define REP16(x) REP8((x)+0): \
643 case REP8((x)+8)
644
645 case CS_segment:
646 case SS_segment:
647 case DS_segment:
648 case ES_segment:
649 case FS_segment:
650 case GS_segment:
651 // Seems dubious
652 LP64_ONLY(assert(false, "shouldn't have that prefix"));
653 assert(ip == inst+1, "only one prefix allowed");
654 goto again_after_prefix;
655
656 case 0x67:
657 case REX:
658 case REX_B:
659 case REX_X:
660 case REX_XB:
661 case REX_R:
662 case REX_RB:
663 case REX_RX:
664 case REX_RXB:
665 NOT_LP64(assert(false, "64bit prefixes"));
666 goto again_after_prefix;
667
668 case REX_W:
669 case REX_WB:
670 case REX_WX:
671 case REX_WXB:
672 case REX_WR:
673 case REX_WRB:
674 case REX_WRX:
675 case REX_WRXB:
676 NOT_LP64(assert(false, "64bit prefixes"));
677 is_64bit = true;
678 goto again_after_prefix;
679
680 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
681 case 0x88: // movb a, r
682 case 0x89: // movl a, r
683 case 0x8A: // movb r, a
684 case 0x8B: // movl r, a
685 case 0x8F: // popl a
686 debug_only(has_disp32 = true);
687 break;
688
689 case 0x68: // pushq #32
690 if (which == end_pc_operand) {
691 return ip + 4;
692 }
693 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
694 return ip; // not produced by emit_operand
695
696 case 0x66: // movw ... (size prefix)
697 again_after_size_prefix2:
698 switch (0xFF & *ip++) {
699 case REX:
700 case REX_B:
701 case REX_X:
702 case REX_XB:
703 case REX_R:
704 case REX_RB:
705 case REX_RX:
706 case REX_RXB:
707 case REX_W:
708 case REX_WB:
709 case REX_WX:
710 case REX_WXB:
711 case REX_WR:
712 case REX_WRB:
713 case REX_WRX:
714 case REX_WRXB:
715 NOT_LP64(assert(false, "64bit prefix found"));
716 goto again_after_size_prefix2;
717 case 0x8B: // movw r, a
718 case 0x89: // movw a, r
719 debug_only(has_disp32 = true);
720 break;
721 case 0xC7: // movw a, #16
722 debug_only(has_disp32 = true);
723 tail_size = 2; // the imm16
724 break;
725 case 0x0F: // several SSE/SSE2 variants
726 ip--; // reparse the 0x0F
727 goto again_after_prefix;
728 default:
729 ShouldNotReachHere();
730 }
731 break;
732
733 case REP8(0xB8): // movl/q r, #32/#64(oop?)
734 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
735 // these asserts are somewhat nonsensical
736 #ifndef _LP64
737 assert(which == imm_operand || which == disp32_operand,
738 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
739 #else
740 assert((which == call32_operand || which == imm_operand) && is_64bit ||
741 which == narrow_oop_operand && !is_64bit,
742 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
743 #endif // _LP64
744 return ip;
745
746 case 0x69: // imul r, a, #32
747 case 0xC7: // movl a, #32(oop?)
748 tail_size = 4;
749 debug_only(has_disp32 = true); // has both kinds of operands!
750 break;
751
752 case 0x0F: // movx..., etc.
753 switch (0xFF & *ip++) {
754 case 0x3A: // pcmpestri
755 tail_size = 1;
756 case 0x38: // ptest, pmovzxbw
757 ip++; // skip opcode
758 debug_only(has_disp32 = true); // has both kinds of operands!
759 break;
760
761 case 0x70: // pshufd r, r/a, #8
762 debug_only(has_disp32 = true); // has both kinds of operands!
763 case 0x73: // psrldq r, #8
764 tail_size = 1;
765 break;
766
767 case 0x12: // movlps
768 case 0x28: // movaps
769 case 0x2E: // ucomiss
770 case 0x2F: // comiss
771 case 0x54: // andps
772 case 0x55: // andnps
773 case 0x56: // orps
774 case 0x57: // xorps
775 case 0x58: // addpd
776 case 0x59: // mulpd
777 case 0x6E: // movd
778 case 0x7E: // movd
779 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
780 case 0xFE: // paddd
781 debug_only(has_disp32 = true);
782 break;
783
784 case 0xAD: // shrd r, a, %cl
785 case 0xAF: // imul r, a
786 case 0xBE: // movsbl r, a (movsxb)
787 case 0xBF: // movswl r, a (movsxw)
788 case 0xB6: // movzbl r, a (movzxb)
789 case 0xB7: // movzwl r, a (movzxw)
790 case REP16(0x40): // cmovl cc, r, a
791 case 0xB0: // cmpxchgb
792 case 0xB1: // cmpxchg
793 case 0xC1: // xaddl
794 case 0xC7: // cmpxchg8
795 case REP16(0x90): // setcc a
796 debug_only(has_disp32 = true);
797 // fall out of the switch to decode the address
798 break;
799
800 case 0xC4: // pinsrw r, a, #8
801 debug_only(has_disp32 = true);
802 case 0xC5: // pextrw r, r, #8
803 tail_size = 1; // the imm8
804 break;
805
806 case 0xAC: // shrd r, a, #8
807 debug_only(has_disp32 = true);
808 tail_size = 1; // the imm8
809 break;
810
811 case REP16(0x80): // jcc rdisp32
812 if (which == end_pc_operand) return ip + 4;
813 assert(which == call32_operand, "jcc has no disp32 or imm");
814 return ip;
815 default:
816 ShouldNotReachHere();
817 }
818 break;
819
820 case 0x81: // addl a, #32; addl r, #32
821 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
822 // on 32bit in the case of cmpl, the imm might be an oop
823 tail_size = 4;
824 debug_only(has_disp32 = true); // has both kinds of operands!
825 break;
826
827 case 0x83: // addl a, #8; addl r, #8
828 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
829 debug_only(has_disp32 = true); // has both kinds of operands!
830 tail_size = 1;
831 break;
832
833 case 0x9B:
834 switch (0xFF & *ip++) {
835 case 0xD9: // fnstcw a
836 debug_only(has_disp32 = true);
837 break;
838 default:
839 ShouldNotReachHere();
840 }
841 break;
842
843 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
844 case REP4(0x10): // adc...
845 case REP4(0x20): // and...
846 case REP4(0x30): // xor...
847 case REP4(0x08): // or...
848 case REP4(0x18): // sbb...
849 case REP4(0x28): // sub...
850 case 0xF7: // mull a
851 case 0x8D: // lea r, a
852 case 0x87: // xchg r, a
853 case REP4(0x38): // cmp...
854 case 0x85: // test r, a
855 debug_only(has_disp32 = true); // has both kinds of operands!
856 break;
857
858 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
859 case 0xC6: // movb a, #8
860 case 0x80: // cmpb a, #8
861 case 0x6B: // imul r, a, #8
862 debug_only(has_disp32 = true); // has both kinds of operands!
863 tail_size = 1; // the imm8
864 break;
865
866 case 0xC4: // VEX_3bytes
867 case 0xC5: // VEX_2bytes
868 assert((UseAVX > 0), "shouldn't have VEX prefix");
869 assert(ip == inst+1, "no prefixes allowed");
870 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
871 // but they have prefix 0x0F and processed when 0x0F processed above.
872 //
873 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
874 // instructions (these instructions are not supported in 64-bit mode).
875 // To distinguish them bits [7:6] are set in the VEX second byte since
876 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
877 // those VEX bits REX and vvvv bits are inverted.
878 //
879 // Fortunately C2 doesn't generate these instructions so we don't need
880 // to check for them in product version.
881
882 // Check second byte
883 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
884
885 int vex_opcode;
886 // First byte
887 if ((0xFF & *inst) == VEX_3bytes) {
888 vex_opcode = VEX_OPCODE_MASK & *ip;
889 ip++; // third byte
890 is_64bit = ((VEX_W & *ip) == VEX_W);
891 } else {
892 vex_opcode = VEX_OPCODE_0F;
893 }
894 ip++; // opcode
895 // To find the end of instruction (which == end_pc_operand).
896 switch (vex_opcode) {
897 case VEX_OPCODE_0F:
898 switch (0xFF & *ip) {
899 case 0x70: // pshufd r, r/a, #8
900 case 0x71: // ps[rl|ra|ll]w r, #8
901 case 0x72: // ps[rl|ra|ll]d r, #8
902 case 0x73: // ps[rl|ra|ll]q r, #8
903 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
904 case 0xC4: // pinsrw r, r, r/a, #8
905 case 0xC5: // pextrw r/a, r, #8
906 case 0xC6: // shufp[s|d] r, r, r/a, #8
907 tail_size = 1; // the imm8
908 break;
909 }
910 break;
911 case VEX_OPCODE_0F_3A:
912 tail_size = 1;
913 break;
914 }
915 ip++; // skip opcode
916 debug_only(has_disp32 = true); // has both kinds of operands!
917 break;
918
919 case 0x62: // EVEX_4bytes
920 assert(VM_Version::supports_evex(), "shouldn't have EVEX prefix");
921 assert(ip == inst+1, "no prefixes allowed");
922 // no EVEX collisions, all instructions that have 0x62 opcodes
923 // have EVEX versions and are subopcodes of 0x66
924 ip++; // skip P0 and exmaine W in P1
925 is_64bit = ((VEX_W & *ip) == VEX_W);
926 ip++; // move to P2
927 ip++; // skip P2, move to opcode
928 // To find the end of instruction (which == end_pc_operand).
929 switch (0xFF & *ip) {
930 case 0x22: // pinsrd r, r/a, #8
931 case 0x61: // pcmpestri r, r/a, #8
932 case 0x70: // pshufd r, r/a, #8
933 case 0x73: // psrldq r, #8
934 tail_size = 1; // the imm8
935 break;
936 default:
937 break;
938 }
939 ip++; // skip opcode
940 debug_only(has_disp32 = true); // has both kinds of operands!
941 break;
942
943 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
944 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
945 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
946 case 0xDD: // fld_d a; fst_d a; fstp_d a
947 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
948 case 0xDF: // fild_d a; fistp_d a
949 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
950 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
951 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
952 debug_only(has_disp32 = true);
953 break;
954
955 case 0xE8: // call rdisp32
956 case 0xE9: // jmp rdisp32
957 if (which == end_pc_operand) return ip + 4;
958 assert(which == call32_operand, "call has no disp32 or imm");
959 return ip;
960
961 case 0xF0: // Lock
962 assert(os::is_MP(), "only on MP");
963 goto again_after_prefix;
964
965 case 0xF3: // For SSE
966 case 0xF2: // For SSE2
967 switch (0xFF & *ip++) {
968 case REX:
969 case REX_B:
970 case REX_X:
971 case REX_XB:
972 case REX_R:
973 case REX_RB:
974 case REX_RX:
975 case REX_RXB:
976 case REX_W:
977 case REX_WB:
978 case REX_WX:
979 case REX_WXB:
980 case REX_WR:
981 case REX_WRB:
982 case REX_WRX:
983 case REX_WRXB:
984 NOT_LP64(assert(false, "found 64bit prefix"));
985 ip++;
986 default:
987 ip++;
988 }
989 debug_only(has_disp32 = true); // has both kinds of operands!
990 break;
991
992 default:
993 ShouldNotReachHere();
994
995 #undef REP8
996 #undef REP16
997 }
998
999 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
1000 #ifdef _LP64
1001 assert(which != imm_operand, "instruction is not a movq reg, imm64");
1002 #else
1003 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1004 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1005 #endif // LP64
1006 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1007
1008 // parse the output of emit_operand
1009 int op2 = 0xFF & *ip++;
1010 int base = op2 & 0x07;
1011 int op3 = -1;
1012 const int b100 = 4;
1013 const int b101 = 5;
1014 if (base == b100 && (op2 >> 6) != 3) {
1015 op3 = 0xFF & *ip++;
1016 base = op3 & 0x07; // refetch the base
1017 }
1018 // now ip points at the disp (if any)
1019
1020 switch (op2 >> 6) {
1021 case 0:
1022 // [00 reg 100][ss index base]
1023 // [00 reg 100][00 100 esp]
1024 // [00 reg base]
1025 // [00 reg 100][ss index 101][disp32]
1026 // [00 reg 101] [disp32]
1027
1028 if (base == b101) {
1029 if (which == disp32_operand)
1030 return ip; // caller wants the disp32
1031 ip += 4; // skip the disp32
1032 }
1033 break;
1034
1035 case 1:
1036 // [01 reg 100][ss index base][disp8]
1037 // [01 reg 100][00 100 esp][disp8]
1038 // [01 reg base] [disp8]
1039 ip += 1; // skip the disp8
1040 break;
1041
1042 case 2:
1043 // [10 reg 100][ss index base][disp32]
1044 // [10 reg 100][00 100 esp][disp32]
1045 // [10 reg base] [disp32]
1046 if (which == disp32_operand)
1047 return ip; // caller wants the disp32
1048 ip += 4; // skip the disp32
1049 break;
1050
1051 case 3:
1052 // [11 reg base] (not a memory addressing mode)
1053 break;
1054 }
1055
1056 if (which == end_pc_operand) {
1057 return ip + tail_size;
1058 }
1059
1060 #ifdef _LP64
1061 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1062 #else
1063 assert(which == imm_operand, "instruction has only an imm field");
1064 #endif // LP64
1065 return ip;
1066 }
1067
1068 address Assembler::locate_next_instruction(address inst) {
1069 // Secretly share code with locate_operand:
1070 return locate_operand(inst, end_pc_operand);
1071 }
1072
1073
1074 #ifdef ASSERT
1075 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1076 address inst = inst_mark();
1077 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1078 address opnd;
1079
1080 Relocation* r = rspec.reloc();
1081 if (r->type() == relocInfo::none) {
1082 return;
1083 } else if (r->is_call() || format == call32_operand) {
1084 // assert(format == imm32_operand, "cannot specify a nonzero format");
1085 opnd = locate_operand(inst, call32_operand);
1086 } else if (r->is_data()) {
1087 assert(format == imm_operand || format == disp32_operand
1088 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1089 opnd = locate_operand(inst, (WhichOperand)format);
1090 } else {
1091 assert(format == imm_operand, "cannot specify a format");
1092 return;
1093 }
1094 assert(opnd == pc(), "must put operand where relocs can find it");
1095 }
1096 #endif // ASSERT
1097
1098 void Assembler::emit_operand32(Register reg, Address adr) {
1099 assert(reg->encoding() < 8, "no extended registers");
1100 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1102 adr._rspec);
1103 }
1104
1105 void Assembler::emit_operand(Register reg, Address adr,
1106 int rip_relative_correction) {
1107 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1108 adr._rspec,
1109 rip_relative_correction);
1110 }
1111
1112 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1113 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1114 adr._rspec);
1115 }
1116
1117 // MMX operations
1118 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1119 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1120 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1121 }
1122
1123 // work around gcc (3.2.1-7a) bug
1124 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1125 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1126 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1127 }
1128
1129
1130 void Assembler::emit_farith(int b1, int b2, int i) {
1131 assert(isByte(b1) && isByte(b2), "wrong opcode");
1132 assert(0 <= i && i < 8, "illegal stack offset");
1133 emit_int8(b1);
1134 emit_int8(b2 + i);
1135 }
1136
1137
1138 // Now the Assembler instructions (identical for 32/64 bits)
1139
1140 void Assembler::adcl(Address dst, int32_t imm32) {
1141 InstructionMark im(this);
1142 prefix(dst);
1143 emit_arith_operand(0x81, rdx, dst, imm32);
1144 }
1145
1146 void Assembler::adcl(Address dst, Register src) {
1147 InstructionMark im(this);
1148 prefix(dst, src);
1149 emit_int8(0x11);
1150 emit_operand(src, dst);
1151 }
1152
1153 void Assembler::adcl(Register dst, int32_t imm32) {
1154 prefix(dst);
1155 emit_arith(0x81, 0xD0, dst, imm32);
1156 }
1157
1158 void Assembler::adcl(Register dst, Address src) {
1159 InstructionMark im(this);
1160 prefix(src, dst);
1161 emit_int8(0x13);
1162 emit_operand(dst, src);
1163 }
1164
1165 void Assembler::adcl(Register dst, Register src) {
1166 (void) prefix_and_encode(dst->encoding(), src->encoding());
1167 emit_arith(0x13, 0xC0, dst, src);
1168 }
1169
1170 void Assembler::addl(Address dst, int32_t imm32) {
1171 InstructionMark im(this);
1172 prefix(dst);
1173 emit_arith_operand(0x81, rax, dst, imm32);
1174 }
1175
1176 void Assembler::addb(Address dst, int imm8) {
1177 InstructionMark im(this);
1178 prefix(dst);
1179 emit_int8((unsigned char)0x80);
1180 emit_operand(rax, dst, 1);
1181 emit_int8(imm8);
1182 }
1183
1184 void Assembler::addw(Address dst, int imm16) {
1185 InstructionMark im(this);
1186 emit_int8(0x66);
1187 prefix(dst);
1188 emit_int8((unsigned char)0x81);
1189 emit_operand(rax, dst, 2);
1190 emit_int16(imm16);
1191 }
1192
1193 void Assembler::addl(Address dst, Register src) {
1194 InstructionMark im(this);
1195 prefix(dst, src);
1196 emit_int8(0x01);
1197 emit_operand(src, dst);
1198 }
1199
1200 void Assembler::addl(Register dst, int32_t imm32) {
1201 prefix(dst);
1202 emit_arith(0x81, 0xC0, dst, imm32);
1203 }
1204
1205 void Assembler::addl(Register dst, Address src) {
1206 InstructionMark im(this);
1207 prefix(src, dst);
1208 emit_int8(0x03);
1209 emit_operand(dst, src);
1210 }
1211
1212 void Assembler::addl(Register dst, Register src) {
1213 (void) prefix_and_encode(dst->encoding(), src->encoding());
1214 emit_arith(0x03, 0xC0, dst, src);
1215 }
1216
1217 void Assembler::addr_nop_4() {
1218 assert(UseAddressNop, "no CPU support");
1219 // 4 bytes: NOP DWORD PTR [EAX+0]
1220 emit_int8(0x0F);
1221 emit_int8(0x1F);
1222 emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1223 emit_int8(0); // 8-bits offset (1 byte)
1224 }
1225
1226 void Assembler::addr_nop_5() {
1227 assert(UseAddressNop, "no CPU support");
1228 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1229 emit_int8(0x0F);
1230 emit_int8(0x1F);
1231 emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1232 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1233 emit_int8(0); // 8-bits offset (1 byte)
1234 }
1235
1236 void Assembler::addr_nop_7() {
1237 assert(UseAddressNop, "no CPU support");
1238 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1239 emit_int8(0x0F);
1240 emit_int8(0x1F);
1241 emit_int8((unsigned char)0x80);
1242 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1243 emit_int32(0); // 32-bits offset (4 bytes)
1244 }
1245
1246 void Assembler::addr_nop_8() {
1247 assert(UseAddressNop, "no CPU support");
1248 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1249 emit_int8(0x0F);
1250 emit_int8(0x1F);
1251 emit_int8((unsigned char)0x84);
1252 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1253 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1254 emit_int32(0); // 32-bits offset (4 bytes)
1255 }
1256
1257 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1258 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1259 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1260 attributes.set_rex_vex_w_reverted();
1261 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1262 emit_int8(0x58);
1263 emit_int8((unsigned char)(0xC0 | encode));
1264 }
1265
1266 void Assembler::addsd(XMMRegister dst, Address src) {
1267 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1268 InstructionMark im(this);
1269 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1270 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1271 attributes.set_rex_vex_w_reverted();
1272 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1273 emit_int8(0x58);
1274 emit_operand(dst, src);
1275 }
1276
1277 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1278 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1279 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1280 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1281 emit_int8(0x58);
1282 emit_int8((unsigned char)(0xC0 | encode));
1283 }
1284
1285 void Assembler::addss(XMMRegister dst, Address src) {
1286 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1287 InstructionMark im(this);
1288 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1289 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1290 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1291 emit_int8(0x58);
1292 emit_operand(dst, src);
1293 }
1294
1295 void Assembler::aesdec(XMMRegister dst, Address src) {
1296 assert(VM_Version::supports_aes(), "");
1297 InstructionMark im(this);
1298 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1299 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1300 emit_int8((unsigned char)0xDE);
1301 emit_operand(dst, src);
1302 }
1303
1304 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1305 assert(VM_Version::supports_aes(), "");
1306 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1307 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1308 emit_int8((unsigned char)0xDE);
1309 emit_int8(0xC0 | encode);
1310 }
1311
1312 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1313 assert(VM_Version::supports_aes(), "");
1314 InstructionMark im(this);
1315 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1316 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1317 emit_int8((unsigned char)0xDF);
1318 emit_operand(dst, src);
1319 }
1320
1321 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1322 assert(VM_Version::supports_aes(), "");
1323 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1324 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1325 emit_int8((unsigned char)0xDF);
1326 emit_int8((unsigned char)(0xC0 | encode));
1327 }
1328
1329 void Assembler::aesenc(XMMRegister dst, Address src) {
1330 assert(VM_Version::supports_aes(), "");
1331 InstructionMark im(this);
1332 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1333 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1334 emit_int8((unsigned char)0xDC);
1335 emit_operand(dst, src);
1336 }
1337
1338 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1339 assert(VM_Version::supports_aes(), "");
1340 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1341 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1342 emit_int8((unsigned char)0xDC);
1343 emit_int8(0xC0 | encode);
1344 }
1345
1346 void Assembler::aesenclast(XMMRegister dst, Address src) {
1347 assert(VM_Version::supports_aes(), "");
1348 InstructionMark im(this);
1349 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1350 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1351 emit_int8((unsigned char)0xDD);
1352 emit_operand(dst, src);
1353 }
1354
1355 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1356 assert(VM_Version::supports_aes(), "");
1357 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1358 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1359 emit_int8((unsigned char)0xDD);
1360 emit_int8((unsigned char)(0xC0 | encode));
1361 }
1362
1363 void Assembler::andl(Address dst, int32_t imm32) {
1364 InstructionMark im(this);
1365 prefix(dst);
1366 emit_int8((unsigned char)0x81);
1367 emit_operand(rsp, dst, 4);
1368 emit_int32(imm32);
1369 }
1370
1371 void Assembler::andl(Register dst, int32_t imm32) {
1372 prefix(dst);
1373 emit_arith(0x81, 0xE0, dst, imm32);
1374 }
1375
1376 void Assembler::andl(Register dst, Address src) {
1377 InstructionMark im(this);
1378 prefix(src, dst);
1379 emit_int8(0x23);
1380 emit_operand(dst, src);
1381 }
1382
1383 void Assembler::andl(Register dst, Register src) {
1384 (void) prefix_and_encode(dst->encoding(), src->encoding());
1385 emit_arith(0x23, 0xC0, dst, src);
1386 }
1387
1388 void Assembler::andnl(Register dst, Register src1, Register src2) {
1389 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1390 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1391 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1392 emit_int8((unsigned char)0xF2);
1393 emit_int8((unsigned char)(0xC0 | encode));
1394 }
1395
1396 void Assembler::andnl(Register dst, Register src1, Address src2) {
1397 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1398 InstructionMark im(this);
1399 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1400 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1401 emit_int8((unsigned char)0xF2);
1402 emit_operand(dst, src2);
1403 }
1404
1405 void Assembler::bsfl(Register dst, Register src) {
1406 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1407 emit_int8(0x0F);
1408 emit_int8((unsigned char)0xBC);
1409 emit_int8((unsigned char)(0xC0 | encode));
1410 }
1411
1412 void Assembler::bsrl(Register dst, Register src) {
1413 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1414 emit_int8(0x0F);
1415 emit_int8((unsigned char)0xBD);
1416 emit_int8((unsigned char)(0xC0 | encode));
1417 }
1418
1419 void Assembler::bswapl(Register reg) { // bswap
1420 int encode = prefix_and_encode(reg->encoding());
1421 emit_int8(0x0F);
1422 emit_int8((unsigned char)(0xC8 | encode));
1423 }
1424
1425 void Assembler::blsil(Register dst, Register src) {
1426 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1427 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1428 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1429 emit_int8((unsigned char)0xF3);
1430 emit_int8((unsigned char)(0xC0 | encode));
1431 }
1432
1433 void Assembler::blsil(Register dst, Address src) {
1434 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1435 InstructionMark im(this);
1436 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1437 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1438 emit_int8((unsigned char)0xF3);
1439 emit_operand(rbx, src);
1440 }
1441
1442 void Assembler::blsmskl(Register dst, Register src) {
1443 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1444 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1445 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1446 emit_int8((unsigned char)0xF3);
1447 emit_int8((unsigned char)(0xC0 | encode));
1448 }
1449
1450 void Assembler::blsmskl(Register dst, Address src) {
1451 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1452 InstructionMark im(this);
1453 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1454 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1455 emit_int8((unsigned char)0xF3);
1456 emit_operand(rdx, src);
1457 }
1458
1459 void Assembler::blsrl(Register dst, Register src) {
1460 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1461 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1462 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1463 emit_int8((unsigned char)0xF3);
1464 emit_int8((unsigned char)(0xC0 | encode));
1465 }
1466
1467 void Assembler::blsrl(Register dst, Address src) {
1468 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1469 InstructionMark im(this);
1470 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
1471 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1472 emit_int8((unsigned char)0xF3);
1473 emit_operand(rcx, src);
1474 }
1475
1476 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1477 // suspect disp32 is always good
1478 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1479
1480 if (L.is_bound()) {
1481 const int long_size = 5;
1482 int offs = (int)( target(L) - pc() );
1483 assert(offs <= 0, "assembler error");
1484 InstructionMark im(this);
1485 // 1110 1000 #32-bit disp
1486 emit_int8((unsigned char)0xE8);
1487 emit_data(offs - long_size, rtype, operand);
1488 } else {
1489 InstructionMark im(this);
1490 // 1110 1000 #32-bit disp
1491 L.add_patch_at(code(), locator());
1492
1493 emit_int8((unsigned char)0xE8);
1494 emit_data(int(0), rtype, operand);
1495 }
1496 }
1497
1498 void Assembler::call(Register dst) {
1499 int encode = prefix_and_encode(dst->encoding());
1500 emit_int8((unsigned char)0xFF);
1501 emit_int8((unsigned char)(0xD0 | encode));
1502 }
1503
1504
1505 void Assembler::call(Address adr) {
1506 InstructionMark im(this);
1507 prefix(adr);
1508 emit_int8((unsigned char)0xFF);
1509 emit_operand(rdx, adr);
1510 }
1511
1512 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1513 InstructionMark im(this);
1514 emit_int8((unsigned char)0xE8);
1515 intptr_t disp = entry - (pc() + sizeof(int32_t));
1516 // Entry is NULL in case of a scratch emit.
1517 assert(entry == NULL || is_simm32(disp), "disp=" INTPTR_FORMAT " must be 32bit offset (call2)", disp);
1518 // Technically, should use call32_operand, but this format is
1519 // implied by the fact that we're emitting a call instruction.
1520
1521 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1522 emit_data((int) disp, rspec, operand);
1523 }
1524
1525 void Assembler::cdql() {
1526 emit_int8((unsigned char)0x99);
1527 }
1528
1529 void Assembler::cld() {
1530 emit_int8((unsigned char)0xFC);
1531 }
1532
1533 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1534 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1535 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1536 emit_int8(0x0F);
1537 emit_int8(0x40 | cc);
1538 emit_int8((unsigned char)(0xC0 | encode));
1539 }
1540
1541
1542 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1543 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1544 prefix(src, dst);
1545 emit_int8(0x0F);
1546 emit_int8(0x40 | cc);
1547 emit_operand(dst, src);
1548 }
1549
1550 void Assembler::cmpb(Address dst, int imm8) {
1551 InstructionMark im(this);
1552 prefix(dst);
1553 emit_int8((unsigned char)0x80);
1554 emit_operand(rdi, dst, 1);
1555 emit_int8(imm8);
1556 }
1557
1558 void Assembler::cmpl(Address dst, int32_t imm32) {
1559 InstructionMark im(this);
1560 prefix(dst);
1561 emit_int8((unsigned char)0x81);
1562 emit_operand(rdi, dst, 4);
1563 emit_int32(imm32);
1564 }
1565
1566 void Assembler::cmpl(Register dst, int32_t imm32) {
1567 prefix(dst);
1568 emit_arith(0x81, 0xF8, dst, imm32);
1569 }
1570
1571 void Assembler::cmpl(Register dst, Register src) {
1572 (void) prefix_and_encode(dst->encoding(), src->encoding());
1573 emit_arith(0x3B, 0xC0, dst, src);
1574 }
1575
1576 void Assembler::cmpl(Register dst, Address src) {
1577 InstructionMark im(this);
1578 prefix(src, dst);
1579 emit_int8((unsigned char)0x3B);
1580 emit_operand(dst, src);
1581 }
1582
1583 void Assembler::cmpw(Address dst, int imm16) {
1584 InstructionMark im(this);
1585 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1586 emit_int8(0x66);
1587 emit_int8((unsigned char)0x81);
1588 emit_operand(rdi, dst, 2);
1589 emit_int16(imm16);
1590 }
1591
1592 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1593 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1594 // The ZF is set if the compared values were equal, and cleared otherwise.
1595 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1596 InstructionMark im(this);
1597 prefix(adr, reg);
1598 emit_int8(0x0F);
1599 emit_int8((unsigned char)0xB1);
1600 emit_operand(reg, adr);
1601 }
1602
1603 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1604 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1605 // The ZF is set if the compared values were equal, and cleared otherwise.
1606 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1607 InstructionMark im(this);
1608 prefix(adr, reg, true);
1609 emit_int8(0x0F);
1610 emit_int8((unsigned char)0xB0);
1611 emit_operand(reg, adr);
1612 }
1613
1614 void Assembler::comisd(XMMRegister dst, Address src) {
1615 // NOTE: dbx seems to decode this as comiss even though the
1616 // 0x66 is there. Strangly ucomisd comes out correct
1617 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1618 InstructionMark im(this);
1619 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1620 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1621 attributes.set_rex_vex_w_reverted();
1622 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1623 emit_int8(0x2F);
1624 emit_operand(dst, src);
1625 }
1626
1627 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1628 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1629 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1630 attributes.set_rex_vex_w_reverted();
1631 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1632 emit_int8(0x2F);
1633 emit_int8((unsigned char)(0xC0 | encode));
1634 }
1635
1636 void Assembler::comiss(XMMRegister dst, Address src) {
1637 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1638 InstructionMark im(this);
1639 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1640 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1641 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1642 emit_int8(0x2F);
1643 emit_operand(dst, src);
1644 }
1645
1646 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1647 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1648 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1649 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1650 emit_int8(0x2F);
1651 emit_int8((unsigned char)(0xC0 | encode));
1652 }
1653
1654 void Assembler::cpuid() {
1655 emit_int8(0x0F);
1656 emit_int8((unsigned char)0xA2);
1657 }
1658
1659 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1660 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1661 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1662 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1663 //
1664 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1665 //
1666 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1667 //
1668 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1669 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1670 assert(VM_Version::supports_sse4_2(), "");
1671 int8_t w = 0x01;
1672 Prefix p = Prefix_EMPTY;
1673
1674 emit_int8((int8_t)0xF2);
1675 switch (sizeInBytes) {
1676 case 1:
1677 w = 0;
1678 break;
1679 case 2:
1680 case 4:
1681 break;
1682 LP64_ONLY(case 8:)
1683 // This instruction is not valid in 32 bits
1684 // Note:
1685 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1686 //
1687 // Page B - 72 Vol. 2C says
1688 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1689 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1690 // F0!!!
1691 // while 3 - 208 Vol. 2A
1692 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1693 //
1694 // the 0 on a last bit is reserved for a different flavor of this instruction :
1695 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1696 p = REX_W;
1697 break;
1698 default:
1699 assert(0, "Unsupported value for a sizeInBytes argument");
1700 break;
1701 }
1702 LP64_ONLY(prefix(crc, v, p);)
1703 emit_int8((int8_t)0x0F);
1704 emit_int8(0x38);
1705 emit_int8((int8_t)(0xF0 | w));
1706 emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1707 }
1708
1709 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1710 assert(VM_Version::supports_sse4_2(), "");
1711 InstructionMark im(this);
1712 int8_t w = 0x01;
1713 Prefix p = Prefix_EMPTY;
1714
1715 emit_int8((int8_t)0xF2);
1716 switch (sizeInBytes) {
1717 case 1:
1718 w = 0;
1719 break;
1720 case 2:
1721 case 4:
1722 break;
1723 LP64_ONLY(case 8:)
1724 // This instruction is not valid in 32 bits
1725 p = REX_W;
1726 break;
1727 default:
1728 assert(0, "Unsupported value for a sizeInBytes argument");
1729 break;
1730 }
1731 LP64_ONLY(prefix(crc, adr, p);)
1732 emit_int8((int8_t)0x0F);
1733 emit_int8(0x38);
1734 emit_int8((int8_t)(0xF0 | w));
1735 emit_operand(crc, adr);
1736 }
1737
1738 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1739 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1741 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1742 emit_int8((unsigned char)0xE6);
1743 emit_int8((unsigned char)(0xC0 | encode));
1744 }
1745
1746 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1747 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1748 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1749 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1750 emit_int8(0x5B);
1751 emit_int8((unsigned char)(0xC0 | encode));
1752 }
1753
1754 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1755 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1756 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1757 attributes.set_rex_vex_w_reverted();
1758 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1759 emit_int8(0x5A);
1760 emit_int8((unsigned char)(0xC0 | encode));
1761 }
1762
1763 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1764 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1765 InstructionMark im(this);
1766 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1767 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1768 attributes.set_rex_vex_w_reverted();
1769 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1770 emit_int8(0x5A);
1771 emit_operand(dst, src);
1772 }
1773
1774 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1775 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1776 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1777 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1778 emit_int8(0x2A);
1779 emit_int8((unsigned char)(0xC0 | encode));
1780 }
1781
1782 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1783 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1784 InstructionMark im(this);
1785 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1786 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1787 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1788 emit_int8(0x2A);
1789 emit_operand(dst, src);
1790 }
1791
1792 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1793 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1794 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1795 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796 emit_int8(0x2A);
1797 emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799
1800 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1801 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1802 InstructionMark im(this);
1803 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1804 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806 emit_int8(0x2A);
1807 emit_operand(dst, src);
1808 }
1809
1810 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1811 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1812 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1813 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1814 emit_int8(0x2A);
1815 emit_int8((unsigned char)(0xC0 | encode));
1816 }
1817
1818 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1819 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1820 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1821 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1822 emit_int8(0x5A);
1823 emit_int8((unsigned char)(0xC0 | encode));
1824 }
1825
1826 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1827 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1828 InstructionMark im(this);
1829 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1830 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1831 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1832 emit_int8(0x5A);
1833 emit_operand(dst, src);
1834 }
1835
1836
1837 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1838 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1839 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1840 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841 emit_int8(0x2C);
1842 emit_int8((unsigned char)(0xC0 | encode));
1843 }
1844
1845 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1846 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1847 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1848 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1849 emit_int8(0x2C);
1850 emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852
1853 void Assembler::cvttpd2dq(XMMRegister dst, XMMRegister src) {
1854 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1855 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
1856 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1857 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1858 emit_int8((unsigned char)0xE6);
1859 emit_int8((unsigned char)(0xC0 | encode));
1860 }
1861
1862 void Assembler::decl(Address dst) {
1863 // Don't use it directly. Use MacroAssembler::decrement() instead.
1864 InstructionMark im(this);
1865 prefix(dst);
1866 emit_int8((unsigned char)0xFF);
1867 emit_operand(rcx, dst);
1868 }
1869
1870 void Assembler::divsd(XMMRegister dst, Address src) {
1871 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1872 InstructionMark im(this);
1873 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1874 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1875 attributes.set_rex_vex_w_reverted();
1876 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1877 emit_int8(0x5E);
1878 emit_operand(dst, src);
1879 }
1880
1881 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1882 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1883 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1884 attributes.set_rex_vex_w_reverted();
1885 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1886 emit_int8(0x5E);
1887 emit_int8((unsigned char)(0xC0 | encode));
1888 }
1889
1890 void Assembler::divss(XMMRegister dst, Address src) {
1891 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1892 InstructionMark im(this);
1893 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1894 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1895 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1896 emit_int8(0x5E);
1897 emit_operand(dst, src);
1898 }
1899
1900 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1901 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1902 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1903 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1904 emit_int8(0x5E);
1905 emit_int8((unsigned char)(0xC0 | encode));
1906 }
1907
1908 void Assembler::emms() {
1909 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1910 emit_int8(0x0F);
1911 emit_int8(0x77);
1912 }
1913
1914 void Assembler::hlt() {
1915 emit_int8((unsigned char)0xF4);
1916 }
1917
1918 void Assembler::idivl(Register src) {
1919 int encode = prefix_and_encode(src->encoding());
1920 emit_int8((unsigned char)0xF7);
1921 emit_int8((unsigned char)(0xF8 | encode));
1922 }
1923
1924 void Assembler::divl(Register src) { // Unsigned
1925 int encode = prefix_and_encode(src->encoding());
1926 emit_int8((unsigned char)0xF7);
1927 emit_int8((unsigned char)(0xF0 | encode));
1928 }
1929
1930 void Assembler::imull(Register src) {
1931 int encode = prefix_and_encode(src->encoding());
1932 emit_int8((unsigned char)0xF7);
1933 emit_int8((unsigned char)(0xE8 | encode));
1934 }
1935
1936 void Assembler::imull(Register dst, Register src) {
1937 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1938 emit_int8(0x0F);
1939 emit_int8((unsigned char)0xAF);
1940 emit_int8((unsigned char)(0xC0 | encode));
1941 }
1942
1943
1944 void Assembler::imull(Register dst, Register src, int value) {
1945 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1946 if (is8bit(value)) {
1947 emit_int8(0x6B);
1948 emit_int8((unsigned char)(0xC0 | encode));
1949 emit_int8(value & 0xFF);
1950 } else {
1951 emit_int8(0x69);
1952 emit_int8((unsigned char)(0xC0 | encode));
1953 emit_int32(value);
1954 }
1955 }
1956
1957 void Assembler::imull(Register dst, Address src) {
1958 InstructionMark im(this);
1959 prefix(src, dst);
1960 emit_int8(0x0F);
1961 emit_int8((unsigned char) 0xAF);
1962 emit_operand(dst, src);
1963 }
1964
1965
1966 void Assembler::incl(Address dst) {
1967 // Don't use it directly. Use MacroAssembler::increment() instead.
1968 InstructionMark im(this);
1969 prefix(dst);
1970 emit_int8((unsigned char)0xFF);
1971 emit_operand(rax, dst);
1972 }
1973
1974 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1975 InstructionMark im(this);
1976 assert((0 <= cc) && (cc < 16), "illegal cc");
1977 if (L.is_bound()) {
1978 address dst = target(L);
1979 assert(dst != NULL, "jcc most probably wrong");
1980
1981 const int short_size = 2;
1982 const int long_size = 6;
1983 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1984 if (maybe_short && is8bit(offs - short_size)) {
1985 // 0111 tttn #8-bit disp
1986 emit_int8(0x70 | cc);
1987 emit_int8((offs - short_size) & 0xFF);
1988 } else {
1989 // 0000 1111 1000 tttn #32-bit disp
1990 assert(is_simm32(offs - long_size),
1991 "must be 32bit offset (call4)");
1992 emit_int8(0x0F);
1993 emit_int8((unsigned char)(0x80 | cc));
1994 emit_int32(offs - long_size);
1995 }
1996 } else {
1997 // Note: could eliminate cond. jumps to this jump if condition
1998 // is the same however, seems to be rather unlikely case.
1999 // Note: use jccb() if label to be bound is very close to get
2000 // an 8-bit displacement
2001 L.add_patch_at(code(), locator());
2002 emit_int8(0x0F);
2003 emit_int8((unsigned char)(0x80 | cc));
2004 emit_int32(0);
2005 }
2006 }
2007
2008 void Assembler::jccb(Condition cc, Label& L) {
2009 if (L.is_bound()) {
2010 const int short_size = 2;
2011 address entry = target(L);
2012 #ifdef ASSERT
2013 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2014 intptr_t delta = short_branch_delta();
2015 if (delta != 0) {
2016 dist += (dist < 0 ? (-delta) :delta);
2017 }
2018 assert(is8bit(dist), "Dispacement too large for a short jmp");
2019 #endif
2020 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
2021 // 0111 tttn #8-bit disp
2022 emit_int8(0x70 | cc);
2023 emit_int8((offs - short_size) & 0xFF);
2024 } else {
2025 InstructionMark im(this);
2026 L.add_patch_at(code(), locator());
2027 emit_int8(0x70 | cc);
2028 emit_int8(0);
2029 }
2030 }
2031
2032 void Assembler::jmp(Address adr) {
2033 InstructionMark im(this);
2034 prefix(adr);
2035 emit_int8((unsigned char)0xFF);
2036 emit_operand(rsp, adr);
2037 }
2038
2039 void Assembler::jmp(Label& L, bool maybe_short) {
2040 if (L.is_bound()) {
2041 address entry = target(L);
2042 assert(entry != NULL, "jmp most probably wrong");
2043 InstructionMark im(this);
2044 const int short_size = 2;
2045 const int long_size = 5;
2046 intptr_t offs = entry - pc();
2047 if (maybe_short && is8bit(offs - short_size)) {
2048 emit_int8((unsigned char)0xEB);
2049 emit_int8((offs - short_size) & 0xFF);
2050 } else {
2051 emit_int8((unsigned char)0xE9);
2052 emit_int32(offs - long_size);
2053 }
2054 } else {
2055 // By default, forward jumps are always 32-bit displacements, since
2056 // we can't yet know where the label will be bound. If you're sure that
2057 // the forward jump will not run beyond 256 bytes, use jmpb to
2058 // force an 8-bit displacement.
2059 InstructionMark im(this);
2060 L.add_patch_at(code(), locator());
2061 emit_int8((unsigned char)0xE9);
2062 emit_int32(0);
2063 }
2064 }
2065
2066 void Assembler::jmp(Register entry) {
2067 int encode = prefix_and_encode(entry->encoding());
2068 emit_int8((unsigned char)0xFF);
2069 emit_int8((unsigned char)(0xE0 | encode));
2070 }
2071
2072 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2073 InstructionMark im(this);
2074 emit_int8((unsigned char)0xE9);
2075 assert(dest != NULL, "must have a target");
2076 intptr_t disp = dest - (pc() + sizeof(int32_t));
2077 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2078 emit_data(disp, rspec.reloc(), call32_operand);
2079 }
2080
2081 void Assembler::jmpb(Label& L) {
2082 if (L.is_bound()) {
2083 const int short_size = 2;
2084 address entry = target(L);
2085 assert(entry != NULL, "jmp most probably wrong");
2086 #ifdef ASSERT
2087 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2088 intptr_t delta = short_branch_delta();
2089 if (delta != 0) {
2090 dist += (dist < 0 ? (-delta) :delta);
2091 }
2092 assert(is8bit(dist), "Dispacement too large for a short jmp");
2093 #endif
2094 intptr_t offs = entry - pc();
2095 emit_int8((unsigned char)0xEB);
2096 emit_int8((offs - short_size) & 0xFF);
2097 } else {
2098 InstructionMark im(this);
2099 L.add_patch_at(code(), locator());
2100 emit_int8((unsigned char)0xEB);
2101 emit_int8(0);
2102 }
2103 }
2104
2105 void Assembler::ldmxcsr( Address src) {
2106 if (UseAVX > 0 ) {
2107 InstructionMark im(this);
2108 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2109 vex_prefix(src, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2110 emit_int8((unsigned char)0xAE);
2111 emit_operand(as_Register(2), src);
2112 } else {
2113 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2114 InstructionMark im(this);
2115 prefix(src);
2116 emit_int8(0x0F);
2117 emit_int8((unsigned char)0xAE);
2118 emit_operand(as_Register(2), src);
2119 }
2120 }
2121
2122 void Assembler::leal(Register dst, Address src) {
2123 InstructionMark im(this);
2124 #ifdef _LP64
2125 emit_int8(0x67); // addr32
2126 prefix(src, dst);
2127 #endif // LP64
2128 emit_int8((unsigned char)0x8D);
2129 emit_operand(dst, src);
2130 }
2131
2132 void Assembler::lfence() {
2133 emit_int8(0x0F);
2134 emit_int8((unsigned char)0xAE);
2135 emit_int8((unsigned char)0xE8);
2136 }
2137
2138 void Assembler::lock() {
2139 emit_int8((unsigned char)0xF0);
2140 }
2141
2142 void Assembler::lzcntl(Register dst, Register src) {
2143 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2144 emit_int8((unsigned char)0xF3);
2145 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2146 emit_int8(0x0F);
2147 emit_int8((unsigned char)0xBD);
2148 emit_int8((unsigned char)(0xC0 | encode));
2149 }
2150
2151 // Emit mfence instruction
2152 void Assembler::mfence() {
2153 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2154 emit_int8(0x0F);
2155 emit_int8((unsigned char)0xAE);
2156 emit_int8((unsigned char)0xF0);
2157 }
2158
2159 void Assembler::mov(Register dst, Register src) {
2160 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2161 }
2162
2163 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2164 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2165 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2166 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2167 attributes.set_rex_vex_w_reverted();
2168 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2169 emit_int8(0x28);
2170 emit_int8((unsigned char)(0xC0 | encode));
2171 }
2172
2173 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2174 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2175 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2176 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2177 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2178 emit_int8(0x28);
2179 emit_int8((unsigned char)(0xC0 | encode));
2180 }
2181
2182 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2183 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2184 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2185 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2186 emit_int8(0x16);
2187 emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189
2190 void Assembler::movb(Register dst, Address src) {
2191 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2192 InstructionMark im(this);
2193 prefix(src, dst, true);
2194 emit_int8((unsigned char)0x8A);
2195 emit_operand(dst, src);
2196 }
2197
2198 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2199 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2200 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2201 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2202 attributes.set_rex_vex_w_reverted();
2203 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2204 emit_int8(0x12);
2205 emit_int8(0xC0 | encode);
2206 }
2207
2208 void Assembler::kmovbl(KRegister dst, Register src) {
2209 assert(VM_Version::supports_avx512dq(), "");
2210 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2211 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2212 emit_int8((unsigned char)0x92);
2213 emit_int8((unsigned char)(0xC0 | encode));
2214 }
2215
2216 void Assembler::kmovbl(Register dst, KRegister src) {
2217 assert(VM_Version::supports_avx512dq(), "");
2218 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2219 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2220 emit_int8((unsigned char)0x93);
2221 emit_int8((unsigned char)(0xC0 | encode));
2222 }
2223
2224 void Assembler::kmovwl(KRegister dst, Register src) {
2225 assert(VM_Version::supports_evex(), "");
2226 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2227 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2228 emit_int8((unsigned char)0x92);
2229 emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231
2232 void Assembler::kmovwl(Register dst, KRegister src) {
2233 assert(VM_Version::supports_evex(), "");
2234 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2235 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2236 emit_int8((unsigned char)0x93);
2237 emit_int8((unsigned char)(0xC0 | encode));
2238 }
2239
2240 void Assembler::kmovwl(KRegister dst, Address src) {
2241 assert(VM_Version::supports_evex(), "");
2242 InstructionMark im(this);
2243 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2244 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2245 emit_int8((unsigned char)0x90);
2246 emit_operand((Register)dst, src);
2247 }
2248
2249 void Assembler::kmovdl(KRegister dst, Register src) {
2250 assert(VM_Version::supports_avx512bw(), "");
2251 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2252 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2253 emit_int8((unsigned char)0x92);
2254 emit_int8((unsigned char)(0xC0 | encode));
2255 }
2256
2257 void Assembler::kmovdl(Register dst, KRegister src) {
2258 assert(VM_Version::supports_avx512bw(), "");
2259 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2260 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2261 emit_int8((unsigned char)0x93);
2262 emit_int8((unsigned char)(0xC0 | encode));
2263 }
2264
2265 void Assembler::kmovql(KRegister dst, KRegister src) {
2266 assert(VM_Version::supports_avx512bw(), "");
2267 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2268 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2269 emit_int8((unsigned char)0x90);
2270 emit_int8((unsigned char)(0xC0 | encode));
2271 }
2272
2273 void Assembler::kmovql(KRegister dst, Address src) {
2274 assert(VM_Version::supports_avx512bw(), "");
2275 InstructionMark im(this);
2276 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2277 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2278 emit_int8((unsigned char)0x90);
2279 emit_operand((Register)dst, src);
2280 }
2281
2282 void Assembler::kmovql(Address dst, KRegister src) {
2283 assert(VM_Version::supports_avx512bw(), "");
2284 InstructionMark im(this);
2285 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2286 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2287 emit_int8((unsigned char)0x90);
2288 emit_operand((Register)src, dst);
2289 }
2290
2291 void Assembler::kmovql(KRegister dst, Register src) {
2292 assert(VM_Version::supports_avx512bw(), "");
2293 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2294 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2295 emit_int8((unsigned char)0x92);
2296 emit_int8((unsigned char)(0xC0 | encode));
2297 }
2298
2299 void Assembler::kmovql(Register dst, KRegister src) {
2300 assert(VM_Version::supports_avx512bw(), "");
2301 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2302 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2303 emit_int8((unsigned char)0x93);
2304 emit_int8((unsigned char)(0xC0 | encode));
2305 }
2306
2307 void Assembler::knotwl(KRegister dst, KRegister src) {
2308 assert(VM_Version::supports_evex(), "");
2309 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2310 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2311 emit_int8((unsigned char)0x44);
2312 emit_int8((unsigned char)(0xC0 | encode));
2313 }
2314
2315 // This instruction produces ZF or CF flags
2316 void Assembler::kortestbl(KRegister src1, KRegister src2) {
2317 assert(VM_Version::supports_avx512dq(), "");
2318 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2319 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2320 emit_int8((unsigned char)0x98);
2321 emit_int8((unsigned char)(0xC0 | encode));
2322 }
2323
2324 // This instruction produces ZF or CF flags
2325 void Assembler::kortestwl(KRegister src1, KRegister src2) {
2326 assert(VM_Version::supports_evex(), "");
2327 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2328 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2329 emit_int8((unsigned char)0x98);
2330 emit_int8((unsigned char)(0xC0 | encode));
2331 }
2332
2333 // This instruction produces ZF or CF flags
2334 void Assembler::kortestdl(KRegister src1, KRegister src2) {
2335 assert(VM_Version::supports_avx512bw(), "");
2336 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2337 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2338 emit_int8((unsigned char)0x98);
2339 emit_int8((unsigned char)(0xC0 | encode));
2340 }
2341
2342 // This instruction produces ZF or CF flags
2343 void Assembler::kortestql(KRegister src1, KRegister src2) {
2344 assert(VM_Version::supports_avx512bw(), "");
2345 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2346 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2347 emit_int8((unsigned char)0x98);
2348 emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350
2351 // This instruction produces ZF or CF flags
2352 void Assembler::ktestql(KRegister src1, KRegister src2) {
2353 assert(VM_Version::supports_avx512bw(), "");
2354 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2355 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2356 emit_int8((unsigned char)0x99);
2357 emit_int8((unsigned char)(0xC0 | encode));
2358 }
2359
2360 void Assembler::ktestq(KRegister src1, KRegister src2) {
2361 assert(VM_Version::supports_avx512bw(), "");
2362 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2363 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2364 emit_int8((unsigned char)0x99);
2365 emit_int8((unsigned char)(0xC0 | encode));
2366 }
2367
2368 void Assembler::ktestd(KRegister src1, KRegister src2) {
2369 assert(VM_Version::supports_avx512bw(), "");
2370 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2371 int encode = vex_prefix_and_encode(src1->encoding(), 0, src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2372 emit_int8((unsigned char)0x99);
2373 emit_int8((unsigned char)(0xC0 | encode));
2374 }
2375
2376 void Assembler::movb(Address dst, int imm8) {
2377 InstructionMark im(this);
2378 prefix(dst);
2379 emit_int8((unsigned char)0xC6);
2380 emit_operand(rax, dst, 1);
2381 emit_int8(imm8);
2382 }
2383
2384
2385 void Assembler::movb(Address dst, Register src) {
2386 assert(src->has_byte_register(), "must have byte register");
2387 InstructionMark im(this);
2388 prefix(dst, src, true);
2389 emit_int8((unsigned char)0x88);
2390 emit_operand(src, dst);
2391 }
2392
2393 void Assembler::movdl(XMMRegister dst, Register src) {
2394 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2395 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2396 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2397 emit_int8(0x6E);
2398 emit_int8((unsigned char)(0xC0 | encode));
2399 }
2400
2401 void Assembler::movdl(Register dst, XMMRegister src) {
2402 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2403 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2404 // swap src/dst to get correct prefix
2405 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2406 emit_int8(0x7E);
2407 emit_int8((unsigned char)(0xC0 | encode));
2408 }
2409
2410 void Assembler::movdl(XMMRegister dst, Address src) {
2411 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2412 InstructionMark im(this);
2413 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2414 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2415 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2416 emit_int8(0x6E);
2417 emit_operand(dst, src);
2418 }
2419
2420 void Assembler::movdl(Address dst, XMMRegister src) {
2421 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2422 InstructionMark im(this);
2423 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2424 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2425 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2426 emit_int8(0x7E);
2427 emit_operand(src, dst);
2428 }
2429
2430 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2431 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2432 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2433 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2434 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2435 emit_int8(0x6F);
2436 emit_int8((unsigned char)(0xC0 | encode));
2437 }
2438
2439 void Assembler::movdqa(XMMRegister dst, Address src) {
2440 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2441 InstructionMark im(this);
2442 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2443 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2444 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2445 emit_int8(0x6F);
2446 emit_operand(dst, src);
2447 }
2448
2449 void Assembler::movdqu(XMMRegister dst, Address src) {
2450 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2451 InstructionMark im(this);
2452 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2453 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2454 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2455 emit_int8(0x6F);
2456 emit_operand(dst, src);
2457 }
2458
2459 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2460 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2461 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2462 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2463 emit_int8(0x6F);
2464 emit_int8((unsigned char)(0xC0 | encode));
2465 }
2466
2467 void Assembler::movdqu(Address dst, XMMRegister src) {
2468 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2469 InstructionMark im(this);
2470 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2471 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2472 attributes.reset_is_clear_context();
2473 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2474 emit_int8(0x7F);
2475 emit_operand(src, dst);
2476 }
2477
2478 // Move Unaligned 256bit Vector
2479 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2480 assert(UseAVX > 0, "");
2481 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2482 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2483 emit_int8(0x6F);
2484 emit_int8((unsigned char)(0xC0 | encode));
2485 }
2486
2487 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2488 assert(UseAVX > 0, "");
2489 InstructionMark im(this);
2490 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2491 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2492 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2493 emit_int8(0x6F);
2494 emit_operand(dst, src);
2495 }
2496
2497 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2498 assert(UseAVX > 0, "");
2499 InstructionMark im(this);
2500 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2501 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2502 attributes.reset_is_clear_context();
2503 // swap src<->dst for encoding
2504 assert(src != xnoreg, "sanity");
2505 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2506 emit_int8(0x7F);
2507 emit_operand(src, dst);
2508 }
2509
2510 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2511 void Assembler::evmovdqub(XMMRegister dst, XMMRegister src, int vector_len) {
2512 assert(VM_Version::supports_evex(), "");
2513 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2514 attributes.set_is_evex_instruction();
2515 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2516 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2517 emit_int8(0x6F);
2518 emit_int8((unsigned char)(0xC0 | encode));
2519 }
2520
2521 void Assembler::evmovdqub(XMMRegister dst, Address src, int vector_len) {
2522 assert(VM_Version::supports_evex(), "");
2523 InstructionMark im(this);
2524 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2525 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2526 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2527 attributes.set_is_evex_instruction();
2528 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2529 emit_int8(0x6F);
2530 emit_operand(dst, src);
2531 }
2532
2533 void Assembler::evmovdqub(Address dst, XMMRegister src, int vector_len) {
2534 assert(VM_Version::supports_evex(), "");
2535 assert(src != xnoreg, "sanity");
2536 InstructionMark im(this);
2537 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2538 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2539 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2540 attributes.set_is_evex_instruction();
2541 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2542 emit_int8(0x7F);
2543 emit_operand(src, dst);
2544 }
2545
2546 void Assembler::evmovdqub(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2547 assert(VM_Version::supports_avx512vlbw(), "");
2548 assert(is_vector_masking(), ""); // For stub code use only
2549 InstructionMark im(this);
2550 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2551 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2552 attributes.set_embedded_opmask_register_specifier(mask);
2553 attributes.set_is_evex_instruction();
2554 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2555 emit_int8(0x6F);
2556 emit_operand(dst, src);
2557 }
2558
2559 void Assembler::evmovdquw(XMMRegister dst, Address src, int vector_len) {
2560 assert(VM_Version::supports_evex(), "");
2561 InstructionMark im(this);
2562 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2563 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2564 attributes.set_is_evex_instruction();
2565 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2566 vex_prefix(src, 0, dst->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2567 emit_int8(0x6F);
2568 emit_operand(dst, src);
2569 }
2570
2571 void Assembler::evmovdquw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
2572 assert(is_vector_masking(), "");
2573 assert(VM_Version::supports_avx512vlbw(), "");
2574 InstructionMark im(this);
2575 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2576 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2577 attributes.set_embedded_opmask_register_specifier(mask);
2578 attributes.set_is_evex_instruction();
2579 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2580 emit_int8(0x6F);
2581 emit_operand(dst, src);
2582 }
2583
2584 void Assembler::evmovdquw(Address dst, XMMRegister src, int vector_len) {
2585 assert(VM_Version::supports_evex(), "");
2586 assert(src != xnoreg, "sanity");
2587 InstructionMark im(this);
2588 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
2589 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2590 attributes.set_is_evex_instruction();
2591 int prefix = (_legacy_mode_bw) ? VEX_SIMD_F2 : VEX_SIMD_F3;
2592 vex_prefix(dst, 0, src->encoding(), (Assembler::VexSimdPrefix)prefix, VEX_OPCODE_0F, &attributes);
2593 emit_int8(0x7F);
2594 emit_operand(src, dst);
2595 }
2596
2597 void Assembler::evmovdquw(Address dst, KRegister mask, XMMRegister src, int vector_len) {
2598 assert(VM_Version::supports_avx512vlbw(), "");
2599 assert(src != xnoreg, "sanity");
2600 InstructionMark im(this);
2601 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2602 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2603 attributes.reset_is_clear_context();
2604 attributes.set_embedded_opmask_register_specifier(mask);
2605 attributes.set_is_evex_instruction();
2606 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2607 emit_int8(0x7F);
2608 emit_operand(src, dst);
2609 }
2610
2611 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2612 assert(VM_Version::supports_evex(), "");
2613 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2614 attributes.set_is_evex_instruction();
2615 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2616 emit_int8(0x6F);
2617 emit_int8((unsigned char)(0xC0 | encode));
2618 }
2619
2620 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2621 assert(VM_Version::supports_evex(), "");
2622 InstructionMark im(this);
2623 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false , /* uses_vl */ true);
2624 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2625 attributes.set_is_evex_instruction();
2626 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2627 emit_int8(0x6F);
2628 emit_operand(dst, src);
2629 }
2630
2631 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2632 assert(VM_Version::supports_evex(), "");
2633 assert(src != xnoreg, "sanity");
2634 InstructionMark im(this);
2635 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2636 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2637 attributes.reset_is_clear_context();
2638 attributes.set_is_evex_instruction();
2639 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2640 emit_int8(0x7F);
2641 emit_operand(src, dst);
2642 }
2643
2644 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2645 assert(VM_Version::supports_evex(), "");
2646 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2647 attributes.set_is_evex_instruction();
2648 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2649 emit_int8(0x6F);
2650 emit_int8((unsigned char)(0xC0 | encode));
2651 }
2652
2653 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2654 assert(VM_Version::supports_evex(), "");
2655 InstructionMark im(this);
2656 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2657 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2658 attributes.set_is_evex_instruction();
2659 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2660 emit_int8(0x6F);
2661 emit_operand(dst, src);
2662 }
2663
2664 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2665 assert(VM_Version::supports_evex(), "");
2666 assert(src != xnoreg, "sanity");
2667 InstructionMark im(this);
2668 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2669 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2670 attributes.reset_is_clear_context();
2671 attributes.set_is_evex_instruction();
2672 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2673 emit_int8(0x7F);
2674 emit_operand(src, dst);
2675 }
2676
2677 // Uses zero extension on 64bit
2678
2679 void Assembler::movl(Register dst, int32_t imm32) {
2680 int encode = prefix_and_encode(dst->encoding());
2681 emit_int8((unsigned char)(0xB8 | encode));
2682 emit_int32(imm32);
2683 }
2684
2685 void Assembler::movl(Register dst, Register src) {
2686 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2687 emit_int8((unsigned char)0x8B);
2688 emit_int8((unsigned char)(0xC0 | encode));
2689 }
2690
2691 void Assembler::movl(Register dst, Address src) {
2692 InstructionMark im(this);
2693 prefix(src, dst);
2694 emit_int8((unsigned char)0x8B);
2695 emit_operand(dst, src);
2696 }
2697
2698 void Assembler::movl(Address dst, int32_t imm32) {
2699 InstructionMark im(this);
2700 prefix(dst);
2701 emit_int8((unsigned char)0xC7);
2702 emit_operand(rax, dst, 4);
2703 emit_int32(imm32);
2704 }
2705
2706 void Assembler::movl(Address dst, Register src) {
2707 InstructionMark im(this);
2708 prefix(dst, src);
2709 emit_int8((unsigned char)0x89);
2710 emit_operand(src, dst);
2711 }
2712
2713 // New cpus require to use movsd and movss to avoid partial register stall
2714 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2715 // The selection is done in MacroAssembler::movdbl() and movflt().
2716 void Assembler::movlpd(XMMRegister dst, Address src) {
2717 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2718 InstructionMark im(this);
2719 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2720 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2721 attributes.set_rex_vex_w_reverted();
2722 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2723 emit_int8(0x12);
2724 emit_operand(dst, src);
2725 }
2726
2727 void Assembler::movq( MMXRegister dst, Address src ) {
2728 assert( VM_Version::supports_mmx(), "" );
2729 emit_int8(0x0F);
2730 emit_int8(0x6F);
2731 emit_operand(dst, src);
2732 }
2733
2734 void Assembler::movq( Address dst, MMXRegister src ) {
2735 assert( VM_Version::supports_mmx(), "" );
2736 emit_int8(0x0F);
2737 emit_int8(0x7F);
2738 // workaround gcc (3.2.1-7a) bug
2739 // In that version of gcc with only an emit_operand(MMX, Address)
2740 // gcc will tail jump and try and reverse the parameters completely
2741 // obliterating dst in the process. By having a version available
2742 // that doesn't need to swap the args at the tail jump the bug is
2743 // avoided.
2744 emit_operand(dst, src);
2745 }
2746
2747 void Assembler::movq(XMMRegister dst, Address src) {
2748 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2749 InstructionMark im(this);
2750 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2751 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2752 attributes.set_rex_vex_w_reverted();
2753 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2754 emit_int8(0x7E);
2755 emit_operand(dst, src);
2756 }
2757
2758 void Assembler::movq(Address dst, XMMRegister src) {
2759 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2760 InstructionMark im(this);
2761 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2762 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2763 attributes.set_rex_vex_w_reverted();
2764 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2765 emit_int8((unsigned char)0xD6);
2766 emit_operand(src, dst);
2767 }
2768
2769 void Assembler::movsbl(Register dst, Address src) { // movsxb
2770 InstructionMark im(this);
2771 prefix(src, dst);
2772 emit_int8(0x0F);
2773 emit_int8((unsigned char)0xBE);
2774 emit_operand(dst, src);
2775 }
2776
2777 void Assembler::movsbl(Register dst, Register src) { // movsxb
2778 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2779 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2780 emit_int8(0x0F);
2781 emit_int8((unsigned char)0xBE);
2782 emit_int8((unsigned char)(0xC0 | encode));
2783 }
2784
2785 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2786 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2787 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2788 attributes.set_rex_vex_w_reverted();
2789 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2790 emit_int8(0x10);
2791 emit_int8((unsigned char)(0xC0 | encode));
2792 }
2793
2794 void Assembler::movsd(XMMRegister dst, Address src) {
2795 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2796 InstructionMark im(this);
2797 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2798 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2799 attributes.set_rex_vex_w_reverted();
2800 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2801 emit_int8(0x10);
2802 emit_operand(dst, src);
2803 }
2804
2805 void Assembler::movsd(Address dst, XMMRegister src) {
2806 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2807 InstructionMark im(this);
2808 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2809 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2810 attributes.reset_is_clear_context();
2811 attributes.set_rex_vex_w_reverted();
2812 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2813 emit_int8(0x11);
2814 emit_operand(src, dst);
2815 }
2816
2817 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2818 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2819 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2820 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2821 emit_int8(0x10);
2822 emit_int8((unsigned char)(0xC0 | encode));
2823 }
2824
2825 void Assembler::movss(XMMRegister dst, Address src) {
2826 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2827 InstructionMark im(this);
2828 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2829 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2830 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2831 emit_int8(0x10);
2832 emit_operand(dst, src);
2833 }
2834
2835 void Assembler::movss(Address dst, XMMRegister src) {
2836 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2837 InstructionMark im(this);
2838 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2839 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2840 attributes.reset_is_clear_context();
2841 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2842 emit_int8(0x11);
2843 emit_operand(src, dst);
2844 }
2845
2846 void Assembler::movswl(Register dst, Address src) { // movsxw
2847 InstructionMark im(this);
2848 prefix(src, dst);
2849 emit_int8(0x0F);
2850 emit_int8((unsigned char)0xBF);
2851 emit_operand(dst, src);
2852 }
2853
2854 void Assembler::movswl(Register dst, Register src) { // movsxw
2855 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2856 emit_int8(0x0F);
2857 emit_int8((unsigned char)0xBF);
2858 emit_int8((unsigned char)(0xC0 | encode));
2859 }
2860
2861 void Assembler::movw(Address dst, int imm16) {
2862 InstructionMark im(this);
2863
2864 emit_int8(0x66); // switch to 16-bit mode
2865 prefix(dst);
2866 emit_int8((unsigned char)0xC7);
2867 emit_operand(rax, dst, 2);
2868 emit_int16(imm16);
2869 }
2870
2871 void Assembler::movw(Register dst, Address src) {
2872 InstructionMark im(this);
2873 emit_int8(0x66);
2874 prefix(src, dst);
2875 emit_int8((unsigned char)0x8B);
2876 emit_operand(dst, src);
2877 }
2878
2879 void Assembler::movw(Address dst, Register src) {
2880 InstructionMark im(this);
2881 emit_int8(0x66);
2882 prefix(dst, src);
2883 emit_int8((unsigned char)0x89);
2884 emit_operand(src, dst);
2885 }
2886
2887 void Assembler::movzbl(Register dst, Address src) { // movzxb
2888 InstructionMark im(this);
2889 prefix(src, dst);
2890 emit_int8(0x0F);
2891 emit_int8((unsigned char)0xB6);
2892 emit_operand(dst, src);
2893 }
2894
2895 void Assembler::movzbl(Register dst, Register src) { // movzxb
2896 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2897 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2898 emit_int8(0x0F);
2899 emit_int8((unsigned char)0xB6);
2900 emit_int8(0xC0 | encode);
2901 }
2902
2903 void Assembler::movzwl(Register dst, Address src) { // movzxw
2904 InstructionMark im(this);
2905 prefix(src, dst);
2906 emit_int8(0x0F);
2907 emit_int8((unsigned char)0xB7);
2908 emit_operand(dst, src);
2909 }
2910
2911 void Assembler::movzwl(Register dst, Register src) { // movzxw
2912 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2913 emit_int8(0x0F);
2914 emit_int8((unsigned char)0xB7);
2915 emit_int8(0xC0 | encode);
2916 }
2917
2918 void Assembler::mull(Address src) {
2919 InstructionMark im(this);
2920 prefix(src);
2921 emit_int8((unsigned char)0xF7);
2922 emit_operand(rsp, src);
2923 }
2924
2925 void Assembler::mull(Register src) {
2926 int encode = prefix_and_encode(src->encoding());
2927 emit_int8((unsigned char)0xF7);
2928 emit_int8((unsigned char)(0xE0 | encode));
2929 }
2930
2931 void Assembler::mulsd(XMMRegister dst, Address src) {
2932 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2933 InstructionMark im(this);
2934 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2935 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2936 attributes.set_rex_vex_w_reverted();
2937 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2938 emit_int8(0x59);
2939 emit_operand(dst, src);
2940 }
2941
2942 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2943 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2944 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2945 attributes.set_rex_vex_w_reverted();
2946 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2947 emit_int8(0x59);
2948 emit_int8((unsigned char)(0xC0 | encode));
2949 }
2950
2951 void Assembler::mulss(XMMRegister dst, Address src) {
2952 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2953 InstructionMark im(this);
2954 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2955 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2956 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2957 emit_int8(0x59);
2958 emit_operand(dst, src);
2959 }
2960
2961 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2962 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2963 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2964 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2965 emit_int8(0x59);
2966 emit_int8((unsigned char)(0xC0 | encode));
2967 }
2968
2969 void Assembler::negl(Register dst) {
2970 int encode = prefix_and_encode(dst->encoding());
2971 emit_int8((unsigned char)0xF7);
2972 emit_int8((unsigned char)(0xD8 | encode));
2973 }
2974
2975 void Assembler::nop(int i) {
2976 #ifdef ASSERT
2977 assert(i > 0, " ");
2978 // The fancy nops aren't currently recognized by debuggers making it a
2979 // pain to disassemble code while debugging. If asserts are on clearly
2980 // speed is not an issue so simply use the single byte traditional nop
2981 // to do alignment.
2982
2983 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2984 return;
2985
2986 #endif // ASSERT
2987
2988 if (UseAddressNop && VM_Version::is_intel()) {
2989 //
2990 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2991 // 1: 0x90
2992 // 2: 0x66 0x90
2993 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2994 // 4: 0x0F 0x1F 0x40 0x00
2995 // 5: 0x0F 0x1F 0x44 0x00 0x00
2996 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2997 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2998 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2999 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3000 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3001 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3002
3003 // The rest coding is Intel specific - don't use consecutive address nops
3004
3005 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3006 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3007 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3008 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3009
3010 while(i >= 15) {
3011 // For Intel don't generate consecutive addess nops (mix with regular nops)
3012 i -= 15;
3013 emit_int8(0x66); // size prefix
3014 emit_int8(0x66); // size prefix
3015 emit_int8(0x66); // size prefix
3016 addr_nop_8();
3017 emit_int8(0x66); // size prefix
3018 emit_int8(0x66); // size prefix
3019 emit_int8(0x66); // size prefix
3020 emit_int8((unsigned char)0x90);
3021 // nop
3022 }
3023 switch (i) {
3024 case 14:
3025 emit_int8(0x66); // size prefix
3026 case 13:
3027 emit_int8(0x66); // size prefix
3028 case 12:
3029 addr_nop_8();
3030 emit_int8(0x66); // size prefix
3031 emit_int8(0x66); // size prefix
3032 emit_int8(0x66); // size prefix
3033 emit_int8((unsigned char)0x90);
3034 // nop
3035 break;
3036 case 11:
3037 emit_int8(0x66); // size prefix
3038 case 10:
3039 emit_int8(0x66); // size prefix
3040 case 9:
3041 emit_int8(0x66); // size prefix
3042 case 8:
3043 addr_nop_8();
3044 break;
3045 case 7:
3046 addr_nop_7();
3047 break;
3048 case 6:
3049 emit_int8(0x66); // size prefix
3050 case 5:
3051 addr_nop_5();
3052 break;
3053 case 4:
3054 addr_nop_4();
3055 break;
3056 case 3:
3057 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3058 emit_int8(0x66); // size prefix
3059 case 2:
3060 emit_int8(0x66); // size prefix
3061 case 1:
3062 emit_int8((unsigned char)0x90);
3063 // nop
3064 break;
3065 default:
3066 assert(i == 0, " ");
3067 }
3068 return;
3069 }
3070 if (UseAddressNop && VM_Version::is_amd()) {
3071 //
3072 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
3073 // 1: 0x90
3074 // 2: 0x66 0x90
3075 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3076 // 4: 0x0F 0x1F 0x40 0x00
3077 // 5: 0x0F 0x1F 0x44 0x00 0x00
3078 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3079 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3080 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3081 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3082 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3083 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3084
3085 // The rest coding is AMD specific - use consecutive address nops
3086
3087 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3088 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
3089 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3090 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3091 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3092 // Size prefixes (0x66) are added for larger sizes
3093
3094 while(i >= 22) {
3095 i -= 11;
3096 emit_int8(0x66); // size prefix
3097 emit_int8(0x66); // size prefix
3098 emit_int8(0x66); // size prefix
3099 addr_nop_8();
3100 }
3101 // Generate first nop for size between 21-12
3102 switch (i) {
3103 case 21:
3104 i -= 1;
3105 emit_int8(0x66); // size prefix
3106 case 20:
3107 case 19:
3108 i -= 1;
3109 emit_int8(0x66); // size prefix
3110 case 18:
3111 case 17:
3112 i -= 1;
3113 emit_int8(0x66); // size prefix
3114 case 16:
3115 case 15:
3116 i -= 8;
3117 addr_nop_8();
3118 break;
3119 case 14:
3120 case 13:
3121 i -= 7;
3122 addr_nop_7();
3123 break;
3124 case 12:
3125 i -= 6;
3126 emit_int8(0x66); // size prefix
3127 addr_nop_5();
3128 break;
3129 default:
3130 assert(i < 12, " ");
3131 }
3132
3133 // Generate second nop for size between 11-1
3134 switch (i) {
3135 case 11:
3136 emit_int8(0x66); // size prefix
3137 case 10:
3138 emit_int8(0x66); // size prefix
3139 case 9:
3140 emit_int8(0x66); // size prefix
3141 case 8:
3142 addr_nop_8();
3143 break;
3144 case 7:
3145 addr_nop_7();
3146 break;
3147 case 6:
3148 emit_int8(0x66); // size prefix
3149 case 5:
3150 addr_nop_5();
3151 break;
3152 case 4:
3153 addr_nop_4();
3154 break;
3155 case 3:
3156 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3157 emit_int8(0x66); // size prefix
3158 case 2:
3159 emit_int8(0x66); // size prefix
3160 case 1:
3161 emit_int8((unsigned char)0x90);
3162 // nop
3163 break;
3164 default:
3165 assert(i == 0, " ");
3166 }
3167 return;
3168 }
3169
3170 if (UseAddressNop && VM_Version::is_zx()) {
3171 //
3172 // Using multi-bytes nops "0x0F 0x1F [address]" for ZX
3173 // 1: 0x90
3174 // 2: 0x66 0x90
3175 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
3176 // 4: 0x0F 0x1F 0x40 0x00
3177 // 5: 0x0F 0x1F 0x44 0x00 0x00
3178 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
3179 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
3180 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3181 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3182 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3183 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
3184
3185 // The rest coding is ZX specific - don't use consecutive address nops
3186
3187 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3188 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3189 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3190 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
3191
3192 while (i >= 15) {
3193 // For ZX don't generate consecutive addess nops (mix with regular nops)
3194 i -= 15;
3195 emit_int8(0x66); // size prefix
3196 emit_int8(0x66); // size prefix
3197 emit_int8(0x66); // size prefix
3198 addr_nop_8();
3199 emit_int8(0x66); // size prefix
3200 emit_int8(0x66); // size prefix
3201 emit_int8(0x66); // size prefix
3202 emit_int8((unsigned char)0x90);
3203 // nop
3204 }
3205 switch (i) {
3206 case 14:
3207 emit_int8(0x66); // size prefix
3208 case 13:
3209 emit_int8(0x66); // size prefix
3210 case 12:
3211 addr_nop_8();
3212 emit_int8(0x66); // size prefix
3213 emit_int8(0x66); // size prefix
3214 emit_int8(0x66); // size prefix
3215 emit_int8((unsigned char)0x90);
3216 // nop
3217 break;
3218 case 11:
3219 emit_int8(0x66); // size prefix
3220 case 10:
3221 emit_int8(0x66); // size prefix
3222 case 9:
3223 emit_int8(0x66); // size prefix
3224 case 8:
3225 addr_nop_8();
3226 break;
3227 case 7:
3228 addr_nop_7();
3229 break;
3230 case 6:
3231 emit_int8(0x66); // size prefix
3232 case 5:
3233 addr_nop_5();
3234 break;
3235 case 4:
3236 addr_nop_4();
3237 break;
3238 case 3:
3239 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
3240 emit_int8(0x66); // size prefix
3241 case 2:
3242 emit_int8(0x66); // size prefix
3243 case 1:
3244 emit_int8((unsigned char)0x90);
3245 // nop
3246 break;
3247 default:
3248 assert(i == 0, " ");
3249 }
3250 return;
3251 }
3252
3253 // Using nops with size prefixes "0x66 0x90".
3254 // From AMD Optimization Guide:
3255 // 1: 0x90
3256 // 2: 0x66 0x90
3257 // 3: 0x66 0x66 0x90
3258 // 4: 0x66 0x66 0x66 0x90
3259 // 5: 0x66 0x66 0x90 0x66 0x90
3260 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
3261 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
3262 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
3263 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3264 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
3265 //
3266 while(i > 12) {
3267 i -= 4;
3268 emit_int8(0x66); // size prefix
3269 emit_int8(0x66);
3270 emit_int8(0x66);
3271 emit_int8((unsigned char)0x90);
3272 // nop
3273 }
3274 // 1 - 12 nops
3275 if(i > 8) {
3276 if(i > 9) {
3277 i -= 1;
3278 emit_int8(0x66);
3279 }
3280 i -= 3;
3281 emit_int8(0x66);
3282 emit_int8(0x66);
3283 emit_int8((unsigned char)0x90);
3284 }
3285 // 1 - 8 nops
3286 if(i > 4) {
3287 if(i > 6) {
3288 i -= 1;
3289 emit_int8(0x66);
3290 }
3291 i -= 3;
3292 emit_int8(0x66);
3293 emit_int8(0x66);
3294 emit_int8((unsigned char)0x90);
3295 }
3296 switch (i) {
3297 case 4:
3298 emit_int8(0x66);
3299 case 3:
3300 emit_int8(0x66);
3301 case 2:
3302 emit_int8(0x66);
3303 case 1:
3304 emit_int8((unsigned char)0x90);
3305 break;
3306 default:
3307 assert(i == 0, " ");
3308 }
3309 }
3310
3311 void Assembler::notl(Register dst) {
3312 int encode = prefix_and_encode(dst->encoding());
3313 emit_int8((unsigned char)0xF7);
3314 emit_int8((unsigned char)(0xD0 | encode));
3315 }
3316
3317 void Assembler::orl(Address dst, int32_t imm32) {
3318 InstructionMark im(this);
3319 prefix(dst);
3320 emit_arith_operand(0x81, rcx, dst, imm32);
3321 }
3322
3323 void Assembler::orl(Register dst, int32_t imm32) {
3324 prefix(dst);
3325 emit_arith(0x81, 0xC8, dst, imm32);
3326 }
3327
3328 void Assembler::orl(Register dst, Address src) {
3329 InstructionMark im(this);
3330 prefix(src, dst);
3331 emit_int8(0x0B);
3332 emit_operand(dst, src);
3333 }
3334
3335 void Assembler::orl(Register dst, Register src) {
3336 (void) prefix_and_encode(dst->encoding(), src->encoding());
3337 emit_arith(0x0B, 0xC0, dst, src);
3338 }
3339
3340 void Assembler::orl(Address dst, Register src) {
3341 InstructionMark im(this);
3342 prefix(dst, src);
3343 emit_int8(0x09);
3344 emit_operand(src, dst);
3345 }
3346
3347 void Assembler::packuswb(XMMRegister dst, Address src) {
3348 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3349 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3350 InstructionMark im(this);
3351 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3352 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3353 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3354 emit_int8(0x67);
3355 emit_operand(dst, src);
3356 }
3357
3358 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
3359 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3360 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3361 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3362 emit_int8(0x67);
3363 emit_int8((unsigned char)(0xC0 | encode));
3364 }
3365
3366 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3367 assert(UseAVX > 0, "some form of AVX must be enabled");
3368 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
3369 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3370 emit_int8(0x67);
3371 emit_int8((unsigned char)(0xC0 | encode));
3372 }
3373
3374 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3375 assert(VM_Version::supports_avx2(), "");
3376 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3377 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3378 emit_int8(0x00);
3379 emit_int8(0xC0 | encode);
3380 emit_int8(imm8);
3381 }
3382
3383 void Assembler::vperm2i128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3384 assert(VM_Version::supports_avx2(), "");
3385 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3386 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3387 emit_int8(0x46);
3388 emit_int8(0xC0 | encode);
3389 emit_int8(imm8);
3390 }
3391
3392 void Assembler::vperm2f128(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8) {
3393 assert(VM_Version::supports_avx(), "");
3394 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3395 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3396 emit_int8(0x06);
3397 emit_int8(0xC0 | encode);
3398 emit_int8(imm8);
3399 }
3400
3401
3402 void Assembler::pause() {
3403 emit_int8((unsigned char)0xF3);
3404 emit_int8((unsigned char)0x90);
3405 }
3406
3407 void Assembler::ud2() {
3408 emit_int8(0x0F);
3409 emit_int8(0x0B);
3410 }
3411
3412 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3413 assert(VM_Version::supports_sse4_2(), "");
3414 InstructionMark im(this);
3415 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3416 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3417 emit_int8(0x61);
3418 emit_operand(dst, src);
3419 emit_int8(imm8);
3420 }
3421
3422 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3423 assert(VM_Version::supports_sse4_2(), "");
3424 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3425 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3426 emit_int8(0x61);
3427 emit_int8((unsigned char)(0xC0 | encode));
3428 emit_int8(imm8);
3429 }
3430
3431 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3432 void Assembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3433 assert(VM_Version::supports_sse2(), "");
3434 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3435 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3436 emit_int8(0x74);
3437 emit_int8((unsigned char)(0xC0 | encode));
3438 }
3439
3440 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3441 void Assembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3442 assert(VM_Version::supports_avx(), "");
3443 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3444 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3445 emit_int8(0x74);
3446 emit_int8((unsigned char)(0xC0 | encode));
3447 }
3448
3449 // In this context, kdst is written the mask used to process the equal components
3450 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3451 assert(VM_Version::supports_avx512bw(), "");
3452 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3453 attributes.set_is_evex_instruction();
3454 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3455 emit_int8(0x74);
3456 emit_int8((unsigned char)(0xC0 | encode));
3457 }
3458
3459 void Assembler::evpcmpgtb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3460 assert(VM_Version::supports_avx512vlbw(), "");
3461 InstructionMark im(this);
3462 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3463 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3464 attributes.set_is_evex_instruction();
3465 int dst_enc = kdst->encoding();
3466 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3467 emit_int8(0x64);
3468 emit_operand(as_Register(dst_enc), src);
3469 }
3470
3471 void Assembler::evpcmpgtb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3472 assert(is_vector_masking(), "");
3473 assert(VM_Version::supports_avx512vlbw(), "");
3474 InstructionMark im(this);
3475 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3476 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3477 attributes.reset_is_clear_context();
3478 attributes.set_embedded_opmask_register_specifier(mask);
3479 attributes.set_is_evex_instruction();
3480 int dst_enc = kdst->encoding();
3481 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3482 emit_int8(0x64);
3483 emit_operand(as_Register(dst_enc), src);
3484 }
3485
3486 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3487 assert(VM_Version::supports_avx512vlbw(), "");
3488 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3489 attributes.set_is_evex_instruction();
3490 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3491 emit_int8(0x3E);
3492 emit_int8((unsigned char)(0xC0 | encode));
3493 emit_int8(vcc);
3494 }
3495
3496 void Assembler::evpcmpuw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src, ComparisonPredicate vcc, int vector_len) {
3497 assert(is_vector_masking(), "");
3498 assert(VM_Version::supports_avx512vlbw(), "");
3499 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3500 attributes.reset_is_clear_context();
3501 attributes.set_embedded_opmask_register_specifier(mask);
3502 attributes.set_is_evex_instruction();
3503 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3504 emit_int8(0x3E);
3505 emit_int8((unsigned char)(0xC0 | encode));
3506 emit_int8(vcc);
3507 }
3508
3509 void Assembler::evpcmpuw(KRegister kdst, XMMRegister nds, Address src, ComparisonPredicate vcc, int vector_len) {
3510 assert(VM_Version::supports_avx512vlbw(), "");
3511 InstructionMark im(this);
3512 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3513 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3514 attributes.set_is_evex_instruction();
3515 int dst_enc = kdst->encoding();
3516 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3517 emit_int8(0x3E);
3518 emit_operand(as_Register(dst_enc), src);
3519 emit_int8(vcc);
3520 }
3521
3522 void Assembler::evpcmpeqb(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3523 assert(VM_Version::supports_avx512bw(), "");
3524 InstructionMark im(this);
3525 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3526 attributes.set_is_evex_instruction();
3527 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3528 int dst_enc = kdst->encoding();
3529 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3530 emit_int8(0x74);
3531 emit_operand(as_Register(dst_enc), src);
3532 }
3533
3534 void Assembler::evpcmpeqb(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int vector_len) {
3535 assert(VM_Version::supports_avx512vlbw(), "");
3536 assert(is_vector_masking(), ""); // For stub code use only
3537 InstructionMark im(this);
3538 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_reg_mask */ false, /* uses_vl */ false);
3539 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3540 attributes.reset_is_clear_context();
3541 attributes.set_embedded_opmask_register_specifier(mask);
3542 attributes.set_is_evex_instruction();
3543 vex_prefix(src, nds->encoding(), kdst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3544 emit_int8(0x74);
3545 emit_operand(as_Register(kdst->encoding()), src);
3546 }
3547
3548 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3549 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3550 assert(VM_Version::supports_sse2(), "");
3551 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3552 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3553 emit_int8(0x75);
3554 emit_int8((unsigned char)(0xC0 | encode));
3555 }
3556
3557 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3558 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3559 assert(VM_Version::supports_avx(), "");
3560 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3561 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3562 emit_int8(0x75);
3563 emit_int8((unsigned char)(0xC0 | encode));
3564 }
3565
3566 // In this context, kdst is written the mask used to process the equal components
3567 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3568 assert(VM_Version::supports_avx512bw(), "");
3569 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3570 attributes.set_is_evex_instruction();
3571 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3572 emit_int8(0x75);
3573 emit_int8((unsigned char)(0xC0 | encode));
3574 }
3575
3576 void Assembler::evpcmpeqw(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3577 assert(VM_Version::supports_avx512bw(), "");
3578 InstructionMark im(this);
3579 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
3580 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3581 attributes.set_is_evex_instruction();
3582 int dst_enc = kdst->encoding();
3583 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3584 emit_int8(0x75);
3585 emit_operand(as_Register(dst_enc), src);
3586 }
3587
3588 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3589 void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
3590 assert(VM_Version::supports_sse2(), "");
3591 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3592 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3593 emit_int8(0x76);
3594 emit_int8((unsigned char)(0xC0 | encode));
3595 }
3596
3597 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3598 void Assembler::vpcmpeqd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3599 assert(VM_Version::supports_avx(), "");
3600 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3601 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3602 emit_int8(0x76);
3603 emit_int8((unsigned char)(0xC0 | encode));
3604 }
3605
3606 // In this context, kdst is written the mask used to process the equal components
3607 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3608 assert(VM_Version::supports_evex(), "");
3609 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3610 attributes.set_is_evex_instruction();
3611 attributes.reset_is_clear_context();
3612 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3613 emit_int8(0x76);
3614 emit_int8((unsigned char)(0xC0 | encode));
3615 }
3616
3617 void Assembler::evpcmpeqd(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3618 assert(VM_Version::supports_evex(), "");
3619 InstructionMark im(this);
3620 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3621 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3622 attributes.reset_is_clear_context();
3623 attributes.set_is_evex_instruction();
3624 int dst_enc = kdst->encoding();
3625 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3626 emit_int8(0x76);
3627 emit_operand(as_Register(dst_enc), src);
3628 }
3629
3630 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3631 void Assembler::pcmpeqq(XMMRegister dst, XMMRegister src) {
3632 assert(VM_Version::supports_sse4_1(), "");
3633 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3634 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3635 emit_int8(0x29);
3636 emit_int8((unsigned char)(0xC0 | encode));
3637 }
3638
3639 // In this context, the dst vector contains the components that are equal, non equal components are zeroed in dst
3640 void Assembler::vpcmpeqq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3641 assert(VM_Version::supports_avx(), "");
3642 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3643 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3644 emit_int8(0x29);
3645 emit_int8((unsigned char)(0xC0 | encode));
3646 }
3647
3648 // In this context, kdst is written the mask used to process the equal components
3649 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, XMMRegister src, int vector_len) {
3650 assert(VM_Version::supports_evex(), "");
3651 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3652 attributes.reset_is_clear_context();
3653 attributes.set_is_evex_instruction();
3654 int encode = vex_prefix_and_encode(kdst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3655 emit_int8(0x29);
3656 emit_int8((unsigned char)(0xC0 | encode));
3657 }
3658
3659 // In this context, kdst is written the mask used to process the equal components
3660 void Assembler::evpcmpeqq(KRegister kdst, XMMRegister nds, Address src, int vector_len) {
3661 assert(VM_Version::supports_evex(), "");
3662 InstructionMark im(this);
3663 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3664 attributes.reset_is_clear_context();
3665 attributes.set_is_evex_instruction();
3666 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
3667 int dst_enc = kdst->encoding();
3668 vex_prefix(src, nds->encoding(), dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3669 emit_int8(0x29);
3670 emit_operand(as_Register(dst_enc), src);
3671 }
3672
3673 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3674 assert(VM_Version::supports_sse2(), "");
3675 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3676 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3677 emit_int8((unsigned char)0xD7);
3678 emit_int8((unsigned char)(0xC0 | encode));
3679 }
3680
3681 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3682 assert(VM_Version::supports_avx2(), "");
3683 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3684 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3685 emit_int8((unsigned char)0xD7);
3686 emit_int8((unsigned char)(0xC0 | encode));
3687 }
3688
3689 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3690 assert(VM_Version::supports_sse4_1(), "");
3691 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3692 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3693 emit_int8(0x16);
3694 emit_int8((unsigned char)(0xC0 | encode));
3695 emit_int8(imm8);
3696 }
3697
3698 void Assembler::pextrd(Address dst, XMMRegister src, int imm8) {
3699 assert(VM_Version::supports_sse4_1(), "");
3700 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3701 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3702 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3703 emit_int8(0x16);
3704 emit_operand(src, dst);
3705 emit_int8(imm8);
3706 }
3707
3708 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3709 assert(VM_Version::supports_sse4_1(), "");
3710 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3711 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3712 emit_int8(0x16);
3713 emit_int8((unsigned char)(0xC0 | encode));
3714 emit_int8(imm8);
3715 }
3716
3717 void Assembler::pextrq(Address dst, XMMRegister src, int imm8) {
3718 assert(VM_Version::supports_sse4_1(), "");
3719 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3720 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3721 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3722 emit_int8(0x16);
3723 emit_operand(src, dst);
3724 emit_int8(imm8);
3725 }
3726
3727 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3728 assert(VM_Version::supports_sse2(), "");
3729 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3730 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3731 emit_int8((unsigned char)0xC5);
3732 emit_int8((unsigned char)(0xC0 | encode));
3733 emit_int8(imm8);
3734 }
3735
3736 void Assembler::pextrw(Address dst, XMMRegister src, int imm8) {
3737 assert(VM_Version::supports_sse4_1(), "");
3738 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3739 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3740 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3741 emit_int8((unsigned char)0x15);
3742 emit_operand(src, dst);
3743 emit_int8(imm8);
3744 }
3745
3746 void Assembler::pextrb(Address dst, XMMRegister src, int imm8) {
3747 assert(VM_Version::supports_sse4_1(), "");
3748 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3749 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3750 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3751 emit_int8(0x14);
3752 emit_operand(src, dst);
3753 emit_int8(imm8);
3754 }
3755
3756 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3757 assert(VM_Version::supports_sse4_1(), "");
3758 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3759 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3760 emit_int8(0x22);
3761 emit_int8((unsigned char)(0xC0 | encode));
3762 emit_int8(imm8);
3763 }
3764
3765 void Assembler::pinsrd(XMMRegister dst, Address src, int imm8) {
3766 assert(VM_Version::supports_sse4_1(), "");
3767 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3768 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3769 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3770 emit_int8(0x22);
3771 emit_operand(dst,src);
3772 emit_int8(imm8);
3773 }
3774
3775 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3776 assert(VM_Version::supports_sse4_1(), "");
3777 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3778 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3779 emit_int8(0x22);
3780 emit_int8((unsigned char)(0xC0 | encode));
3781 emit_int8(imm8);
3782 }
3783
3784 void Assembler::pinsrq(XMMRegister dst, Address src, int imm8) {
3785 assert(VM_Version::supports_sse4_1(), "");
3786 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3787 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3788 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3789 emit_int8(0x22);
3790 emit_operand(dst, src);
3791 emit_int8(imm8);
3792 }
3793
3794 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3795 assert(VM_Version::supports_sse2(), "");
3796 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3797 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3798 emit_int8((unsigned char)0xC4);
3799 emit_int8((unsigned char)(0xC0 | encode));
3800 emit_int8(imm8);
3801 }
3802
3803 void Assembler::pinsrw(XMMRegister dst, Address src, int imm8) {
3804 assert(VM_Version::supports_sse2(), "");
3805 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3806 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
3807 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3808 emit_int8((unsigned char)0xC4);
3809 emit_operand(dst, src);
3810 emit_int8(imm8);
3811 }
3812
3813 void Assembler::pinsrb(XMMRegister dst, Address src, int imm8) {
3814 assert(VM_Version::supports_sse4_1(), "");
3815 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3816 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
3817 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3818 emit_int8(0x20);
3819 emit_operand(dst, src);
3820 emit_int8(imm8);
3821 }
3822
3823 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3824 assert(VM_Version::supports_sse4_1(), "");
3825 InstructionMark im(this);
3826 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3827 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3828 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3829 emit_int8(0x30);
3830 emit_operand(dst, src);
3831 }
3832
3833 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3834 assert(VM_Version::supports_sse4_1(), "");
3835 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3836 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3837 emit_int8(0x30);
3838 emit_int8((unsigned char)(0xC0 | encode));
3839 }
3840
3841 void Assembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
3842 assert(VM_Version::supports_avx(), "");
3843 InstructionMark im(this);
3844 assert(dst != xnoreg, "sanity");
3845 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3846 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3847 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3848 emit_int8(0x30);
3849 emit_operand(dst, src);
3850 }
3851
3852 void Assembler::evpmovzxbw(XMMRegister dst, KRegister mask, Address src, int vector_len) {
3853 assert(is_vector_masking(), "");
3854 assert(VM_Version::supports_avx512vlbw(), "");
3855 assert(dst != xnoreg, "sanity");
3856 InstructionMark im(this);
3857 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3858 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3859 attributes.set_embedded_opmask_register_specifier(mask);
3860 attributes.set_is_evex_instruction();
3861 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3862 emit_int8(0x30);
3863 emit_operand(dst, src);
3864 }
3865
3866 void Assembler::evpmovwb(Address dst, XMMRegister src, int vector_len) {
3867 assert(VM_Version::supports_avx512vlbw(), "");
3868 assert(src != xnoreg, "sanity");
3869 InstructionMark im(this);
3870 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3871 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3872 attributes.set_is_evex_instruction();
3873 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3874 emit_int8(0x30);
3875 emit_operand(src, dst);
3876 }
3877
3878 void Assembler::evpmovwb(Address dst, KRegister mask, XMMRegister src, int vector_len) {
3879 assert(is_vector_masking(), "");
3880 assert(VM_Version::supports_avx512vlbw(), "");
3881 assert(src != xnoreg, "sanity");
3882 InstructionMark im(this);
3883 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
3884 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3885 attributes.reset_is_clear_context();
3886 attributes.set_embedded_opmask_register_specifier(mask);
3887 attributes.set_is_evex_instruction();
3888 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes);
3889 emit_int8(0x30);
3890 emit_operand(src, dst);
3891 }
3892
3893 // generic
3894 void Assembler::pop(Register dst) {
3895 int encode = prefix_and_encode(dst->encoding());
3896 emit_int8(0x58 | encode);
3897 }
3898
3899 void Assembler::popcntl(Register dst, Address src) {
3900 assert(VM_Version::supports_popcnt(), "must support");
3901 InstructionMark im(this);
3902 emit_int8((unsigned char)0xF3);
3903 prefix(src, dst);
3904 emit_int8(0x0F);
3905 emit_int8((unsigned char)0xB8);
3906 emit_operand(dst, src);
3907 }
3908
3909 void Assembler::popcntl(Register dst, Register src) {
3910 assert(VM_Version::supports_popcnt(), "must support");
3911 emit_int8((unsigned char)0xF3);
3912 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3913 emit_int8(0x0F);
3914 emit_int8((unsigned char)0xB8);
3915 emit_int8((unsigned char)(0xC0 | encode));
3916 }
3917
3918 void Assembler::vpopcntd(XMMRegister dst, XMMRegister src, int vector_len) {
3919 assert(VM_Version::supports_vpopcntdq(), "must support vpopcntdq feature");
3920 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3921 attributes.set_is_evex_instruction();
3922 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3923 emit_int8(0x55);
3924 emit_int8((unsigned char)(0xC0 | encode));
3925 }
3926
3927 void Assembler::popf() {
3928 emit_int8((unsigned char)0x9D);
3929 }
3930
3931 #ifndef _LP64 // no 32bit push/pop on amd64
3932 void Assembler::popl(Address dst) {
3933 // NOTE: this will adjust stack by 8byte on 64bits
3934 InstructionMark im(this);
3935 prefix(dst);
3936 emit_int8((unsigned char)0x8F);
3937 emit_operand(rax, dst);
3938 }
3939 #endif
3940
3941 void Assembler::prefetch_prefix(Address src) {
3942 prefix(src);
3943 emit_int8(0x0F);
3944 }
3945
3946 void Assembler::prefetchnta(Address src) {
3947 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3948 InstructionMark im(this);
3949 prefetch_prefix(src);
3950 emit_int8(0x18);
3951 emit_operand(rax, src); // 0, src
3952 }
3953
3954 void Assembler::prefetchr(Address src) {
3955 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3956 InstructionMark im(this);
3957 prefetch_prefix(src);
3958 emit_int8(0x0D);
3959 emit_operand(rax, src); // 0, src
3960 }
3961
3962 void Assembler::prefetcht0(Address src) {
3963 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3964 InstructionMark im(this);
3965 prefetch_prefix(src);
3966 emit_int8(0x18);
3967 emit_operand(rcx, src); // 1, src
3968 }
3969
3970 void Assembler::prefetcht1(Address src) {
3971 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3972 InstructionMark im(this);
3973 prefetch_prefix(src);
3974 emit_int8(0x18);
3975 emit_operand(rdx, src); // 2, src
3976 }
3977
3978 void Assembler::prefetcht2(Address src) {
3979 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3980 InstructionMark im(this);
3981 prefetch_prefix(src);
3982 emit_int8(0x18);
3983 emit_operand(rbx, src); // 3, src
3984 }
3985
3986 void Assembler::prefetchw(Address src) {
3987 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3988 InstructionMark im(this);
3989 prefetch_prefix(src);
3990 emit_int8(0x0D);
3991 emit_operand(rcx, src); // 1, src
3992 }
3993
3994 void Assembler::prefix(Prefix p) {
3995 emit_int8(p);
3996 }
3997
3998 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3999 assert(VM_Version::supports_ssse3(), "");
4000 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4001 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4002 emit_int8(0x00);
4003 emit_int8((unsigned char)(0xC0 | encode));
4004 }
4005
4006 void Assembler::vpshufb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4007 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4008 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4009 0, "");
4010 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
4011 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4012 emit_int8(0x00);
4013 emit_int8((unsigned char)(0xC0 | encode));
4014 }
4015
4016 void Assembler::pshufb(XMMRegister dst, Address src) {
4017 assert(VM_Version::supports_ssse3(), "");
4018 InstructionMark im(this);
4019 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4020 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4021 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4022 emit_int8(0x00);
4023 emit_operand(dst, src);
4024 }
4025
4026 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
4027 assert(isByte(mode), "invalid value");
4028 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4029 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
4030 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4031 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4032 emit_int8(0x70);
4033 emit_int8((unsigned char)(0xC0 | encode));
4034 emit_int8(mode & 0xFF);
4035 }
4036
4037 void Assembler::vpshufd(XMMRegister dst, XMMRegister src, int mode, int vector_len) {
4038 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4039 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4040 0, "");
4041 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4042 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4043 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4044 emit_int8(0x70);
4045 emit_int8((unsigned char)(0xC0 | encode));
4046 emit_int8(mode & 0xFF);
4047 }
4048
4049 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
4050 assert(isByte(mode), "invalid value");
4051 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4052 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4053 InstructionMark im(this);
4054 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4055 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4056 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4057 emit_int8(0x70);
4058 emit_operand(dst, src);
4059 emit_int8(mode & 0xFF);
4060 }
4061
4062 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4063 assert(isByte(mode), "invalid value");
4064 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4065 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4066 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4067 emit_int8(0x70);
4068 emit_int8((unsigned char)(0xC0 | encode));
4069 emit_int8(mode & 0xFF);
4070 }
4071
4072 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
4073 assert(isByte(mode), "invalid value");
4074 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4075 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4076 InstructionMark im(this);
4077 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4078 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4079 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4080 emit_int8(0x70);
4081 emit_operand(dst, src);
4082 emit_int8(mode & 0xFF);
4083 }
4084
4085 void Assembler::psrldq(XMMRegister dst, int shift) {
4086 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4087 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4088 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4089 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4090 emit_int8(0x73);
4091 emit_int8((unsigned char)(0xC0 | encode));
4092 emit_int8(shift);
4093 }
4094
4095 void Assembler::pslldq(XMMRegister dst, int shift) {
4096 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
4097 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4098 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
4099 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
4100 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4101 emit_int8(0x73);
4102 emit_int8((unsigned char)(0xC0 | encode));
4103 emit_int8(shift);
4104 }
4105
4106 void Assembler::ptest(XMMRegister dst, Address src) {
4107 assert(VM_Version::supports_sse4_1(), "");
4108 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4109 InstructionMark im(this);
4110 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4111 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4112 emit_int8(0x17);
4113 emit_operand(dst, src);
4114 }
4115
4116 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
4117 assert(VM_Version::supports_sse4_1(), "");
4118 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4119 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4120 emit_int8(0x17);
4121 emit_int8((unsigned char)(0xC0 | encode));
4122 }
4123
4124 void Assembler::vptest(XMMRegister dst, Address src) {
4125 assert(VM_Version::supports_avx(), "");
4126 InstructionMark im(this);
4127 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4128 assert(dst != xnoreg, "sanity");
4129 // swap src<->dst for encoding
4130 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4131 emit_int8(0x17);
4132 emit_operand(dst, src);
4133 }
4134
4135 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
4136 assert(VM_Version::supports_avx(), "");
4137 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4138 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4139 emit_int8(0x17);
4140 emit_int8((unsigned char)(0xC0 | encode));
4141 }
4142
4143 void Assembler::punpcklbw(XMMRegister dst, Address src) {
4144 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4145 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4146 InstructionMark im(this);
4147 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4148 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4149 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4150 emit_int8(0x60);
4151 emit_operand(dst, src);
4152 }
4153
4154 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4155 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4156 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
4157 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4158 emit_int8(0x60);
4159 emit_int8((unsigned char)(0xC0 | encode));
4160 }
4161
4162 void Assembler::punpckldq(XMMRegister dst, Address src) {
4163 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4164 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
4165 InstructionMark im(this);
4166 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4167 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4168 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4169 emit_int8(0x62);
4170 emit_operand(dst, src);
4171 }
4172
4173 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
4174 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4175 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4176 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4177 emit_int8(0x62);
4178 emit_int8((unsigned char)(0xC0 | encode));
4179 }
4180
4181 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
4182 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4183 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4184 attributes.set_rex_vex_w_reverted();
4185 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4186 emit_int8(0x6C);
4187 emit_int8((unsigned char)(0xC0 | encode));
4188 }
4189
4190 void Assembler::push(int32_t imm32) {
4191 // in 64bits we push 64bits onto the stack but only
4192 // take a 32bit immediate
4193 emit_int8(0x68);
4194 emit_int32(imm32);
4195 }
4196
4197 void Assembler::push(Register src) {
4198 int encode = prefix_and_encode(src->encoding());
4199
4200 emit_int8(0x50 | encode);
4201 }
4202
4203 void Assembler::pushf() {
4204 emit_int8((unsigned char)0x9C);
4205 }
4206
4207 #ifndef _LP64 // no 32bit push/pop on amd64
4208 void Assembler::pushl(Address src) {
4209 // Note this will push 64bit on 64bit
4210 InstructionMark im(this);
4211 prefix(src);
4212 emit_int8((unsigned char)0xFF);
4213 emit_operand(rsi, src);
4214 }
4215 #endif
4216
4217 void Assembler::rcll(Register dst, int imm8) {
4218 assert(isShiftCount(imm8), "illegal shift count");
4219 int encode = prefix_and_encode(dst->encoding());
4220 if (imm8 == 1) {
4221 emit_int8((unsigned char)0xD1);
4222 emit_int8((unsigned char)(0xD0 | encode));
4223 } else {
4224 emit_int8((unsigned char)0xC1);
4225 emit_int8((unsigned char)0xD0 | encode);
4226 emit_int8(imm8);
4227 }
4228 }
4229
4230 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
4231 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4232 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4233 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4234 emit_int8(0x53);
4235 emit_int8((unsigned char)(0xC0 | encode));
4236 }
4237
4238 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
4239 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4240 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4241 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4242 emit_int8(0x53);
4243 emit_int8((unsigned char)(0xC0 | encode));
4244 }
4245
4246 void Assembler::rdtsc() {
4247 emit_int8((unsigned char)0x0F);
4248 emit_int8((unsigned char)0x31);
4249 }
4250
4251 // copies data from [esi] to [edi] using rcx pointer sized words
4252 // generic
4253 void Assembler::rep_mov() {
4254 emit_int8((unsigned char)0xF3);
4255 // MOVSQ
4256 LP64_ONLY(prefix(REX_W));
4257 emit_int8((unsigned char)0xA5);
4258 }
4259
4260 // sets rcx bytes with rax, value at [edi]
4261 void Assembler::rep_stosb() {
4262 emit_int8((unsigned char)0xF3); // REP
4263 LP64_ONLY(prefix(REX_W));
4264 emit_int8((unsigned char)0xAA); // STOSB
4265 }
4266
4267 // sets rcx pointer sized words with rax, value at [edi]
4268 // generic
4269 void Assembler::rep_stos() {
4270 emit_int8((unsigned char)0xF3); // REP
4271 LP64_ONLY(prefix(REX_W)); // LP64:STOSQ, LP32:STOSD
4272 emit_int8((unsigned char)0xAB);
4273 }
4274
4275 // scans rcx pointer sized words at [edi] for occurance of rax,
4276 // generic
4277 void Assembler::repne_scan() { // repne_scan
4278 emit_int8((unsigned char)0xF2);
4279 // SCASQ
4280 LP64_ONLY(prefix(REX_W));
4281 emit_int8((unsigned char)0xAF);
4282 }
4283
4284 #ifdef _LP64
4285 // scans rcx 4 byte words at [edi] for occurance of rax,
4286 // generic
4287 void Assembler::repne_scanl() { // repne_scan
4288 emit_int8((unsigned char)0xF2);
4289 // SCASL
4290 emit_int8((unsigned char)0xAF);
4291 }
4292 #endif
4293
4294 void Assembler::ret(int imm16) {
4295 if (imm16 == 0) {
4296 emit_int8((unsigned char)0xC3);
4297 } else {
4298 emit_int8((unsigned char)0xC2);
4299 emit_int16(imm16);
4300 }
4301 }
4302
4303 void Assembler::sahf() {
4304 #ifdef _LP64
4305 // Not supported in 64bit mode
4306 ShouldNotReachHere();
4307 #endif
4308 emit_int8((unsigned char)0x9E);
4309 }
4310
4311 void Assembler::sarl(Register dst, int imm8) {
4312 int encode = prefix_and_encode(dst->encoding());
4313 assert(isShiftCount(imm8), "illegal shift count");
4314 if (imm8 == 1) {
4315 emit_int8((unsigned char)0xD1);
4316 emit_int8((unsigned char)(0xF8 | encode));
4317 } else {
4318 emit_int8((unsigned char)0xC1);
4319 emit_int8((unsigned char)(0xF8 | encode));
4320 emit_int8(imm8);
4321 }
4322 }
4323
4324 void Assembler::sarl(Register dst) {
4325 int encode = prefix_and_encode(dst->encoding());
4326 emit_int8((unsigned char)0xD3);
4327 emit_int8((unsigned char)(0xF8 | encode));
4328 }
4329
4330 void Assembler::sbbl(Address dst, int32_t imm32) {
4331 InstructionMark im(this);
4332 prefix(dst);
4333 emit_arith_operand(0x81, rbx, dst, imm32);
4334 }
4335
4336 void Assembler::sbbl(Register dst, int32_t imm32) {
4337 prefix(dst);
4338 emit_arith(0x81, 0xD8, dst, imm32);
4339 }
4340
4341
4342 void Assembler::sbbl(Register dst, Address src) {
4343 InstructionMark im(this);
4344 prefix(src, dst);
4345 emit_int8(0x1B);
4346 emit_operand(dst, src);
4347 }
4348
4349 void Assembler::sbbl(Register dst, Register src) {
4350 (void) prefix_and_encode(dst->encoding(), src->encoding());
4351 emit_arith(0x1B, 0xC0, dst, src);
4352 }
4353
4354 void Assembler::setb(Condition cc, Register dst) {
4355 assert(0 <= cc && cc < 16, "illegal cc");
4356 int encode = prefix_and_encode(dst->encoding(), true);
4357 emit_int8(0x0F);
4358 emit_int8((unsigned char)0x90 | cc);
4359 emit_int8((unsigned char)(0xC0 | encode));
4360 }
4361
4362 void Assembler::palignr(XMMRegister dst, XMMRegister src, int imm8) {
4363 assert(VM_Version::supports_ssse3(), "");
4364 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
4365 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4366 emit_int8((unsigned char)0x0F);
4367 emit_int8((unsigned char)(0xC0 | encode));
4368 emit_int8(imm8);
4369 }
4370
4371 void Assembler::vpalignr(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
4372 assert(vector_len == AVX_128bit? VM_Version::supports_avx() :
4373 vector_len == AVX_256bit? VM_Version::supports_avx2() :
4374 0, "");
4375 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4376 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4377 emit_int8((unsigned char)0x0F);
4378 emit_int8((unsigned char)(0xC0 | encode));
4379 emit_int8(imm8);
4380 }
4381
4382 void Assembler::pblendw(XMMRegister dst, XMMRegister src, int imm8) {
4383 assert(VM_Version::supports_sse4_1(), "");
4384 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
4385 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
4386 emit_int8((unsigned char)0x0E);
4387 emit_int8((unsigned char)(0xC0 | encode));
4388 emit_int8(imm8);
4389 }
4390
4391 void Assembler::sha1rnds4(XMMRegister dst, XMMRegister src, int imm8) {
4392 assert(VM_Version::supports_sha(), "");
4393 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_3A, /* rex_w */ false);
4394 emit_int8((unsigned char)0xCC);
4395 emit_int8((unsigned char)(0xC0 | encode));
4396 emit_int8((unsigned char)imm8);
4397 }
4398
4399 void Assembler::sha1nexte(XMMRegister dst, XMMRegister src) {
4400 assert(VM_Version::supports_sha(), "");
4401 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4402 emit_int8((unsigned char)0xC8);
4403 emit_int8((unsigned char)(0xC0 | encode));
4404 }
4405
4406 void Assembler::sha1msg1(XMMRegister dst, XMMRegister src) {
4407 assert(VM_Version::supports_sha(), "");
4408 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4409 emit_int8((unsigned char)0xC9);
4410 emit_int8((unsigned char)(0xC0 | encode));
4411 }
4412
4413 void Assembler::sha1msg2(XMMRegister dst, XMMRegister src) {
4414 assert(VM_Version::supports_sha(), "");
4415 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4416 emit_int8((unsigned char)0xCA);
4417 emit_int8((unsigned char)(0xC0 | encode));
4418 }
4419
4420 // xmm0 is implicit additional source to this instruction.
4421 void Assembler::sha256rnds2(XMMRegister dst, XMMRegister src) {
4422 assert(VM_Version::supports_sha(), "");
4423 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4424 emit_int8((unsigned char)0xCB);
4425 emit_int8((unsigned char)(0xC0 | encode));
4426 }
4427
4428 void Assembler::sha256msg1(XMMRegister dst, XMMRegister src) {
4429 assert(VM_Version::supports_sha(), "");
4430 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4431 emit_int8((unsigned char)0xCC);
4432 emit_int8((unsigned char)(0xC0 | encode));
4433 }
4434
4435 void Assembler::sha256msg2(XMMRegister dst, XMMRegister src) {
4436 assert(VM_Version::supports_sha(), "");
4437 int encode = rex_prefix_and_encode(dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, /* rex_w */ false);
4438 emit_int8((unsigned char)0xCD);
4439 emit_int8((unsigned char)(0xC0 | encode));
4440 }
4441
4442
4443 void Assembler::shll(Register dst, int imm8) {
4444 assert(isShiftCount(imm8), "illegal shift count");
4445 int encode = prefix_and_encode(dst->encoding());
4446 if (imm8 == 1 ) {
4447 emit_int8((unsigned char)0xD1);
4448 emit_int8((unsigned char)(0xE0 | encode));
4449 } else {
4450 emit_int8((unsigned char)0xC1);
4451 emit_int8((unsigned char)(0xE0 | encode));
4452 emit_int8(imm8);
4453 }
4454 }
4455
4456 void Assembler::shll(Register dst) {
4457 int encode = prefix_and_encode(dst->encoding());
4458 emit_int8((unsigned char)0xD3);
4459 emit_int8((unsigned char)(0xE0 | encode));
4460 }
4461
4462 void Assembler::shrl(Register dst, int imm8) {
4463 assert(isShiftCount(imm8), "illegal shift count");
4464 int encode = prefix_and_encode(dst->encoding());
4465 emit_int8((unsigned char)0xC1);
4466 emit_int8((unsigned char)(0xE8 | encode));
4467 emit_int8(imm8);
4468 }
4469
4470 void Assembler::shrl(Register dst) {
4471 int encode = prefix_and_encode(dst->encoding());
4472 emit_int8((unsigned char)0xD3);
4473 emit_int8((unsigned char)(0xE8 | encode));
4474 }
4475
4476 // copies a single word from [esi] to [edi]
4477 void Assembler::smovl() {
4478 emit_int8((unsigned char)0xA5);
4479 }
4480
4481 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
4482 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4483 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4484 attributes.set_rex_vex_w_reverted();
4485 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4486 emit_int8(0x51);
4487 emit_int8((unsigned char)(0xC0 | encode));
4488 }
4489
4490 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4491 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4492 InstructionMark im(this);
4493 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4494 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4495 attributes.set_rex_vex_w_reverted();
4496 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4497 emit_int8(0x51);
4498 emit_operand(dst, src);
4499 }
4500
4501 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
4502 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4503 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4504 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4505 emit_int8(0x51);
4506 emit_int8((unsigned char)(0xC0 | encode));
4507 }
4508
4509 void Assembler::std() {
4510 emit_int8((unsigned char)0xFD);
4511 }
4512
4513 void Assembler::sqrtss(XMMRegister dst, Address src) {
4514 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4515 InstructionMark im(this);
4516 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4517 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4518 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4519 emit_int8(0x51);
4520 emit_operand(dst, src);
4521 }
4522
4523 void Assembler::stmxcsr( Address dst) {
4524 if (UseAVX > 0 ) {
4525 assert(VM_Version::supports_avx(), "");
4526 InstructionMark im(this);
4527 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4528 vex_prefix(dst, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4529 emit_int8((unsigned char)0xAE);
4530 emit_operand(as_Register(3), dst);
4531 } else {
4532 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4533 InstructionMark im(this);
4534 prefix(dst);
4535 emit_int8(0x0F);
4536 emit_int8((unsigned char)0xAE);
4537 emit_operand(as_Register(3), dst);
4538 }
4539 }
4540
4541 void Assembler::subl(Address dst, int32_t imm32) {
4542 InstructionMark im(this);
4543 prefix(dst);
4544 emit_arith_operand(0x81, rbp, dst, imm32);
4545 }
4546
4547 void Assembler::subl(Address dst, Register src) {
4548 InstructionMark im(this);
4549 prefix(dst, src);
4550 emit_int8(0x29);
4551 emit_operand(src, dst);
4552 }
4553
4554 void Assembler::subl(Register dst, int32_t imm32) {
4555 prefix(dst);
4556 emit_arith(0x81, 0xE8, dst, imm32);
4557 }
4558
4559 // Force generation of a 4 byte immediate value even if it fits into 8bit
4560 void Assembler::subl_imm32(Register dst, int32_t imm32) {
4561 prefix(dst);
4562 emit_arith_imm32(0x81, 0xE8, dst, imm32);
4563 }
4564
4565 void Assembler::subl(Register dst, Address src) {
4566 InstructionMark im(this);
4567 prefix(src, dst);
4568 emit_int8(0x2B);
4569 emit_operand(dst, src);
4570 }
4571
4572 void Assembler::subl(Register dst, Register src) {
4573 (void) prefix_and_encode(dst->encoding(), src->encoding());
4574 emit_arith(0x2B, 0xC0, dst, src);
4575 }
4576
4577 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
4578 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4579 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4580 attributes.set_rex_vex_w_reverted();
4581 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4582 emit_int8(0x5C);
4583 emit_int8((unsigned char)(0xC0 | encode));
4584 }
4585
4586 void Assembler::subsd(XMMRegister dst, Address src) {
4587 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4588 InstructionMark im(this);
4589 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4590 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4591 attributes.set_rex_vex_w_reverted();
4592 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4593 emit_int8(0x5C);
4594 emit_operand(dst, src);
4595 }
4596
4597 void Assembler::subss(XMMRegister dst, XMMRegister src) {
4598 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4599 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true , /* uses_vl */ false);
4600 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4601 emit_int8(0x5C);
4602 emit_int8((unsigned char)(0xC0 | encode));
4603 }
4604
4605 void Assembler::subss(XMMRegister dst, Address src) {
4606 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4607 InstructionMark im(this);
4608 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4609 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4610 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4611 emit_int8(0x5C);
4612 emit_operand(dst, src);
4613 }
4614
4615 void Assembler::testb(Register dst, int imm8) {
4616 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
4617 (void) prefix_and_encode(dst->encoding(), true);
4618 emit_arith_b(0xF6, 0xC0, dst, imm8);
4619 }
4620
4621 void Assembler::testb(Address dst, int imm8) {
4622 InstructionMark im(this);
4623 prefix(dst);
4624 emit_int8((unsigned char)0xF6);
4625 emit_operand(rax, dst, 1);
4626 emit_int8(imm8);
4627 }
4628
4629 void Assembler::testl(Register dst, int32_t imm32) {
4630 // not using emit_arith because test
4631 // doesn't support sign-extension of
4632 // 8bit operands
4633 int encode = dst->encoding();
4634 if (encode == 0) {
4635 emit_int8((unsigned char)0xA9);
4636 } else {
4637 encode = prefix_and_encode(encode);
4638 emit_int8((unsigned char)0xF7);
4639 emit_int8((unsigned char)(0xC0 | encode));
4640 }
4641 emit_int32(imm32);
4642 }
4643
4644 void Assembler::testl(Register dst, Register src) {
4645 (void) prefix_and_encode(dst->encoding(), src->encoding());
4646 emit_arith(0x85, 0xC0, dst, src);
4647 }
4648
4649 void Assembler::testl(Register dst, Address src) {
4650 InstructionMark im(this);
4651 prefix(src, dst);
4652 emit_int8((unsigned char)0x85);
4653 emit_operand(dst, src);
4654 }
4655
4656 void Assembler::tzcntl(Register dst, Register src) {
4657 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4658 emit_int8((unsigned char)0xF3);
4659 int encode = prefix_and_encode(dst->encoding(), src->encoding());
4660 emit_int8(0x0F);
4661 emit_int8((unsigned char)0xBC);
4662 emit_int8((unsigned char)0xC0 | encode);
4663 }
4664
4665 void Assembler::tzcntq(Register dst, Register src) {
4666 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
4667 emit_int8((unsigned char)0xF3);
4668 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4669 emit_int8(0x0F);
4670 emit_int8((unsigned char)0xBC);
4671 emit_int8((unsigned char)(0xC0 | encode));
4672 }
4673
4674 void Assembler::ucomisd(XMMRegister dst, Address src) {
4675 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4676 InstructionMark im(this);
4677 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4678 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4679 attributes.set_rex_vex_w_reverted();
4680 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4681 emit_int8(0x2E);
4682 emit_operand(dst, src);
4683 }
4684
4685 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
4686 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4687 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4688 attributes.set_rex_vex_w_reverted();
4689 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4690 emit_int8(0x2E);
4691 emit_int8((unsigned char)(0xC0 | encode));
4692 }
4693
4694 void Assembler::ucomiss(XMMRegister dst, Address src) {
4695 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4696 InstructionMark im(this);
4697 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4698 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4699 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4700 emit_int8(0x2E);
4701 emit_operand(dst, src);
4702 }
4703
4704 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
4705 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4706 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4707 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4708 emit_int8(0x2E);
4709 emit_int8((unsigned char)(0xC0 | encode));
4710 }
4711
4712 void Assembler::xabort(int8_t imm8) {
4713 emit_int8((unsigned char)0xC6);
4714 emit_int8((unsigned char)0xF8);
4715 emit_int8((unsigned char)(imm8 & 0xFF));
4716 }
4717
4718 void Assembler::xaddb(Address dst, Register src) {
4719 InstructionMark im(this);
4720 prefix(dst, src, true);
4721 emit_int8(0x0F);
4722 emit_int8((unsigned char)0xC0);
4723 emit_operand(src, dst);
4724 }
4725
4726 void Assembler::xaddw(Address dst, Register src) {
4727 InstructionMark im(this);
4728 emit_int8(0x66);
4729 prefix(dst, src);
4730 emit_int8(0x0F);
4731 emit_int8((unsigned char)0xC1);
4732 emit_operand(src, dst);
4733 }
4734
4735 void Assembler::xaddl(Address dst, Register src) {
4736 InstructionMark im(this);
4737 prefix(dst, src);
4738 emit_int8(0x0F);
4739 emit_int8((unsigned char)0xC1);
4740 emit_operand(src, dst);
4741 }
4742
4743 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
4744 InstructionMark im(this);
4745 relocate(rtype);
4746 if (abort.is_bound()) {
4747 address entry = target(abort);
4748 assert(entry != NULL, "abort entry NULL");
4749 intptr_t offset = entry - pc();
4750 emit_int8((unsigned char)0xC7);
4751 emit_int8((unsigned char)0xF8);
4752 emit_int32(offset - 6); // 2 opcode + 4 address
4753 } else {
4754 abort.add_patch_at(code(), locator());
4755 emit_int8((unsigned char)0xC7);
4756 emit_int8((unsigned char)0xF8);
4757 emit_int32(0);
4758 }
4759 }
4760
4761 void Assembler::xchgb(Register dst, Address src) { // xchg
4762 InstructionMark im(this);
4763 prefix(src, dst, true);
4764 emit_int8((unsigned char)0x86);
4765 emit_operand(dst, src);
4766 }
4767
4768 void Assembler::xchgw(Register dst, Address src) { // xchg
4769 InstructionMark im(this);
4770 emit_int8(0x66);
4771 prefix(src, dst);
4772 emit_int8((unsigned char)0x87);
4773 emit_operand(dst, src);
4774 }
4775
4776 void Assembler::xchgl(Register dst, Address src) { // xchg
4777 InstructionMark im(this);
4778 prefix(src, dst);
4779 emit_int8((unsigned char)0x87);
4780 emit_operand(dst, src);
4781 }
4782
4783 void Assembler::xchgl(Register dst, Register src) {
4784 int encode = prefix_and_encode(dst->encoding(), src->encoding());
4785 emit_int8((unsigned char)0x87);
4786 emit_int8((unsigned char)(0xC0 | encode));
4787 }
4788
4789 void Assembler::xend() {
4790 emit_int8((unsigned char)0x0F);
4791 emit_int8((unsigned char)0x01);
4792 emit_int8((unsigned char)0xD5);
4793 }
4794
4795 void Assembler::xgetbv() {
4796 emit_int8(0x0F);
4797 emit_int8(0x01);
4798 emit_int8((unsigned char)0xD0);
4799 }
4800
4801 void Assembler::xorl(Register dst, int32_t imm32) {
4802 prefix(dst);
4803 emit_arith(0x81, 0xF0, dst, imm32);
4804 }
4805
4806 void Assembler::xorl(Register dst, Address src) {
4807 InstructionMark im(this);
4808 prefix(src, dst);
4809 emit_int8(0x33);
4810 emit_operand(dst, src);
4811 }
4812
4813 void Assembler::xorl(Register dst, Register src) {
4814 (void) prefix_and_encode(dst->encoding(), src->encoding());
4815 emit_arith(0x33, 0xC0, dst, src);
4816 }
4817
4818 void Assembler::xorb(Register dst, Address src) {
4819 InstructionMark im(this);
4820 prefix(src, dst);
4821 emit_int8(0x32);
4822 emit_operand(dst, src);
4823 }
4824
4825 // AVX 3-operands scalar float-point arithmetic instructions
4826
4827 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
4828 assert(VM_Version::supports_avx(), "");
4829 InstructionMark im(this);
4830 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4831 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4832 attributes.set_rex_vex_w_reverted();
4833 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4834 emit_int8(0x58);
4835 emit_operand(dst, src);
4836 }
4837
4838 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4839 assert(VM_Version::supports_avx(), "");
4840 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4841 attributes.set_rex_vex_w_reverted();
4842 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4843 emit_int8(0x58);
4844 emit_int8((unsigned char)(0xC0 | encode));
4845 }
4846
4847 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
4848 assert(VM_Version::supports_avx(), "");
4849 InstructionMark im(this);
4850 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4851 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4852 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4853 emit_int8(0x58);
4854 emit_operand(dst, src);
4855 }
4856
4857 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4858 assert(VM_Version::supports_avx(), "");
4859 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4860 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4861 emit_int8(0x58);
4862 emit_int8((unsigned char)(0xC0 | encode));
4863 }
4864
4865 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
4866 assert(VM_Version::supports_avx(), "");
4867 InstructionMark im(this);
4868 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4869 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4870 attributes.set_rex_vex_w_reverted();
4871 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4872 emit_int8(0x5E);
4873 emit_operand(dst, src);
4874 }
4875
4876 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4877 assert(VM_Version::supports_avx(), "");
4878 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4879 attributes.set_rex_vex_w_reverted();
4880 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4881 emit_int8(0x5E);
4882 emit_int8((unsigned char)(0xC0 | encode));
4883 }
4884
4885 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
4886 assert(VM_Version::supports_avx(), "");
4887 InstructionMark im(this);
4888 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4889 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4890 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4891 emit_int8(0x5E);
4892 emit_operand(dst, src);
4893 }
4894
4895 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4896 assert(VM_Version::supports_avx(), "");
4897 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4898 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4899 emit_int8(0x5E);
4900 emit_int8((unsigned char)(0xC0 | encode));
4901 }
4902
4903 void Assembler::vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4904 assert(VM_Version::supports_fma(), "");
4905 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4906 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4907 emit_int8((unsigned char)0xB9);
4908 emit_int8((unsigned char)(0xC0 | encode));
4909 }
4910
4911 void Assembler::vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
4912 assert(VM_Version::supports_fma(), "");
4913 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4914 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4915 emit_int8((unsigned char)0xB9);
4916 emit_int8((unsigned char)(0xC0 | encode));
4917 }
4918
4919 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
4920 assert(VM_Version::supports_avx(), "");
4921 InstructionMark im(this);
4922 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4923 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4924 attributes.set_rex_vex_w_reverted();
4925 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4926 emit_int8(0x59);
4927 emit_operand(dst, src);
4928 }
4929
4930 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4931 assert(VM_Version::supports_avx(), "");
4932 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4933 attributes.set_rex_vex_w_reverted();
4934 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4935 emit_int8(0x59);
4936 emit_int8((unsigned char)(0xC0 | encode));
4937 }
4938
4939 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4940 assert(VM_Version::supports_avx(), "");
4941 InstructionMark im(this);
4942 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4943 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4944 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4945 emit_int8(0x59);
4946 emit_operand(dst, src);
4947 }
4948
4949 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4950 assert(VM_Version::supports_avx(), "");
4951 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4952 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4953 emit_int8(0x59);
4954 emit_int8((unsigned char)(0xC0 | encode));
4955 }
4956
4957 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4958 assert(VM_Version::supports_avx(), "");
4959 InstructionMark im(this);
4960 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4961 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4962 attributes.set_rex_vex_w_reverted();
4963 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4964 emit_int8(0x5C);
4965 emit_operand(dst, src);
4966 }
4967
4968 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4969 assert(VM_Version::supports_avx(), "");
4970 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4971 attributes.set_rex_vex_w_reverted();
4972 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4973 emit_int8(0x5C);
4974 emit_int8((unsigned char)(0xC0 | encode));
4975 }
4976
4977 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4978 assert(VM_Version::supports_avx(), "");
4979 InstructionMark im(this);
4980 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4981 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4982 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4983 emit_int8(0x5C);
4984 emit_operand(dst, src);
4985 }
4986
4987 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4988 assert(VM_Version::supports_avx(), "");
4989 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4990 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4991 emit_int8(0x5C);
4992 emit_int8((unsigned char)(0xC0 | encode));
4993 }
4994
4995 //====================VECTOR ARITHMETIC=====================================
4996
4997 // Float-point vector arithmetic
4998
4999 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
5000 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5001 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5002 attributes.set_rex_vex_w_reverted();
5003 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5004 emit_int8(0x58);
5005 emit_int8((unsigned char)(0xC0 | encode));
5006 }
5007
5008 void Assembler::addpd(XMMRegister dst, Address src) {
5009 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5010 InstructionMark im(this);
5011 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5012 attributes.set_rex_vex_w_reverted();
5013 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5014 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5015 emit_int8(0x58);
5016 emit_operand(dst, src);
5017 }
5018
5019
5020 void Assembler::addps(XMMRegister dst, XMMRegister src) {
5021 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5022 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5023 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5024 emit_int8(0x58);
5025 emit_int8((unsigned char)(0xC0 | encode));
5026 }
5027
5028 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5029 assert(VM_Version::supports_avx(), "");
5030 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5031 attributes.set_rex_vex_w_reverted();
5032 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5033 emit_int8(0x58);
5034 emit_int8((unsigned char)(0xC0 | encode));
5035 }
5036
5037 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5038 assert(VM_Version::supports_avx(), "");
5039 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5040 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5041 emit_int8(0x58);
5042 emit_int8((unsigned char)(0xC0 | encode));
5043 }
5044
5045 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5046 assert(VM_Version::supports_avx(), "");
5047 InstructionMark im(this);
5048 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5049 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5050 attributes.set_rex_vex_w_reverted();
5051 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5052 emit_int8(0x58);
5053 emit_operand(dst, src);
5054 }
5055
5056 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5057 assert(VM_Version::supports_avx(), "");
5058 InstructionMark im(this);
5059 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5060 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5061 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5062 emit_int8(0x58);
5063 emit_operand(dst, src);
5064 }
5065
5066 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
5067 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5068 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5069 attributes.set_rex_vex_w_reverted();
5070 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5071 emit_int8(0x5C);
5072 emit_int8((unsigned char)(0xC0 | encode));
5073 }
5074
5075 void Assembler::subps(XMMRegister dst, XMMRegister src) {
5076 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5077 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5078 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5079 emit_int8(0x5C);
5080 emit_int8((unsigned char)(0xC0 | encode));
5081 }
5082
5083 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5084 assert(VM_Version::supports_avx(), "");
5085 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5086 attributes.set_rex_vex_w_reverted();
5087 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5088 emit_int8(0x5C);
5089 emit_int8((unsigned char)(0xC0 | encode));
5090 }
5091
5092 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5093 assert(VM_Version::supports_avx(), "");
5094 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5095 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5096 emit_int8(0x5C);
5097 emit_int8((unsigned char)(0xC0 | encode));
5098 }
5099
5100 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5101 assert(VM_Version::supports_avx(), "");
5102 InstructionMark im(this);
5103 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5104 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5105 attributes.set_rex_vex_w_reverted();
5106 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5107 emit_int8(0x5C);
5108 emit_operand(dst, src);
5109 }
5110
5111 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5112 assert(VM_Version::supports_avx(), "");
5113 InstructionMark im(this);
5114 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5115 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5116 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5117 emit_int8(0x5C);
5118 emit_operand(dst, src);
5119 }
5120
5121 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
5122 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5123 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5124 attributes.set_rex_vex_w_reverted();
5125 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5126 emit_int8(0x59);
5127 emit_int8((unsigned char)(0xC0 | encode));
5128 }
5129
5130 void Assembler::mulpd(XMMRegister dst, Address src) {
5131 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5132 InstructionMark im(this);
5133 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5134 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5135 attributes.set_rex_vex_w_reverted();
5136 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5137 emit_int8(0x59);
5138 emit_operand(dst, src);
5139 }
5140
5141 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
5142 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5143 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5144 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5145 emit_int8(0x59);
5146 emit_int8((unsigned char)(0xC0 | encode));
5147 }
5148
5149 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5150 assert(VM_Version::supports_avx(), "");
5151 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5152 attributes.set_rex_vex_w_reverted();
5153 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5154 emit_int8(0x59);
5155 emit_int8((unsigned char)(0xC0 | encode));
5156 }
5157
5158 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5159 assert(VM_Version::supports_avx(), "");
5160 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5161 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5162 emit_int8(0x59);
5163 emit_int8((unsigned char)(0xC0 | encode));
5164 }
5165
5166 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5167 assert(VM_Version::supports_avx(), "");
5168 InstructionMark im(this);
5169 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5170 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5171 attributes.set_rex_vex_w_reverted();
5172 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5173 emit_int8(0x59);
5174 emit_operand(dst, src);
5175 }
5176
5177 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5178 assert(VM_Version::supports_avx(), "");
5179 InstructionMark im(this);
5180 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5181 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5182 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5183 emit_int8(0x59);
5184 emit_operand(dst, src);
5185 }
5186
5187 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5188 assert(VM_Version::supports_fma(), "");
5189 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5190 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5191 emit_int8((unsigned char)0xB8);
5192 emit_int8((unsigned char)(0xC0 | encode));
5193 }
5194
5195 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int vector_len) {
5196 assert(VM_Version::supports_fma(), "");
5197 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5198 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5199 emit_int8((unsigned char)0xB8);
5200 emit_int8((unsigned char)(0xC0 | encode));
5201 }
5202
5203 void Assembler::vfmadd231pd(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5204 assert(VM_Version::supports_fma(), "");
5205 InstructionMark im(this);
5206 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5207 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5208 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5209 emit_int8((unsigned char)0xB8);
5210 emit_operand(dst, src2);
5211 }
5212
5213 void Assembler::vfmadd231ps(XMMRegister dst, XMMRegister src1, Address src2, int vector_len) {
5214 assert(VM_Version::supports_fma(), "");
5215 InstructionMark im(this);
5216 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5217 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5218 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5219 emit_int8((unsigned char)0xB8);
5220 emit_operand(dst, src2);
5221 }
5222
5223 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
5224 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5225 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5226 attributes.set_rex_vex_w_reverted();
5227 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5228 emit_int8(0x5E);
5229 emit_int8((unsigned char)(0xC0 | encode));
5230 }
5231
5232 void Assembler::divps(XMMRegister dst, XMMRegister src) {
5233 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5234 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5235 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5236 emit_int8(0x5E);
5237 emit_int8((unsigned char)(0xC0 | encode));
5238 }
5239
5240 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5241 assert(VM_Version::supports_avx(), "");
5242 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5243 attributes.set_rex_vex_w_reverted();
5244 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5245 emit_int8(0x5E);
5246 emit_int8((unsigned char)(0xC0 | encode));
5247 }
5248
5249 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5250 assert(VM_Version::supports_avx(), "");
5251 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5252 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5253 emit_int8(0x5E);
5254 emit_int8((unsigned char)(0xC0 | encode));
5255 }
5256
5257 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5258 assert(VM_Version::supports_avx(), "");
5259 InstructionMark im(this);
5260 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5261 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5262 attributes.set_rex_vex_w_reverted();
5263 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5264 emit_int8(0x5E);
5265 emit_operand(dst, src);
5266 }
5267
5268 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5269 assert(VM_Version::supports_avx(), "");
5270 InstructionMark im(this);
5271 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5272 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5273 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5274 emit_int8(0x5E);
5275 emit_operand(dst, src);
5276 }
5277
5278 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
5279 assert(VM_Version::supports_avx(), "");
5280 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5281 attributes.set_rex_vex_w_reverted();
5282 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5283 emit_int8(0x51);
5284 emit_int8((unsigned char)(0xC0 | encode));
5285 }
5286
5287 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
5288 assert(VM_Version::supports_avx(), "");
5289 InstructionMark im(this);
5290 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5291 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5292 attributes.set_rex_vex_w_reverted();
5293 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5294 emit_int8(0x51);
5295 emit_operand(dst, src);
5296 }
5297
5298 void Assembler::vsqrtps(XMMRegister dst, XMMRegister src, int vector_len) {
5299 assert(VM_Version::supports_avx(), "");
5300 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5301 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5302 emit_int8(0x51);
5303 emit_int8((unsigned char)(0xC0 | encode));
5304 }
5305
5306 void Assembler::vsqrtps(XMMRegister dst, Address src, int vector_len) {
5307 assert(VM_Version::supports_avx(), "");
5308 InstructionMark im(this);
5309 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5310 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5311 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5312 emit_int8(0x51);
5313 emit_operand(dst, src);
5314 }
5315
5316 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
5317 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5318 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5319 attributes.set_rex_vex_w_reverted();
5320 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5321 emit_int8(0x54);
5322 emit_int8((unsigned char)(0xC0 | encode));
5323 }
5324
5325 void Assembler::andps(XMMRegister dst, XMMRegister src) {
5326 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5327 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5328 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5329 emit_int8(0x54);
5330 emit_int8((unsigned char)(0xC0 | encode));
5331 }
5332
5333 void Assembler::andps(XMMRegister dst, Address src) {
5334 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5335 InstructionMark im(this);
5336 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5337 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5338 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5339 emit_int8(0x54);
5340 emit_operand(dst, src);
5341 }
5342
5343 void Assembler::andpd(XMMRegister dst, Address src) {
5344 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5345 InstructionMark im(this);
5346 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5347 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5348 attributes.set_rex_vex_w_reverted();
5349 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5350 emit_int8(0x54);
5351 emit_operand(dst, src);
5352 }
5353
5354 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5355 assert(VM_Version::supports_avx(), "");
5356 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5357 attributes.set_rex_vex_w_reverted();
5358 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5359 emit_int8(0x54);
5360 emit_int8((unsigned char)(0xC0 | encode));
5361 }
5362
5363 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5364 assert(VM_Version::supports_avx(), "");
5365 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5366 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5367 emit_int8(0x54);
5368 emit_int8((unsigned char)(0xC0 | encode));
5369 }
5370
5371 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5372 assert(VM_Version::supports_avx(), "");
5373 InstructionMark im(this);
5374 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5375 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5376 attributes.set_rex_vex_w_reverted();
5377 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5378 emit_int8(0x54);
5379 emit_operand(dst, src);
5380 }
5381
5382 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5383 assert(VM_Version::supports_avx(), "");
5384 InstructionMark im(this);
5385 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5386 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5387 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5388 emit_int8(0x54);
5389 emit_operand(dst, src);
5390 }
5391
5392 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
5393 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5394 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5395 attributes.set_rex_vex_w_reverted();
5396 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5397 emit_int8(0x15);
5398 emit_int8((unsigned char)(0xC0 | encode));
5399 }
5400
5401 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
5402 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5403 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5404 attributes.set_rex_vex_w_reverted();
5405 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5406 emit_int8(0x14);
5407 emit_int8((unsigned char)(0xC0 | encode));
5408 }
5409
5410 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
5411 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5412 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5413 attributes.set_rex_vex_w_reverted();
5414 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5415 emit_int8(0x57);
5416 emit_int8((unsigned char)(0xC0 | encode));
5417 }
5418
5419 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
5420 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5421 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5422 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5423 emit_int8(0x57);
5424 emit_int8((unsigned char)(0xC0 | encode));
5425 }
5426
5427 void Assembler::xorpd(XMMRegister dst, Address src) {
5428 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5429 InstructionMark im(this);
5430 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5431 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5432 attributes.set_rex_vex_w_reverted();
5433 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5434 emit_int8(0x57);
5435 emit_operand(dst, src);
5436 }
5437
5438 void Assembler::xorps(XMMRegister dst, Address src) {
5439 NOT_LP64(assert(VM_Version::supports_sse(), ""));
5440 InstructionMark im(this);
5441 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5442 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5443 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5444 emit_int8(0x57);
5445 emit_operand(dst, src);
5446 }
5447
5448 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5449 assert(VM_Version::supports_avx(), "");
5450 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5451 attributes.set_rex_vex_w_reverted();
5452 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5453 emit_int8(0x57);
5454 emit_int8((unsigned char)(0xC0 | encode));
5455 }
5456
5457 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5458 assert(VM_Version::supports_avx(), "");
5459 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5460 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5461 emit_int8(0x57);
5462 emit_int8((unsigned char)(0xC0 | encode));
5463 }
5464
5465 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5466 assert(VM_Version::supports_avx(), "");
5467 InstructionMark im(this);
5468 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5469 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5470 attributes.set_rex_vex_w_reverted();
5471 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5472 emit_int8(0x57);
5473 emit_operand(dst, src);
5474 }
5475
5476 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5477 assert(VM_Version::supports_avx(), "");
5478 InstructionMark im(this);
5479 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5480 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5481 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5482 emit_int8(0x57);
5483 emit_operand(dst, src);
5484 }
5485
5486 // Integer vector arithmetic
5487 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5488 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5489 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5490 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5491 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5492 emit_int8(0x01);
5493 emit_int8((unsigned char)(0xC0 | encode));
5494 }
5495
5496 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5497 assert(VM_Version::supports_avx() && (vector_len == 0) ||
5498 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
5499 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5500 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5501 emit_int8(0x02);
5502 emit_int8((unsigned char)(0xC0 | encode));
5503 }
5504
5505 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
5506 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5507 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5508 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5509 emit_int8((unsigned char)0xFC);
5510 emit_int8((unsigned char)(0xC0 | encode));
5511 }
5512
5513 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
5514 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5515 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5516 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5517 emit_int8((unsigned char)0xFD);
5518 emit_int8((unsigned char)(0xC0 | encode));
5519 }
5520
5521 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
5522 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5523 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5524 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5525 emit_int8((unsigned char)0xFE);
5526 emit_int8((unsigned char)(0xC0 | encode));
5527 }
5528
5529 void Assembler::paddd(XMMRegister dst, Address src) {
5530 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5531 InstructionMark im(this);
5532 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5533 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5534 emit_int8((unsigned char)0xFE);
5535 emit_operand(dst, src);
5536 }
5537
5538 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
5539 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5540 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5541 attributes.set_rex_vex_w_reverted();
5542 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5543 emit_int8((unsigned char)0xD4);
5544 emit_int8((unsigned char)(0xC0 | encode));
5545 }
5546
5547 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
5548 assert(VM_Version::supports_sse3(), "");
5549 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
5550 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5551 emit_int8(0x01);
5552 emit_int8((unsigned char)(0xC0 | encode));
5553 }
5554
5555 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
5556 assert(VM_Version::supports_sse3(), "");
5557 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5558 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5559 emit_int8(0x02);
5560 emit_int8((unsigned char)(0xC0 | encode));
5561 }
5562
5563 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5564 assert(UseAVX > 0, "requires some form of AVX");
5565 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5566 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5567 emit_int8((unsigned char)0xFC);
5568 emit_int8((unsigned char)(0xC0 | encode));
5569 }
5570
5571 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5572 assert(UseAVX > 0, "requires some form of AVX");
5573 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5574 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5575 emit_int8((unsigned char)0xFD);
5576 emit_int8((unsigned char)(0xC0 | encode));
5577 }
5578
5579 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5580 assert(UseAVX > 0, "requires some form of AVX");
5581 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5582 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5583 emit_int8((unsigned char)0xFE);
5584 emit_int8((unsigned char)(0xC0 | encode));
5585 }
5586
5587 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5588 assert(UseAVX > 0, "requires some form of AVX");
5589 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5590 attributes.set_rex_vex_w_reverted();
5591 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5592 emit_int8((unsigned char)0xD4);
5593 emit_int8((unsigned char)(0xC0 | encode));
5594 }
5595
5596 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5597 assert(UseAVX > 0, "requires some form of AVX");
5598 InstructionMark im(this);
5599 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5600 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5601 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5602 emit_int8((unsigned char)0xFC);
5603 emit_operand(dst, src);
5604 }
5605
5606 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5607 assert(UseAVX > 0, "requires some form of AVX");
5608 InstructionMark im(this);
5609 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5610 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5611 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5612 emit_int8((unsigned char)0xFD);
5613 emit_operand(dst, src);
5614 }
5615
5616 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5617 assert(UseAVX > 0, "requires some form of AVX");
5618 InstructionMark im(this);
5619 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5620 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5621 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5622 emit_int8((unsigned char)0xFE);
5623 emit_operand(dst, src);
5624 }
5625
5626 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5627 assert(UseAVX > 0, "requires some form of AVX");
5628 InstructionMark im(this);
5629 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5630 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5631 attributes.set_rex_vex_w_reverted();
5632 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5633 emit_int8((unsigned char)0xD4);
5634 emit_operand(dst, src);
5635 }
5636
5637 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
5638 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5639 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5640 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5641 emit_int8((unsigned char)0xF8);
5642 emit_int8((unsigned char)(0xC0 | encode));
5643 }
5644
5645 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
5646 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5647 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5648 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5649 emit_int8((unsigned char)0xF9);
5650 emit_int8((unsigned char)(0xC0 | encode));
5651 }
5652
5653 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
5654 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5655 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5656 emit_int8((unsigned char)0xFA);
5657 emit_int8((unsigned char)(0xC0 | encode));
5658 }
5659
5660 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
5661 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5662 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5663 attributes.set_rex_vex_w_reverted();
5664 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5665 emit_int8((unsigned char)0xFB);
5666 emit_int8((unsigned char)(0xC0 | encode));
5667 }
5668
5669 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5670 assert(UseAVX > 0, "requires some form of AVX");
5671 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5672 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5673 emit_int8((unsigned char)0xF8);
5674 emit_int8((unsigned char)(0xC0 | encode));
5675 }
5676
5677 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5678 assert(UseAVX > 0, "requires some form of AVX");
5679 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5680 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5681 emit_int8((unsigned char)0xF9);
5682 emit_int8((unsigned char)(0xC0 | encode));
5683 }
5684
5685 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5686 assert(UseAVX > 0, "requires some form of AVX");
5687 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5688 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5689 emit_int8((unsigned char)0xFA);
5690 emit_int8((unsigned char)(0xC0 | encode));
5691 }
5692
5693 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5694 assert(UseAVX > 0, "requires some form of AVX");
5695 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5696 attributes.set_rex_vex_w_reverted();
5697 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5698 emit_int8((unsigned char)0xFB);
5699 emit_int8((unsigned char)(0xC0 | encode));
5700 }
5701
5702 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5703 assert(UseAVX > 0, "requires some form of AVX");
5704 InstructionMark im(this);
5705 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5706 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5707 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5708 emit_int8((unsigned char)0xF8);
5709 emit_operand(dst, src);
5710 }
5711
5712 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5713 assert(UseAVX > 0, "requires some form of AVX");
5714 InstructionMark im(this);
5715 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5716 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5717 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5718 emit_int8((unsigned char)0xF9);
5719 emit_operand(dst, src);
5720 }
5721
5722 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5723 assert(UseAVX > 0, "requires some form of AVX");
5724 InstructionMark im(this);
5725 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5726 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5727 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5728 emit_int8((unsigned char)0xFA);
5729 emit_operand(dst, src);
5730 }
5731
5732 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5733 assert(UseAVX > 0, "requires some form of AVX");
5734 InstructionMark im(this);
5735 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5736 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5737 attributes.set_rex_vex_w_reverted();
5738 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5739 emit_int8((unsigned char)0xFB);
5740 emit_operand(dst, src);
5741 }
5742
5743 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
5744 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5745 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5746 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5747 emit_int8((unsigned char)0xD5);
5748 emit_int8((unsigned char)(0xC0 | encode));
5749 }
5750
5751 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
5752 assert(VM_Version::supports_sse4_1(), "");
5753 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5754 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5755 emit_int8(0x40);
5756 emit_int8((unsigned char)(0xC0 | encode));
5757 }
5758
5759 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5760 assert(UseAVX > 0, "requires some form of AVX");
5761 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5762 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5763 emit_int8((unsigned char)0xD5);
5764 emit_int8((unsigned char)(0xC0 | encode));
5765 }
5766
5767 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5768 assert(UseAVX > 0, "requires some form of AVX");
5769 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5770 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5771 emit_int8(0x40);
5772 emit_int8((unsigned char)(0xC0 | encode));
5773 }
5774
5775 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5776 assert(UseAVX > 2, "requires some form of EVEX");
5777 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5778 attributes.set_is_evex_instruction();
5779 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5780 emit_int8(0x40);
5781 emit_int8((unsigned char)(0xC0 | encode));
5782 }
5783
5784 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5785 assert(UseAVX > 0, "requires some form of AVX");
5786 InstructionMark im(this);
5787 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5788 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
5789 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5790 emit_int8((unsigned char)0xD5);
5791 emit_operand(dst, src);
5792 }
5793
5794 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5795 assert(UseAVX > 0, "requires some form of AVX");
5796 InstructionMark im(this);
5797 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5798 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5799 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5800 emit_int8(0x40);
5801 emit_operand(dst, src);
5802 }
5803
5804 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5805 assert(UseAVX > 2, "requires some form of EVEX");
5806 InstructionMark im(this);
5807 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
5808 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
5809 attributes.set_is_evex_instruction();
5810 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5811 emit_int8(0x40);
5812 emit_operand(dst, src);
5813 }
5814
5815 // Shift packed integers left by specified number of bits.
5816 void Assembler::psllw(XMMRegister dst, int shift) {
5817 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5818 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5819 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5820 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5821 emit_int8(0x71);
5822 emit_int8((unsigned char)(0xC0 | encode));
5823 emit_int8(shift & 0xFF);
5824 }
5825
5826 void Assembler::pslld(XMMRegister dst, int shift) {
5827 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5828 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5829 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5830 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5831 emit_int8(0x72);
5832 emit_int8((unsigned char)(0xC0 | encode));
5833 emit_int8(shift & 0xFF);
5834 }
5835
5836 void Assembler::psllq(XMMRegister dst, int shift) {
5837 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5838 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5839 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5840 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5841 emit_int8(0x73);
5842 emit_int8((unsigned char)(0xC0 | encode));
5843 emit_int8(shift & 0xFF);
5844 }
5845
5846 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
5847 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5848 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5849 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5850 emit_int8((unsigned char)0xF1);
5851 emit_int8((unsigned char)(0xC0 | encode));
5852 }
5853
5854 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
5855 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5856 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5857 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5858 emit_int8((unsigned char)0xF2);
5859 emit_int8((unsigned char)(0xC0 | encode));
5860 }
5861
5862 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
5863 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5864 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5865 attributes.set_rex_vex_w_reverted();
5866 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5867 emit_int8((unsigned char)0xF3);
5868 emit_int8((unsigned char)(0xC0 | encode));
5869 }
5870
5871 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5872 assert(UseAVX > 0, "requires some form of AVX");
5873 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5874 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
5875 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5876 emit_int8(0x71);
5877 emit_int8((unsigned char)(0xC0 | encode));
5878 emit_int8(shift & 0xFF);
5879 }
5880
5881 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5882 assert(UseAVX > 0, "requires some form of AVX");
5883 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5884 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5885 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
5886 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5887 emit_int8(0x72);
5888 emit_int8((unsigned char)(0xC0 | encode));
5889 emit_int8(shift & 0xFF);
5890 }
5891
5892 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5893 assert(UseAVX > 0, "requires some form of AVX");
5894 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5895 attributes.set_rex_vex_w_reverted();
5896 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
5897 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5898 emit_int8(0x73);
5899 emit_int8((unsigned char)(0xC0 | encode));
5900 emit_int8(shift & 0xFF);
5901 }
5902
5903 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5904 assert(UseAVX > 0, "requires some form of AVX");
5905 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5906 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5907 emit_int8((unsigned char)0xF1);
5908 emit_int8((unsigned char)(0xC0 | encode));
5909 }
5910
5911 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5912 assert(UseAVX > 0, "requires some form of AVX");
5913 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5914 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5915 emit_int8((unsigned char)0xF2);
5916 emit_int8((unsigned char)(0xC0 | encode));
5917 }
5918
5919 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5920 assert(UseAVX > 0, "requires some form of AVX");
5921 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5922 attributes.set_rex_vex_w_reverted();
5923 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5924 emit_int8((unsigned char)0xF3);
5925 emit_int8((unsigned char)(0xC0 | encode));
5926 }
5927
5928 // Shift packed integers logically right by specified number of bits.
5929 void Assembler::psrlw(XMMRegister dst, int shift) {
5930 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5931 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5932 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5933 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5934 emit_int8(0x71);
5935 emit_int8((unsigned char)(0xC0 | encode));
5936 emit_int8(shift & 0xFF);
5937 }
5938
5939 void Assembler::psrld(XMMRegister dst, int shift) {
5940 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5941 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5942 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5943 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5944 emit_int8(0x72);
5945 emit_int8((unsigned char)(0xC0 | encode));
5946 emit_int8(shift & 0xFF);
5947 }
5948
5949 void Assembler::psrlq(XMMRegister dst, int shift) {
5950 // Do not confuse it with psrldq SSE2 instruction which
5951 // shifts 128 bit value in xmm register by number of bytes.
5952 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5953 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5954 attributes.set_rex_vex_w_reverted();
5955 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5956 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5957 emit_int8(0x73);
5958 emit_int8((unsigned char)(0xC0 | encode));
5959 emit_int8(shift & 0xFF);
5960 }
5961
5962 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
5963 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5964 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5965 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5966 emit_int8((unsigned char)0xD1);
5967 emit_int8((unsigned char)(0xC0 | encode));
5968 }
5969
5970 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
5971 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5972 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5973 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5974 emit_int8((unsigned char)0xD2);
5975 emit_int8((unsigned char)(0xC0 | encode));
5976 }
5977
5978 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
5979 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5980 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5981 attributes.set_rex_vex_w_reverted();
5982 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5983 emit_int8((unsigned char)0xD3);
5984 emit_int8((unsigned char)(0xC0 | encode));
5985 }
5986
5987 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5988 assert(UseAVX > 0, "requires some form of AVX");
5989 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
5990 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5991 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5992 emit_int8(0x71);
5993 emit_int8((unsigned char)(0xC0 | encode));
5994 emit_int8(shift & 0xFF);
5995 }
5996
5997 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5998 assert(UseAVX > 0, "requires some form of AVX");
5999 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6000 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
6001 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6002 emit_int8(0x72);
6003 emit_int8((unsigned char)(0xC0 | encode));
6004 emit_int8(shift & 0xFF);
6005 }
6006
6007 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6008 assert(UseAVX > 0, "requires some form of AVX");
6009 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6010 attributes.set_rex_vex_w_reverted();
6011 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
6012 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6013 emit_int8(0x73);
6014 emit_int8((unsigned char)(0xC0 | encode));
6015 emit_int8(shift & 0xFF);
6016 }
6017
6018 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6019 assert(UseAVX > 0, "requires some form of AVX");
6020 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6021 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6022 emit_int8((unsigned char)0xD1);
6023 emit_int8((unsigned char)(0xC0 | encode));
6024 }
6025
6026 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6027 assert(UseAVX > 0, "requires some form of AVX");
6028 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6029 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6030 emit_int8((unsigned char)0xD2);
6031 emit_int8((unsigned char)(0xC0 | encode));
6032 }
6033
6034 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6035 assert(UseAVX > 0, "requires some form of AVX");
6036 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6037 attributes.set_rex_vex_w_reverted();
6038 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6039 emit_int8((unsigned char)0xD3);
6040 emit_int8((unsigned char)(0xC0 | encode));
6041 }
6042
6043 // Shift packed integers arithmetically right by specified number of bits.
6044 void Assembler::psraw(XMMRegister dst, int shift) {
6045 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6046 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6047 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6048 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6049 emit_int8(0x71);
6050 emit_int8((unsigned char)(0xC0 | encode));
6051 emit_int8(shift & 0xFF);
6052 }
6053
6054 void Assembler::psrad(XMMRegister dst, int shift) {
6055 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6056 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6057 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
6058 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6059 emit_int8(0x72);
6060 emit_int8((unsigned char)(0xC0 | encode));
6061 emit_int8(shift & 0xFF);
6062 }
6063
6064 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
6065 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6066 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6067 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6068 emit_int8((unsigned char)0xE1);
6069 emit_int8((unsigned char)(0xC0 | encode));
6070 }
6071
6072 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
6073 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6074 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6075 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6076 emit_int8((unsigned char)0xE2);
6077 emit_int8((unsigned char)(0xC0 | encode));
6078 }
6079
6080 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6081 assert(UseAVX > 0, "requires some form of AVX");
6082 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6083 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6084 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6085 emit_int8(0x71);
6086 emit_int8((unsigned char)(0xC0 | encode));
6087 emit_int8(shift & 0xFF);
6088 }
6089
6090 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
6091 assert(UseAVX > 0, "requires some form of AVX");
6092 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6093 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
6094 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6095 emit_int8(0x72);
6096 emit_int8((unsigned char)(0xC0 | encode));
6097 emit_int8(shift & 0xFF);
6098 }
6099
6100 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6101 assert(UseAVX > 0, "requires some form of AVX");
6102 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6103 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6104 emit_int8((unsigned char)0xE1);
6105 emit_int8((unsigned char)(0xC0 | encode));
6106 }
6107
6108 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
6109 assert(UseAVX > 0, "requires some form of AVX");
6110 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6111 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6112 emit_int8((unsigned char)0xE2);
6113 emit_int8((unsigned char)(0xC0 | encode));
6114 }
6115
6116
6117 // logical operations packed integers
6118 void Assembler::pand(XMMRegister dst, XMMRegister src) {
6119 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6120 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6121 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6122 emit_int8((unsigned char)0xDB);
6123 emit_int8((unsigned char)(0xC0 | encode));
6124 }
6125
6126 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6127 assert(UseAVX > 0, "requires some form of AVX");
6128 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6129 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6130 emit_int8((unsigned char)0xDB);
6131 emit_int8((unsigned char)(0xC0 | encode));
6132 }
6133
6134 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6135 assert(UseAVX > 0, "requires some form of AVX");
6136 InstructionMark im(this);
6137 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6138 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6139 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6140 emit_int8((unsigned char)0xDB);
6141 emit_operand(dst, src);
6142 }
6143
6144 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
6145 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6146 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6147 attributes.set_rex_vex_w_reverted();
6148 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6149 emit_int8((unsigned char)0xDF);
6150 emit_int8((unsigned char)(0xC0 | encode));
6151 }
6152
6153 void Assembler::por(XMMRegister dst, XMMRegister src) {
6154 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6155 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6156 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6157 emit_int8((unsigned char)0xEB);
6158 emit_int8((unsigned char)(0xC0 | encode));
6159 }
6160
6161 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6162 assert(UseAVX > 0, "requires some form of AVX");
6163 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6164 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6165 emit_int8((unsigned char)0xEB);
6166 emit_int8((unsigned char)(0xC0 | encode));
6167 }
6168
6169 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6170 assert(UseAVX > 0, "requires some form of AVX");
6171 InstructionMark im(this);
6172 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6173 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6174 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6175 emit_int8((unsigned char)0xEB);
6176 emit_operand(dst, src);
6177 }
6178
6179 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
6180 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
6181 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6182 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6183 emit_int8((unsigned char)0xEF);
6184 emit_int8((unsigned char)(0xC0 | encode));
6185 }
6186
6187 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
6188 assert(UseAVX > 0, "requires some form of AVX");
6189 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6190 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6191 emit_int8((unsigned char)0xEF);
6192 emit_int8((unsigned char)(0xC0 | encode));
6193 }
6194
6195 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
6196 assert(UseAVX > 0, "requires some form of AVX");
6197 InstructionMark im(this);
6198 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6199 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
6200 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6201 emit_int8((unsigned char)0xEF);
6202 emit_operand(dst, src);
6203 }
6204
6205
6206 // vinserti forms
6207
6208 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6209 assert(VM_Version::supports_avx2(), "");
6210 assert(imm8 <= 0x01, "imm8: %u", imm8);
6211 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6212 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6213 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6214 emit_int8(0x38);
6215 emit_int8((unsigned char)(0xC0 | encode));
6216 // 0x00 - insert into lower 128 bits
6217 // 0x01 - insert into upper 128 bits
6218 emit_int8(imm8 & 0x01);
6219 }
6220
6221 void Assembler::vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6222 assert(VM_Version::supports_avx2(), "");
6223 assert(dst != xnoreg, "sanity");
6224 assert(imm8 <= 0x01, "imm8: %u", imm8);
6225 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6226 InstructionMark im(this);
6227 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6228 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6229 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6230 emit_int8(0x38);
6231 emit_operand(dst, src);
6232 // 0x00 - insert into lower 128 bits
6233 // 0x01 - insert into upper 128 bits
6234 emit_int8(imm8 & 0x01);
6235 }
6236
6237 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6238 assert(VM_Version::supports_evex(), "");
6239 assert(imm8 <= 0x03, "imm8: %u", imm8);
6240 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6241 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6242 emit_int8(0x38);
6243 emit_int8((unsigned char)(0xC0 | encode));
6244 // 0x00 - insert into q0 128 bits (0..127)
6245 // 0x01 - insert into q1 128 bits (128..255)
6246 // 0x02 - insert into q2 128 bits (256..383)
6247 // 0x03 - insert into q3 128 bits (384..511)
6248 emit_int8(imm8 & 0x03);
6249 }
6250
6251 void Assembler::vinserti32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6252 assert(VM_Version::supports_avx(), "");
6253 assert(dst != xnoreg, "sanity");
6254 assert(imm8 <= 0x03, "imm8: %u", imm8);
6255 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6256 InstructionMark im(this);
6257 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6258 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6259 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6260 emit_int8(0x18);
6261 emit_operand(dst, src);
6262 // 0x00 - insert into q0 128 bits (0..127)
6263 // 0x01 - insert into q1 128 bits (128..255)
6264 // 0x02 - insert into q2 128 bits (256..383)
6265 // 0x03 - insert into q3 128 bits (384..511)
6266 emit_int8(imm8 & 0x03);
6267 }
6268
6269 void Assembler::vinserti64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6270 assert(VM_Version::supports_evex(), "");
6271 assert(imm8 <= 0x01, "imm8: %u", imm8);
6272 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6273 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6274 emit_int8(0x38);
6275 emit_int8((unsigned char)(0xC0 | encode));
6276 // 0x00 - insert into lower 256 bits
6277 // 0x01 - insert into upper 256 bits
6278 emit_int8(imm8 & 0x01);
6279 }
6280
6281
6282 // vinsertf forms
6283
6284 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6285 assert(VM_Version::supports_avx(), "");
6286 assert(imm8 <= 0x01, "imm8: %u", imm8);
6287 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6288 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6289 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6290 emit_int8(0x18);
6291 emit_int8((unsigned char)(0xC0 | encode));
6292 // 0x00 - insert into lower 128 bits
6293 // 0x01 - insert into upper 128 bits
6294 emit_int8(imm8 & 0x01);
6295 }
6296
6297 void Assembler::vinsertf128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6298 assert(VM_Version::supports_avx(), "");
6299 assert(dst != xnoreg, "sanity");
6300 assert(imm8 <= 0x01, "imm8: %u", imm8);
6301 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6302 InstructionMark im(this);
6303 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6304 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6305 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6306 emit_int8(0x18);
6307 emit_operand(dst, src);
6308 // 0x00 - insert into lower 128 bits
6309 // 0x01 - insert into upper 128 bits
6310 emit_int8(imm8 & 0x01);
6311 }
6312
6313 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6314 assert(VM_Version::supports_evex(), "");
6315 assert(imm8 <= 0x03, "imm8: %u", imm8);
6316 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6317 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6318 emit_int8(0x18);
6319 emit_int8((unsigned char)(0xC0 | encode));
6320 // 0x00 - insert into q0 128 bits (0..127)
6321 // 0x01 - insert into q1 128 bits (128..255)
6322 // 0x02 - insert into q2 128 bits (256..383)
6323 // 0x03 - insert into q3 128 bits (384..511)
6324 emit_int8(imm8 & 0x03);
6325 }
6326
6327 void Assembler::vinsertf32x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6328 assert(VM_Version::supports_avx(), "");
6329 assert(dst != xnoreg, "sanity");
6330 assert(imm8 <= 0x03, "imm8: %u", imm8);
6331 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6332 InstructionMark im(this);
6333 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6334 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6335 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6336 emit_int8(0x18);
6337 emit_operand(dst, src);
6338 // 0x00 - insert into q0 128 bits (0..127)
6339 // 0x01 - insert into q1 128 bits (128..255)
6340 // 0x02 - insert into q2 128 bits (256..383)
6341 // 0x03 - insert into q3 128 bits (384..511)
6342 emit_int8(imm8 & 0x03);
6343 }
6344
6345 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
6346 assert(VM_Version::supports_evex(), "");
6347 assert(imm8 <= 0x01, "imm8: %u", imm8);
6348 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6349 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6350 emit_int8(0x1A);
6351 emit_int8((unsigned char)(0xC0 | encode));
6352 // 0x00 - insert into lower 256 bits
6353 // 0x01 - insert into upper 256 bits
6354 emit_int8(imm8 & 0x01);
6355 }
6356
6357 void Assembler::vinsertf64x4(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
6358 assert(VM_Version::supports_evex(), "");
6359 assert(dst != xnoreg, "sanity");
6360 assert(imm8 <= 0x01, "imm8: %u", imm8);
6361 InstructionMark im(this);
6362 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6363 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
6364 vex_prefix(src, nds->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6365 emit_int8(0x1A);
6366 emit_operand(dst, src);
6367 // 0x00 - insert into lower 256 bits
6368 // 0x01 - insert into upper 256 bits
6369 emit_int8(imm8 & 0x01);
6370 }
6371
6372
6373 // vextracti forms
6374
6375 void Assembler::vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6376 assert(VM_Version::supports_avx(), "");
6377 assert(imm8 <= 0x01, "imm8: %u", imm8);
6378 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6379 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6380 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6381 emit_int8(0x39);
6382 emit_int8((unsigned char)(0xC0 | encode));
6383 // 0x00 - extract from lower 128 bits
6384 // 0x01 - extract from upper 128 bits
6385 emit_int8(imm8 & 0x01);
6386 }
6387
6388 void Assembler::vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
6389 assert(VM_Version::supports_avx2(), "");
6390 assert(src != xnoreg, "sanity");
6391 assert(imm8 <= 0x01, "imm8: %u", imm8);
6392 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6393 InstructionMark im(this);
6394 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6395 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6396 attributes.reset_is_clear_context();
6397 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6398 emit_int8(0x39);
6399 emit_operand(src, dst);
6400 // 0x00 - extract from lower 128 bits
6401 // 0x01 - extract from upper 128 bits
6402 emit_int8(imm8 & 0x01);
6403 }
6404
6405 void Assembler::vextracti32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6406 assert(VM_Version::supports_avx(), "");
6407 assert(imm8 <= 0x03, "imm8: %u", imm8);
6408 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6409 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6410 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6411 emit_int8(0x39);
6412 emit_int8((unsigned char)(0xC0 | encode));
6413 // 0x00 - extract from bits 127:0
6414 // 0x01 - extract from bits 255:128
6415 // 0x02 - extract from bits 383:256
6416 // 0x03 - extract from bits 511:384
6417 emit_int8(imm8 & 0x03);
6418 }
6419
6420 void Assembler::vextracti32x4(Address dst, XMMRegister src, uint8_t imm8) {
6421 assert(VM_Version::supports_evex(), "");
6422 assert(src != xnoreg, "sanity");
6423 assert(imm8 <= 0x03, "imm8: %u", imm8);
6424 InstructionMark im(this);
6425 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6426 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6427 attributes.reset_is_clear_context();
6428 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6429 emit_int8(0x39);
6430 emit_operand(src, dst);
6431 // 0x00 - extract from bits 127:0
6432 // 0x01 - extract from bits 255:128
6433 // 0x02 - extract from bits 383:256
6434 // 0x03 - extract from bits 511:384
6435 emit_int8(imm8 & 0x03);
6436 }
6437
6438 void Assembler::vextracti64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6439 assert(VM_Version::supports_avx512dq(), "");
6440 assert(imm8 <= 0x03, "imm8: %u", imm8);
6441 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6442 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6443 emit_int8(0x39);
6444 emit_int8((unsigned char)(0xC0 | encode));
6445 // 0x00 - extract from bits 127:0
6446 // 0x01 - extract from bits 255:128
6447 // 0x02 - extract from bits 383:256
6448 // 0x03 - extract from bits 511:384
6449 emit_int8(imm8 & 0x03);
6450 }
6451
6452 void Assembler::vextracti64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6453 assert(VM_Version::supports_evex(), "");
6454 assert(imm8 <= 0x01, "imm8: %u", imm8);
6455 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6456 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6457 emit_int8(0x3B);
6458 emit_int8((unsigned char)(0xC0 | encode));
6459 // 0x00 - extract from lower 256 bits
6460 // 0x01 - extract from upper 256 bits
6461 emit_int8(imm8 & 0x01);
6462 }
6463
6464
6465 // vextractf forms
6466
6467 void Assembler::vextractf128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6468 assert(VM_Version::supports_avx(), "");
6469 assert(imm8 <= 0x01, "imm8: %u", imm8);
6470 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6471 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6472 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6473 emit_int8(0x19);
6474 emit_int8((unsigned char)(0xC0 | encode));
6475 // 0x00 - extract from lower 128 bits
6476 // 0x01 - extract from upper 128 bits
6477 emit_int8(imm8 & 0x01);
6478 }
6479
6480 void Assembler::vextractf128(Address dst, XMMRegister src, uint8_t imm8) {
6481 assert(VM_Version::supports_avx(), "");
6482 assert(src != xnoreg, "sanity");
6483 assert(imm8 <= 0x01, "imm8: %u", imm8);
6484 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_256bit;
6485 InstructionMark im(this);
6486 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6487 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6488 attributes.reset_is_clear_context();
6489 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6490 emit_int8(0x19);
6491 emit_operand(src, dst);
6492 // 0x00 - extract from lower 128 bits
6493 // 0x01 - extract from upper 128 bits
6494 emit_int8(imm8 & 0x01);
6495 }
6496
6497 void Assembler::vextractf32x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6498 assert(VM_Version::supports_avx(), "");
6499 assert(imm8 <= 0x03, "imm8: %u", imm8);
6500 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
6501 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6502 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6503 emit_int8(0x19);
6504 emit_int8((unsigned char)(0xC0 | encode));
6505 // 0x00 - extract from bits 127:0
6506 // 0x01 - extract from bits 255:128
6507 // 0x02 - extract from bits 383:256
6508 // 0x03 - extract from bits 511:384
6509 emit_int8(imm8 & 0x03);
6510 }
6511
6512 void Assembler::vextractf32x4(Address dst, XMMRegister src, uint8_t imm8) {
6513 assert(VM_Version::supports_evex(), "");
6514 assert(src != xnoreg, "sanity");
6515 assert(imm8 <= 0x03, "imm8: %u", imm8);
6516 InstructionMark im(this);
6517 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6518 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
6519 attributes.reset_is_clear_context();
6520 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6521 emit_int8(0x19);
6522 emit_operand(src, dst);
6523 // 0x00 - extract from bits 127:0
6524 // 0x01 - extract from bits 255:128
6525 // 0x02 - extract from bits 383:256
6526 // 0x03 - extract from bits 511:384
6527 emit_int8(imm8 & 0x03);
6528 }
6529
6530 void Assembler::vextractf64x2(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6531 assert(VM_Version::supports_avx512dq(), "");
6532 assert(imm8 <= 0x03, "imm8: %u", imm8);
6533 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6534 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6535 emit_int8(0x19);
6536 emit_int8((unsigned char)(0xC0 | encode));
6537 // 0x00 - extract from bits 127:0
6538 // 0x01 - extract from bits 255:128
6539 // 0x02 - extract from bits 383:256
6540 // 0x03 - extract from bits 511:384
6541 emit_int8(imm8 & 0x03);
6542 }
6543
6544 void Assembler::vextractf64x4(XMMRegister dst, XMMRegister src, uint8_t imm8) {
6545 assert(VM_Version::supports_evex(), "");
6546 assert(imm8 <= 0x01, "imm8: %u", imm8);
6547 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6548 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6549 emit_int8(0x1B);
6550 emit_int8((unsigned char)(0xC0 | encode));
6551 // 0x00 - extract from lower 256 bits
6552 // 0x01 - extract from upper 256 bits
6553 emit_int8(imm8 & 0x01);
6554 }
6555
6556 void Assembler::vextractf64x4(Address dst, XMMRegister src, uint8_t imm8) {
6557 assert(VM_Version::supports_evex(), "");
6558 assert(src != xnoreg, "sanity");
6559 assert(imm8 <= 0x01, "imm8: %u", imm8);
6560 InstructionMark im(this);
6561 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
6562 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
6563 attributes.reset_is_clear_context();
6564 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6565 emit_int8(0x1B);
6566 emit_operand(src, dst);
6567 // 0x00 - extract from lower 256 bits
6568 // 0x01 - extract from upper 256 bits
6569 emit_int8(imm8 & 0x01);
6570 }
6571
6572
6573 // legacy word/dword replicate
6574 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
6575 assert(VM_Version::supports_avx2(), "");
6576 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6577 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6578 emit_int8(0x79);
6579 emit_int8((unsigned char)(0xC0 | encode));
6580 }
6581
6582 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
6583 assert(VM_Version::supports_avx2(), "");
6584 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6585 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6586 emit_int8(0x58);
6587 emit_int8((unsigned char)(0xC0 | encode));
6588 }
6589
6590
6591 // xmm/mem sourced byte/word/dword/qword replicate
6592
6593 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6594 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
6595 assert(VM_Version::supports_evex(), "");
6596 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6597 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6598 emit_int8(0x78);
6599 emit_int8((unsigned char)(0xC0 | encode));
6600 }
6601
6602 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
6603 assert(VM_Version::supports_evex(), "");
6604 assert(dst != xnoreg, "sanity");
6605 InstructionMark im(this);
6606 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6607 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
6608 // swap src<->dst for encoding
6609 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6610 emit_int8(0x78);
6611 emit_operand(dst, src);
6612 }
6613
6614 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6615 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
6616 assert(VM_Version::supports_evex(), "");
6617 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6618 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6619 emit_int8(0x79);
6620 emit_int8((unsigned char)(0xC0 | encode));
6621 }
6622
6623 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
6624 assert(VM_Version::supports_evex(), "");
6625 assert(dst != xnoreg, "sanity");
6626 InstructionMark im(this);
6627 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6628 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
6629 // swap src<->dst for encoding
6630 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6631 emit_int8(0x79);
6632 emit_operand(dst, src);
6633 }
6634
6635 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6636 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
6637 assert(VM_Version::supports_evex(), "");
6638 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6639 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6640 emit_int8(0x58);
6641 emit_int8((unsigned char)(0xC0 | encode));
6642 }
6643
6644 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
6645 assert(VM_Version::supports_evex(), "");
6646 assert(dst != xnoreg, "sanity");
6647 InstructionMark im(this);
6648 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6649 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6650 // swap src<->dst for encoding
6651 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6652 emit_int8(0x58);
6653 emit_operand(dst, src);
6654 }
6655
6656 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6657 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
6658 assert(VM_Version::supports_evex(), "");
6659 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6660 attributes.set_rex_vex_w_reverted();
6661 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6662 emit_int8(0x59);
6663 emit_int8((unsigned char)(0xC0 | encode));
6664 }
6665
6666 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
6667 assert(VM_Version::supports_evex(), "");
6668 assert(dst != xnoreg, "sanity");
6669 InstructionMark im(this);
6670 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6671 attributes.set_rex_vex_w_reverted();
6672 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6673 // swap src<->dst for encoding
6674 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6675 emit_int8(0x59);
6676 emit_operand(dst, src);
6677 }
6678
6679
6680 // scalar single/double precision replicate
6681
6682 // duplicate single precision data from src into programmed locations in dest : requires AVX512VL
6683 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
6684 assert(VM_Version::supports_evex(), "");
6685 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6686 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6687 emit_int8(0x18);
6688 emit_int8((unsigned char)(0xC0 | encode));
6689 }
6690
6691 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
6692 assert(VM_Version::supports_evex(), "");
6693 assert(dst != xnoreg, "sanity");
6694 InstructionMark im(this);
6695 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6696 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
6697 // swap src<->dst for encoding
6698 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6699 emit_int8(0x18);
6700 emit_operand(dst, src);
6701 }
6702
6703 // duplicate double precision data from src into programmed locations in dest : requires AVX512VL
6704 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
6705 assert(VM_Version::supports_evex(), "");
6706 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6707 attributes.set_rex_vex_w_reverted();
6708 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6709 emit_int8(0x19);
6710 emit_int8((unsigned char)(0xC0 | encode));
6711 }
6712
6713 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
6714 assert(VM_Version::supports_evex(), "");
6715 assert(dst != xnoreg, "sanity");
6716 InstructionMark im(this);
6717 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6718 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
6719 attributes.set_rex_vex_w_reverted();
6720 // swap src<->dst for encoding
6721 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6722 emit_int8(0x19);
6723 emit_operand(dst, src);
6724 }
6725
6726
6727 // gpr source broadcast forms
6728
6729 // duplicate 1-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6730 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
6731 assert(VM_Version::supports_evex(), "");
6732 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6733 attributes.set_is_evex_instruction();
6734 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6735 emit_int8(0x7A);
6736 emit_int8((unsigned char)(0xC0 | encode));
6737 }
6738
6739 // duplicate 2-byte integer data from src into programmed locations in dest : requires AVX512BW and AVX512VL
6740 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
6741 assert(VM_Version::supports_evex(), "");
6742 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ true);
6743 attributes.set_is_evex_instruction();
6744 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6745 emit_int8(0x7B);
6746 emit_int8((unsigned char)(0xC0 | encode));
6747 }
6748
6749 // duplicate 4-byte integer data from src into programmed locations in dest : requires AVX512VL
6750 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
6751 assert(VM_Version::supports_evex(), "");
6752 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6753 attributes.set_is_evex_instruction();
6754 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6755 emit_int8(0x7C);
6756 emit_int8((unsigned char)(0xC0 | encode));
6757 }
6758
6759 // duplicate 8-byte integer data from src into programmed locations in dest : requires AVX512VL
6760 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
6761 assert(VM_Version::supports_evex(), "");
6762 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
6763 attributes.set_is_evex_instruction();
6764 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
6765 emit_int8(0x7C);
6766 emit_int8((unsigned char)(0xC0 | encode));
6767 }
6768
6769
6770 // Carry-Less Multiplication Quadword
6771 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
6772 assert(VM_Version::supports_clmul(), "");
6773 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6774 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6775 emit_int8(0x44);
6776 emit_int8((unsigned char)(0xC0 | encode));
6777 emit_int8((unsigned char)mask);
6778 }
6779
6780 // Carry-Less Multiplication Quadword
6781 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
6782 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
6783 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6784 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6785 emit_int8(0x44);
6786 emit_int8((unsigned char)(0xC0 | encode));
6787 emit_int8((unsigned char)mask);
6788 }
6789
6790 void Assembler::vzeroupper() {
6791 if (VM_Version::supports_vzeroupper()) {
6792 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
6793 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
6794 emit_int8(0x77);
6795 }
6796 }
6797
6798 #ifndef _LP64
6799 // 32bit only pieces of the assembler
6800
6801 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
6802 // NO PREFIX AS NEVER 64BIT
6803 InstructionMark im(this);
6804 emit_int8((unsigned char)0x81);
6805 emit_int8((unsigned char)(0xF8 | src1->encoding()));
6806 emit_data(imm32, rspec, 0);
6807 }
6808
6809 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
6810 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
6811 InstructionMark im(this);
6812 emit_int8((unsigned char)0x81);
6813 emit_operand(rdi, src1);
6814 emit_data(imm32, rspec, 0);
6815 }
6816
6817 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
6818 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
6819 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
6820 void Assembler::cmpxchg8(Address adr) {
6821 InstructionMark im(this);
6822 emit_int8(0x0F);
6823 emit_int8((unsigned char)0xC7);
6824 emit_operand(rcx, adr);
6825 }
6826
6827 void Assembler::decl(Register dst) {
6828 // Don't use it directly. Use MacroAssembler::decrementl() instead.
6829 emit_int8(0x48 | dst->encoding());
6830 }
6831
6832 #endif // _LP64
6833
6834 // 64bit typically doesn't use the x87 but needs to for the trig funcs
6835
6836 void Assembler::fabs() {
6837 emit_int8((unsigned char)0xD9);
6838 emit_int8((unsigned char)0xE1);
6839 }
6840
6841 void Assembler::fadd(int i) {
6842 emit_farith(0xD8, 0xC0, i);
6843 }
6844
6845 void Assembler::fadd_d(Address src) {
6846 InstructionMark im(this);
6847 emit_int8((unsigned char)0xDC);
6848 emit_operand32(rax, src);
6849 }
6850
6851 void Assembler::fadd_s(Address src) {
6852 InstructionMark im(this);
6853 emit_int8((unsigned char)0xD8);
6854 emit_operand32(rax, src);
6855 }
6856
6857 void Assembler::fadda(int i) {
6858 emit_farith(0xDC, 0xC0, i);
6859 }
6860
6861 void Assembler::faddp(int i) {
6862 emit_farith(0xDE, 0xC0, i);
6863 }
6864
6865 void Assembler::fchs() {
6866 emit_int8((unsigned char)0xD9);
6867 emit_int8((unsigned char)0xE0);
6868 }
6869
6870 void Assembler::fcom(int i) {
6871 emit_farith(0xD8, 0xD0, i);
6872 }
6873
6874 void Assembler::fcomp(int i) {
6875 emit_farith(0xD8, 0xD8, i);
6876 }
6877
6878 void Assembler::fcomp_d(Address src) {
6879 InstructionMark im(this);
6880 emit_int8((unsigned char)0xDC);
6881 emit_operand32(rbx, src);
6882 }
6883
6884 void Assembler::fcomp_s(Address src) {
6885 InstructionMark im(this);
6886 emit_int8((unsigned char)0xD8);
6887 emit_operand32(rbx, src);
6888 }
6889
6890 void Assembler::fcompp() {
6891 emit_int8((unsigned char)0xDE);
6892 emit_int8((unsigned char)0xD9);
6893 }
6894
6895 void Assembler::fcos() {
6896 emit_int8((unsigned char)0xD9);
6897 emit_int8((unsigned char)0xFF);
6898 }
6899
6900 void Assembler::fdecstp() {
6901 emit_int8((unsigned char)0xD9);
6902 emit_int8((unsigned char)0xF6);
6903 }
6904
6905 void Assembler::fdiv(int i) {
6906 emit_farith(0xD8, 0xF0, i);
6907 }
6908
6909 void Assembler::fdiv_d(Address src) {
6910 InstructionMark im(this);
6911 emit_int8((unsigned char)0xDC);
6912 emit_operand32(rsi, src);
6913 }
6914
6915 void Assembler::fdiv_s(Address src) {
6916 InstructionMark im(this);
6917 emit_int8((unsigned char)0xD8);
6918 emit_operand32(rsi, src);
6919 }
6920
6921 void Assembler::fdiva(int i) {
6922 emit_farith(0xDC, 0xF8, i);
6923 }
6924
6925 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
6926 // is erroneous for some of the floating-point instructions below.
6927
6928 void Assembler::fdivp(int i) {
6929 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
6930 }
6931
6932 void Assembler::fdivr(int i) {
6933 emit_farith(0xD8, 0xF8, i);
6934 }
6935
6936 void Assembler::fdivr_d(Address src) {
6937 InstructionMark im(this);
6938 emit_int8((unsigned char)0xDC);
6939 emit_operand32(rdi, src);
6940 }
6941
6942 void Assembler::fdivr_s(Address src) {
6943 InstructionMark im(this);
6944 emit_int8((unsigned char)0xD8);
6945 emit_operand32(rdi, src);
6946 }
6947
6948 void Assembler::fdivra(int i) {
6949 emit_farith(0xDC, 0xF0, i);
6950 }
6951
6952 void Assembler::fdivrp(int i) {
6953 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
6954 }
6955
6956 void Assembler::ffree(int i) {
6957 emit_farith(0xDD, 0xC0, i);
6958 }
6959
6960 void Assembler::fild_d(Address adr) {
6961 InstructionMark im(this);
6962 emit_int8((unsigned char)0xDF);
6963 emit_operand32(rbp, adr);
6964 }
6965
6966 void Assembler::fild_s(Address adr) {
6967 InstructionMark im(this);
6968 emit_int8((unsigned char)0xDB);
6969 emit_operand32(rax, adr);
6970 }
6971
6972 void Assembler::fincstp() {
6973 emit_int8((unsigned char)0xD9);
6974 emit_int8((unsigned char)0xF7);
6975 }
6976
6977 void Assembler::finit() {
6978 emit_int8((unsigned char)0x9B);
6979 emit_int8((unsigned char)0xDB);
6980 emit_int8((unsigned char)0xE3);
6981 }
6982
6983 void Assembler::fist_s(Address adr) {
6984 InstructionMark im(this);
6985 emit_int8((unsigned char)0xDB);
6986 emit_operand32(rdx, adr);
6987 }
6988
6989 void Assembler::fistp_d(Address adr) {
6990 InstructionMark im(this);
6991 emit_int8((unsigned char)0xDF);
6992 emit_operand32(rdi, adr);
6993 }
6994
6995 void Assembler::fistp_s(Address adr) {
6996 InstructionMark im(this);
6997 emit_int8((unsigned char)0xDB);
6998 emit_operand32(rbx, adr);
6999 }
7000
7001 void Assembler::fld1() {
7002 emit_int8((unsigned char)0xD9);
7003 emit_int8((unsigned char)0xE8);
7004 }
7005
7006 void Assembler::fld_d(Address adr) {
7007 InstructionMark im(this);
7008 emit_int8((unsigned char)0xDD);
7009 emit_operand32(rax, adr);
7010 }
7011
7012 void Assembler::fld_s(Address adr) {
7013 InstructionMark im(this);
7014 emit_int8((unsigned char)0xD9);
7015 emit_operand32(rax, adr);
7016 }
7017
7018
7019 void Assembler::fld_s(int index) {
7020 emit_farith(0xD9, 0xC0, index);
7021 }
7022
7023 void Assembler::fld_x(Address adr) {
7024 InstructionMark im(this);
7025 emit_int8((unsigned char)0xDB);
7026 emit_operand32(rbp, adr);
7027 }
7028
7029 void Assembler::fldcw(Address src) {
7030 InstructionMark im(this);
7031 emit_int8((unsigned char)0xD9);
7032 emit_operand32(rbp, src);
7033 }
7034
7035 void Assembler::fldenv(Address src) {
7036 InstructionMark im(this);
7037 emit_int8((unsigned char)0xD9);
7038 emit_operand32(rsp, src);
7039 }
7040
7041 void Assembler::fldlg2() {
7042 emit_int8((unsigned char)0xD9);
7043 emit_int8((unsigned char)0xEC);
7044 }
7045
7046 void Assembler::fldln2() {
7047 emit_int8((unsigned char)0xD9);
7048 emit_int8((unsigned char)0xED);
7049 }
7050
7051 void Assembler::fldz() {
7052 emit_int8((unsigned char)0xD9);
7053 emit_int8((unsigned char)0xEE);
7054 }
7055
7056 void Assembler::flog() {
7057 fldln2();
7058 fxch();
7059 fyl2x();
7060 }
7061
7062 void Assembler::flog10() {
7063 fldlg2();
7064 fxch();
7065 fyl2x();
7066 }
7067
7068 void Assembler::fmul(int i) {
7069 emit_farith(0xD8, 0xC8, i);
7070 }
7071
7072 void Assembler::fmul_d(Address src) {
7073 InstructionMark im(this);
7074 emit_int8((unsigned char)0xDC);
7075 emit_operand32(rcx, src);
7076 }
7077
7078 void Assembler::fmul_s(Address src) {
7079 InstructionMark im(this);
7080 emit_int8((unsigned char)0xD8);
7081 emit_operand32(rcx, src);
7082 }
7083
7084 void Assembler::fmula(int i) {
7085 emit_farith(0xDC, 0xC8, i);
7086 }
7087
7088 void Assembler::fmulp(int i) {
7089 emit_farith(0xDE, 0xC8, i);
7090 }
7091
7092 void Assembler::fnsave(Address dst) {
7093 InstructionMark im(this);
7094 emit_int8((unsigned char)0xDD);
7095 emit_operand32(rsi, dst);
7096 }
7097
7098 void Assembler::fnstcw(Address src) {
7099 InstructionMark im(this);
7100 emit_int8((unsigned char)0x9B);
7101 emit_int8((unsigned char)0xD9);
7102 emit_operand32(rdi, src);
7103 }
7104
7105 void Assembler::fnstsw_ax() {
7106 emit_int8((unsigned char)0xDF);
7107 emit_int8((unsigned char)0xE0);
7108 }
7109
7110 void Assembler::fprem() {
7111 emit_int8((unsigned char)0xD9);
7112 emit_int8((unsigned char)0xF8);
7113 }
7114
7115 void Assembler::fprem1() {
7116 emit_int8((unsigned char)0xD9);
7117 emit_int8((unsigned char)0xF5);
7118 }
7119
7120 void Assembler::frstor(Address src) {
7121 InstructionMark im(this);
7122 emit_int8((unsigned char)0xDD);
7123 emit_operand32(rsp, src);
7124 }
7125
7126 void Assembler::fsin() {
7127 emit_int8((unsigned char)0xD9);
7128 emit_int8((unsigned char)0xFE);
7129 }
7130
7131 void Assembler::fsqrt() {
7132 emit_int8((unsigned char)0xD9);
7133 emit_int8((unsigned char)0xFA);
7134 }
7135
7136 void Assembler::fst_d(Address adr) {
7137 InstructionMark im(this);
7138 emit_int8((unsigned char)0xDD);
7139 emit_operand32(rdx, adr);
7140 }
7141
7142 void Assembler::fst_s(Address adr) {
7143 InstructionMark im(this);
7144 emit_int8((unsigned char)0xD9);
7145 emit_operand32(rdx, adr);
7146 }
7147
7148 void Assembler::fstp_d(Address adr) {
7149 InstructionMark im(this);
7150 emit_int8((unsigned char)0xDD);
7151 emit_operand32(rbx, adr);
7152 }
7153
7154 void Assembler::fstp_d(int index) {
7155 emit_farith(0xDD, 0xD8, index);
7156 }
7157
7158 void Assembler::fstp_s(Address adr) {
7159 InstructionMark im(this);
7160 emit_int8((unsigned char)0xD9);
7161 emit_operand32(rbx, adr);
7162 }
7163
7164 void Assembler::fstp_x(Address adr) {
7165 InstructionMark im(this);
7166 emit_int8((unsigned char)0xDB);
7167 emit_operand32(rdi, adr);
7168 }
7169
7170 void Assembler::fsub(int i) {
7171 emit_farith(0xD8, 0xE0, i);
7172 }
7173
7174 void Assembler::fsub_d(Address src) {
7175 InstructionMark im(this);
7176 emit_int8((unsigned char)0xDC);
7177 emit_operand32(rsp, src);
7178 }
7179
7180 void Assembler::fsub_s(Address src) {
7181 InstructionMark im(this);
7182 emit_int8((unsigned char)0xD8);
7183 emit_operand32(rsp, src);
7184 }
7185
7186 void Assembler::fsuba(int i) {
7187 emit_farith(0xDC, 0xE8, i);
7188 }
7189
7190 void Assembler::fsubp(int i) {
7191 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
7192 }
7193
7194 void Assembler::fsubr(int i) {
7195 emit_farith(0xD8, 0xE8, i);
7196 }
7197
7198 void Assembler::fsubr_d(Address src) {
7199 InstructionMark im(this);
7200 emit_int8((unsigned char)0xDC);
7201 emit_operand32(rbp, src);
7202 }
7203
7204 void Assembler::fsubr_s(Address src) {
7205 InstructionMark im(this);
7206 emit_int8((unsigned char)0xD8);
7207 emit_operand32(rbp, src);
7208 }
7209
7210 void Assembler::fsubra(int i) {
7211 emit_farith(0xDC, 0xE0, i);
7212 }
7213
7214 void Assembler::fsubrp(int i) {
7215 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
7216 }
7217
7218 void Assembler::ftan() {
7219 emit_int8((unsigned char)0xD9);
7220 emit_int8((unsigned char)0xF2);
7221 emit_int8((unsigned char)0xDD);
7222 emit_int8((unsigned char)0xD8);
7223 }
7224
7225 void Assembler::ftst() {
7226 emit_int8((unsigned char)0xD9);
7227 emit_int8((unsigned char)0xE4);
7228 }
7229
7230 void Assembler::fucomi(int i) {
7231 // make sure the instruction is supported (introduced for P6, together with cmov)
7232 guarantee(VM_Version::supports_cmov(), "illegal instruction");
7233 emit_farith(0xDB, 0xE8, i);
7234 }
7235
7236 void Assembler::fucomip(int i) {
7237 // make sure the instruction is supported (introduced for P6, together with cmov)
7238 guarantee(VM_Version::supports_cmov(), "illegal instruction");
7239 emit_farith(0xDF, 0xE8, i);
7240 }
7241
7242 void Assembler::fwait() {
7243 emit_int8((unsigned char)0x9B);
7244 }
7245
7246 void Assembler::fxch(int i) {
7247 emit_farith(0xD9, 0xC8, i);
7248 }
7249
7250 void Assembler::fyl2x() {
7251 emit_int8((unsigned char)0xD9);
7252 emit_int8((unsigned char)0xF1);
7253 }
7254
7255 void Assembler::frndint() {
7256 emit_int8((unsigned char)0xD9);
7257 emit_int8((unsigned char)0xFC);
7258 }
7259
7260 void Assembler::f2xm1() {
7261 emit_int8((unsigned char)0xD9);
7262 emit_int8((unsigned char)0xF0);
7263 }
7264
7265 void Assembler::fldl2e() {
7266 emit_int8((unsigned char)0xD9);
7267 emit_int8((unsigned char)0xEA);
7268 }
7269
7270 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
7271 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
7272 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
7273 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
7274
7275 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
7276 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7277 if (pre > 0) {
7278 emit_int8(simd_pre[pre]);
7279 }
7280 if (rex_w) {
7281 prefixq(adr, xreg);
7282 } else {
7283 prefix(adr, xreg);
7284 }
7285 if (opc > 0) {
7286 emit_int8(0x0F);
7287 int opc2 = simd_opc[opc];
7288 if (opc2 > 0) {
7289 emit_int8(opc2);
7290 }
7291 }
7292 }
7293
7294 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
7295 if (pre > 0) {
7296 emit_int8(simd_pre[pre]);
7297 }
7298 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
7299 if (opc > 0) {
7300 emit_int8(0x0F);
7301 int opc2 = simd_opc[opc];
7302 if (opc2 > 0) {
7303 emit_int8(opc2);
7304 }
7305 }
7306 return encode;
7307 }
7308
7309
7310 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
7311 int vector_len = _attributes->get_vector_len();
7312 bool vex_w = _attributes->is_rex_vex_w();
7313 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
7314 prefix(VEX_3bytes);
7315
7316 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
7317 byte1 = (~byte1) & 0xE0;
7318 byte1 |= opc;
7319 emit_int8(byte1);
7320
7321 int byte2 = ((~nds_enc) & 0xf) << 3;
7322 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
7323 emit_int8(byte2);
7324 } else {
7325 prefix(VEX_2bytes);
7326
7327 int byte1 = vex_r ? VEX_R : 0;
7328 byte1 = (~byte1) & 0x80;
7329 byte1 |= ((~nds_enc) & 0xf) << 3;
7330 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
7331 emit_int8(byte1);
7332 }
7333 }
7334
7335 // This is a 4 byte encoding
7336 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
7337 // EVEX 0x62 prefix
7338 prefix(EVEX_4bytes);
7339 bool vex_w = _attributes->is_rex_vex_w();
7340 int evex_encoding = (vex_w ? VEX_W : 0);
7341 // EVEX.b is not currently used for broadcast of single element or data rounding modes
7342 _attributes->set_evex_encoding(evex_encoding);
7343
7344 // P0: byte 2, initialized to RXBR`00mm
7345 // instead of not'd
7346 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
7347 byte2 = (~byte2) & 0xF0;
7348 // confine opc opcode extensions in mm bits to lower two bits
7349 // of form {0F, 0F_38, 0F_3A}
7350 byte2 |= opc;
7351 emit_int8(byte2);
7352
7353 // P1: byte 3 as Wvvvv1pp
7354 int byte3 = ((~nds_enc) & 0xf) << 3;
7355 // p[10] is always 1
7356 byte3 |= EVEX_F;
7357 byte3 |= (vex_w & 1) << 7;
7358 // confine pre opcode extensions in pp bits to lower two bits
7359 // of form {66, F3, F2}
7360 byte3 |= pre;
7361 emit_int8(byte3);
7362
7363 // P2: byte 4 as zL'Lbv'aaa
7364 // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
7365 int byte4 = (_attributes->is_no_reg_mask()) ?
7366 0 :
7367 _attributes->get_embedded_opmask_register_specifier();
7368 // EVEX.v` for extending EVEX.vvvv or VIDX
7369 byte4 |= (evex_v ? 0: EVEX_V);
7370 // third EXEC.b for broadcast actions
7371 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
7372 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
7373 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
7374 // last is EVEX.z for zero/merge actions
7375 if (_attributes->is_no_reg_mask() == false) {
7376 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
7377 }
7378 emit_int8(byte4);
7379 }
7380
7381 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7382 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
7383 bool vex_b = adr.base_needs_rex();
7384 bool vex_x = adr.index_needs_rex();
7385 set_attributes(attributes);
7386 attributes->set_current_assembler(this);
7387
7388 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7389 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7390 switch (attributes->get_vector_len()) {
7391 case AVX_128bit:
7392 case AVX_256bit:
7393 attributes->set_is_legacy_mode();
7394 break;
7395 }
7396 }
7397
7398 // For pure EVEX check and see if this instruction
7399 // is allowed in legacy mode and has resources which will
7400 // fit in it. Pure EVEX instructions will use set_is_evex_instruction in their definition,
7401 // else that field is set when we encode to EVEX
7402 if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7403 !_is_managed && !attributes->is_evex_instruction()) {
7404 if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7405 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7406 if (check_register_bank) {
7407 // check nds_enc and xreg_enc for upper bank usage
7408 if (nds_enc < 16 && xreg_enc < 16) {
7409 attributes->set_is_legacy_mode();
7410 }
7411 } else {
7412 attributes->set_is_legacy_mode();
7413 }
7414 }
7415 }
7416
7417 _is_managed = false;
7418 if (UseAVX > 2 && !attributes->is_legacy_mode())
7419 {
7420 bool evex_r = (xreg_enc >= 16);
7421 bool evex_v = (nds_enc >= 16);
7422 attributes->set_is_evex_instruction();
7423 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7424 } else {
7425 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7426 attributes->set_rex_vex_w(false);
7427 }
7428 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7429 }
7430 }
7431
7432 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
7433 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
7434 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
7435 bool vex_x = false;
7436 set_attributes(attributes);
7437 attributes->set_current_assembler(this);
7438 bool check_register_bank = NOT_IA32(true) IA32_ONLY(false);
7439
7440 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
7441 if (UseAVX > 2 && _legacy_mode_vl && attributes->uses_vl()) {
7442 switch (attributes->get_vector_len()) {
7443 case AVX_128bit:
7444 case AVX_256bit:
7445 if (check_register_bank) {
7446 if (dst_enc >= 16 || nds_enc >= 16 || src_enc >= 16) {
7447 // up propagate arithmetic instructions to meet RA requirements
7448 attributes->set_vector_len(AVX_512bit);
7449 } else {
7450 attributes->set_is_legacy_mode();
7451 }
7452 } else {
7453 attributes->set_is_legacy_mode();
7454 }
7455 break;
7456 }
7457 }
7458
7459 // For pure EVEX check and see if this instruction
7460 // is allowed in legacy mode and has resources which will
7461 // fit in it. Pure EVEX instructions will use set_is_evex_instruction in their definition,
7462 // else that field is set when we encode to EVEX
7463 if (UseAVX > 2 && !attributes->is_legacy_mode() &&
7464 !_is_managed && !attributes->is_evex_instruction()) {
7465 if (!_legacy_mode_vl && attributes->get_vector_len() != AVX_512bit) {
7466 if (check_register_bank) {
7467 // check dst_enc, nds_enc and src_enc for upper bank usage
7468 if (dst_enc < 16 && nds_enc < 16 && src_enc < 16) {
7469 attributes->set_is_legacy_mode();
7470 }
7471 } else {
7472 attributes->set_is_legacy_mode();
7473 }
7474 }
7475 }
7476
7477 _is_managed = false;
7478 if (UseAVX > 2 && !attributes->is_legacy_mode())
7479 {
7480 bool evex_r = (dst_enc >= 16);
7481 bool evex_v = (nds_enc >= 16);
7482 // can use vex_x as bank extender on rm encoding
7483 vex_x = (src_enc >= 16);
7484 attributes->set_is_evex_instruction();
7485 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
7486 } else {
7487 if (UseAVX > 2 && attributes->is_rex_vex_w_reverted()) {
7488 attributes->set_rex_vex_w(false);
7489 }
7490 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
7491 }
7492
7493 // return modrm byte components for operands
7494 return (((dst_enc & 7) << 3) | (src_enc & 7));
7495 }
7496
7497
7498 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
7499 VexOpcode opc, InstructionAttr *attributes) {
7500 if (UseAVX > 0) {
7501 int xreg_enc = xreg->encoding();
7502 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7503 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
7504 } else {
7505 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
7506 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
7507 }
7508 }
7509
7510 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
7511 VexOpcode opc, InstructionAttr *attributes) {
7512 int dst_enc = dst->encoding();
7513 int src_enc = src->encoding();
7514 if (UseAVX > 0) {
7515 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
7516 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
7517 } else {
7518 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
7519 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
7520 }
7521 }
7522
7523 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7524 assert(VM_Version::supports_avx(), "");
7525 assert(!VM_Version::supports_evex(), "");
7526 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7527 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7528 emit_int8((unsigned char)0xC2);
7529 emit_int8((unsigned char)(0xC0 | encode));
7530 emit_int8((unsigned char)(0xF & cop));
7531 }
7532
7533 void Assembler::blendvpd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7534 assert(VM_Version::supports_avx(), "");
7535 assert(!VM_Version::supports_evex(), "");
7536 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7537 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7538 emit_int8((unsigned char)0x4B);
7539 emit_int8((unsigned char)(0xC0 | encode));
7540 int src2_enc = src2->encoding();
7541 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7542 }
7543
7544 void Assembler::cmpps(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
7545 assert(VM_Version::supports_avx(), "");
7546 assert(!VM_Version::supports_evex(), "");
7547 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7548 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
7549 emit_int8((unsigned char)0xC2);
7550 emit_int8((unsigned char)(0xC0 | encode));
7551 emit_int8((unsigned char)(0xF & cop));
7552 }
7553
7554 void Assembler::blendvps(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
7555 assert(VM_Version::supports_avx(), "");
7556 assert(!VM_Version::supports_evex(), "");
7557 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7558 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7559 emit_int8((unsigned char)0x4A);
7560 emit_int8((unsigned char)(0xC0 | encode));
7561 int src2_enc = src2->encoding();
7562 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
7563 }
7564
7565 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src, int imm8, int vector_len) {
7566 assert(VM_Version::supports_avx2(), "");
7567 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7568 int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
7569 emit_int8((unsigned char)0x02);
7570 emit_int8((unsigned char)(0xC0 | encode));
7571 emit_int8((unsigned char)imm8);
7572 }
7573
7574 void Assembler::shlxl(Register dst, Register src1, Register src2) {
7575 assert(VM_Version::supports_bmi2(), "");
7576 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7577 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7578 emit_int8((unsigned char)0xF7);
7579 emit_int8((unsigned char)(0xC0 | encode));
7580 }
7581
7582 void Assembler::shlxq(Register dst, Register src1, Register src2) {
7583 assert(VM_Version::supports_bmi2(), "");
7584 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
7585 int encode = vex_prefix_and_encode(dst->encoding(), src2->encoding(), src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
7586 emit_int8((unsigned char)0xF7);
7587 emit_int8((unsigned char)(0xC0 | encode));
7588 }
7589
7590 #ifndef _LP64
7591
7592 void Assembler::incl(Register dst) {
7593 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7594 emit_int8(0x40 | dst->encoding());
7595 }
7596
7597 void Assembler::lea(Register dst, Address src) {
7598 leal(dst, src);
7599 }
7600
7601 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7602 InstructionMark im(this);
7603 emit_int8((unsigned char)0xC7);
7604 emit_operand(rax, dst);
7605 emit_data((int)imm32, rspec, 0);
7606 }
7607
7608 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7609 InstructionMark im(this);
7610 int encode = prefix_and_encode(dst->encoding());
7611 emit_int8((unsigned char)(0xB8 | encode));
7612 emit_data((int)imm32, rspec, 0);
7613 }
7614
7615 void Assembler::popa() { // 32bit
7616 emit_int8(0x61);
7617 }
7618
7619 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
7620 InstructionMark im(this);
7621 emit_int8(0x68);
7622 emit_data(imm32, rspec, 0);
7623 }
7624
7625 void Assembler::pusha() { // 32bit
7626 emit_int8(0x60);
7627 }
7628
7629 void Assembler::set_byte_if_not_zero(Register dst) {
7630 emit_int8(0x0F);
7631 emit_int8((unsigned char)0x95);
7632 emit_int8((unsigned char)(0xE0 | dst->encoding()));
7633 }
7634
7635 void Assembler::shldl(Register dst, Register src) {
7636 emit_int8(0x0F);
7637 emit_int8((unsigned char)0xA5);
7638 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7639 }
7640
7641 // 0F A4 / r ib
7642 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
7643 emit_int8(0x0F);
7644 emit_int8((unsigned char)0xA4);
7645 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7646 emit_int8(imm8);
7647 }
7648
7649 void Assembler::shrdl(Register dst, Register src) {
7650 emit_int8(0x0F);
7651 emit_int8((unsigned char)0xAD);
7652 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
7653 }
7654
7655 #else // LP64
7656
7657 void Assembler::set_byte_if_not_zero(Register dst) {
7658 int enc = prefix_and_encode(dst->encoding(), true);
7659 emit_int8(0x0F);
7660 emit_int8((unsigned char)0x95);
7661 emit_int8((unsigned char)(0xE0 | enc));
7662 }
7663
7664 // 64bit only pieces of the assembler
7665 // This should only be used by 64bit instructions that can use rip-relative
7666 // it cannot be used by instructions that want an immediate value.
7667
7668 bool Assembler::reachable(AddressLiteral adr) {
7669 int64_t disp;
7670 // None will force a 64bit literal to the code stream. Likely a placeholder
7671 // for something that will be patched later and we need to certain it will
7672 // always be reachable.
7673 if (adr.reloc() == relocInfo::none) {
7674 return false;
7675 }
7676 if (adr.reloc() == relocInfo::internal_word_type) {
7677 // This should be rip relative and easily reachable.
7678 return true;
7679 }
7680 if (adr.reloc() == relocInfo::virtual_call_type ||
7681 adr.reloc() == relocInfo::opt_virtual_call_type ||
7682 adr.reloc() == relocInfo::static_call_type ||
7683 adr.reloc() == relocInfo::static_stub_type ) {
7684 // This should be rip relative within the code cache and easily
7685 // reachable until we get huge code caches. (At which point
7686 // ic code is going to have issues).
7687 return true;
7688 }
7689 if (adr.reloc() != relocInfo::external_word_type &&
7690 adr.reloc() != relocInfo::poll_return_type && // these are really external_word but need special
7691 adr.reloc() != relocInfo::poll_type && // relocs to identify them
7692 adr.reloc() != relocInfo::runtime_call_type ) {
7693 return false;
7694 }
7695
7696 // Stress the correction code
7697 if (ForceUnreachable) {
7698 // Must be runtimecall reloc, see if it is in the codecache
7699 // Flipping stuff in the codecache to be unreachable causes issues
7700 // with things like inline caches where the additional instructions
7701 // are not handled.
7702 if (CodeCache::find_blob(adr._target) == NULL) {
7703 return false;
7704 }
7705 }
7706 // For external_word_type/runtime_call_type if it is reachable from where we
7707 // are now (possibly a temp buffer) and where we might end up
7708 // anywhere in the codeCache then we are always reachable.
7709 // This would have to change if we ever save/restore shared code
7710 // to be more pessimistic.
7711 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
7712 if (!is_simm32(disp)) return false;
7713 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
7714 if (!is_simm32(disp)) return false;
7715
7716 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
7717
7718 // Because rip relative is a disp + address_of_next_instruction and we
7719 // don't know the value of address_of_next_instruction we apply a fudge factor
7720 // to make sure we will be ok no matter the size of the instruction we get placed into.
7721 // We don't have to fudge the checks above here because they are already worst case.
7722
7723 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
7724 // + 4 because better safe than sorry.
7725 const int fudge = 12 + 4;
7726 if (disp < 0) {
7727 disp -= fudge;
7728 } else {
7729 disp += fudge;
7730 }
7731 return is_simm32(disp);
7732 }
7733
7734 // Check if the polling page is not reachable from the code cache using rip-relative
7735 // addressing.
7736 bool Assembler::is_polling_page_far() {
7737 intptr_t addr = (intptr_t)os::get_polling_page();
7738 return ForceUnreachable ||
7739 !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
7740 !is_simm32(addr - (intptr_t)CodeCache::high_bound());
7741 }
7742
7743 void Assembler::emit_data64(jlong data,
7744 relocInfo::relocType rtype,
7745 int format) {
7746 if (rtype == relocInfo::none) {
7747 emit_int64(data);
7748 } else {
7749 emit_data64(data, Relocation::spec_simple(rtype), format);
7750 }
7751 }
7752
7753 void Assembler::emit_data64(jlong data,
7754 RelocationHolder const& rspec,
7755 int format) {
7756 assert(imm_operand == 0, "default format must be immediate in this file");
7757 assert(imm_operand == format, "must be immediate");
7758 assert(inst_mark() != NULL, "must be inside InstructionMark");
7759 // Do not use AbstractAssembler::relocate, which is not intended for
7760 // embedded words. Instead, relocate to the enclosing instruction.
7761 code_section()->relocate(inst_mark(), rspec, format);
7762 #ifdef ASSERT
7763 check_relocation(rspec, format);
7764 #endif
7765 emit_int64(data);
7766 }
7767
7768 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
7769 if (reg_enc >= 8) {
7770 prefix(REX_B);
7771 reg_enc -= 8;
7772 } else if (byteinst && reg_enc >= 4) {
7773 prefix(REX);
7774 }
7775 return reg_enc;
7776 }
7777
7778 int Assembler::prefixq_and_encode(int reg_enc) {
7779 if (reg_enc < 8) {
7780 prefix(REX_W);
7781 } else {
7782 prefix(REX_WB);
7783 reg_enc -= 8;
7784 }
7785 return reg_enc;
7786 }
7787
7788 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
7789 if (dst_enc < 8) {
7790 if (src_enc >= 8) {
7791 prefix(REX_B);
7792 src_enc -= 8;
7793 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
7794 prefix(REX);
7795 }
7796 } else {
7797 if (src_enc < 8) {
7798 prefix(REX_R);
7799 } else {
7800 prefix(REX_RB);
7801 src_enc -= 8;
7802 }
7803 dst_enc -= 8;
7804 }
7805 return dst_enc << 3 | src_enc;
7806 }
7807
7808 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
7809 if (dst_enc < 8) {
7810 if (src_enc < 8) {
7811 prefix(REX_W);
7812 } else {
7813 prefix(REX_WB);
7814 src_enc -= 8;
7815 }
7816 } else {
7817 if (src_enc < 8) {
7818 prefix(REX_WR);
7819 } else {
7820 prefix(REX_WRB);
7821 src_enc -= 8;
7822 }
7823 dst_enc -= 8;
7824 }
7825 return dst_enc << 3 | src_enc;
7826 }
7827
7828 void Assembler::prefix(Register reg) {
7829 if (reg->encoding() >= 8) {
7830 prefix(REX_B);
7831 }
7832 }
7833
7834 void Assembler::prefix(Register dst, Register src, Prefix p) {
7835 if (src->encoding() >= 8) {
7836 p = (Prefix)(p | REX_B);
7837 }
7838 if (dst->encoding() >= 8) {
7839 p = (Prefix)( p | REX_R);
7840 }
7841 if (p != Prefix_EMPTY) {
7842 // do not generate an empty prefix
7843 prefix(p);
7844 }
7845 }
7846
7847 void Assembler::prefix(Register dst, Address adr, Prefix p) {
7848 if (adr.base_needs_rex()) {
7849 if (adr.index_needs_rex()) {
7850 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7851 } else {
7852 prefix(REX_B);
7853 }
7854 } else {
7855 if (adr.index_needs_rex()) {
7856 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
7857 }
7858 }
7859 if (dst->encoding() >= 8) {
7860 p = (Prefix)(p | REX_R);
7861 }
7862 if (p != Prefix_EMPTY) {
7863 // do not generate an empty prefix
7864 prefix(p);
7865 }
7866 }
7867
7868 void Assembler::prefix(Address adr) {
7869 if (adr.base_needs_rex()) {
7870 if (adr.index_needs_rex()) {
7871 prefix(REX_XB);
7872 } else {
7873 prefix(REX_B);
7874 }
7875 } else {
7876 if (adr.index_needs_rex()) {
7877 prefix(REX_X);
7878 }
7879 }
7880 }
7881
7882 void Assembler::prefixq(Address adr) {
7883 if (adr.base_needs_rex()) {
7884 if (adr.index_needs_rex()) {
7885 prefix(REX_WXB);
7886 } else {
7887 prefix(REX_WB);
7888 }
7889 } else {
7890 if (adr.index_needs_rex()) {
7891 prefix(REX_WX);
7892 } else {
7893 prefix(REX_W);
7894 }
7895 }
7896 }
7897
7898
7899 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
7900 if (reg->encoding() < 8) {
7901 if (adr.base_needs_rex()) {
7902 if (adr.index_needs_rex()) {
7903 prefix(REX_XB);
7904 } else {
7905 prefix(REX_B);
7906 }
7907 } else {
7908 if (adr.index_needs_rex()) {
7909 prefix(REX_X);
7910 } else if (byteinst && reg->encoding() >= 4 ) {
7911 prefix(REX);
7912 }
7913 }
7914 } else {
7915 if (adr.base_needs_rex()) {
7916 if (adr.index_needs_rex()) {
7917 prefix(REX_RXB);
7918 } else {
7919 prefix(REX_RB);
7920 }
7921 } else {
7922 if (adr.index_needs_rex()) {
7923 prefix(REX_RX);
7924 } else {
7925 prefix(REX_R);
7926 }
7927 }
7928 }
7929 }
7930
7931 void Assembler::prefixq(Address adr, Register src) {
7932 if (src->encoding() < 8) {
7933 if (adr.base_needs_rex()) {
7934 if (adr.index_needs_rex()) {
7935 prefix(REX_WXB);
7936 } else {
7937 prefix(REX_WB);
7938 }
7939 } else {
7940 if (adr.index_needs_rex()) {
7941 prefix(REX_WX);
7942 } else {
7943 prefix(REX_W);
7944 }
7945 }
7946 } else {
7947 if (adr.base_needs_rex()) {
7948 if (adr.index_needs_rex()) {
7949 prefix(REX_WRXB);
7950 } else {
7951 prefix(REX_WRB);
7952 }
7953 } else {
7954 if (adr.index_needs_rex()) {
7955 prefix(REX_WRX);
7956 } else {
7957 prefix(REX_WR);
7958 }
7959 }
7960 }
7961 }
7962
7963 void Assembler::prefix(Address adr, XMMRegister reg) {
7964 if (reg->encoding() < 8) {
7965 if (adr.base_needs_rex()) {
7966 if (adr.index_needs_rex()) {
7967 prefix(REX_XB);
7968 } else {
7969 prefix(REX_B);
7970 }
7971 } else {
7972 if (adr.index_needs_rex()) {
7973 prefix(REX_X);
7974 }
7975 }
7976 } else {
7977 if (adr.base_needs_rex()) {
7978 if (adr.index_needs_rex()) {
7979 prefix(REX_RXB);
7980 } else {
7981 prefix(REX_RB);
7982 }
7983 } else {
7984 if (adr.index_needs_rex()) {
7985 prefix(REX_RX);
7986 } else {
7987 prefix(REX_R);
7988 }
7989 }
7990 }
7991 }
7992
7993 void Assembler::prefixq(Address adr, XMMRegister src) {
7994 if (src->encoding() < 8) {
7995 if (adr.base_needs_rex()) {
7996 if (adr.index_needs_rex()) {
7997 prefix(REX_WXB);
7998 } else {
7999 prefix(REX_WB);
8000 }
8001 } else {
8002 if (adr.index_needs_rex()) {
8003 prefix(REX_WX);
8004 } else {
8005 prefix(REX_W);
8006 }
8007 }
8008 } else {
8009 if (adr.base_needs_rex()) {
8010 if (adr.index_needs_rex()) {
8011 prefix(REX_WRXB);
8012 } else {
8013 prefix(REX_WRB);
8014 }
8015 } else {
8016 if (adr.index_needs_rex()) {
8017 prefix(REX_WRX);
8018 } else {
8019 prefix(REX_WR);
8020 }
8021 }
8022 }
8023 }
8024
8025 void Assembler::adcq(Register dst, int32_t imm32) {
8026 (void) prefixq_and_encode(dst->encoding());
8027 emit_arith(0x81, 0xD0, dst, imm32);
8028 }
8029
8030 void Assembler::adcq(Register dst, Address src) {
8031 InstructionMark im(this);
8032 prefixq(src, dst);
8033 emit_int8(0x13);
8034 emit_operand(dst, src);
8035 }
8036
8037 void Assembler::adcq(Register dst, Register src) {
8038 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8039 emit_arith(0x13, 0xC0, dst, src);
8040 }
8041
8042 void Assembler::addq(Address dst, int32_t imm32) {
8043 InstructionMark im(this);
8044 prefixq(dst);
8045 emit_arith_operand(0x81, rax, dst,imm32);
8046 }
8047
8048 void Assembler::addq(Address dst, Register src) {
8049 InstructionMark im(this);
8050 prefixq(dst, src);
8051 emit_int8(0x01);
8052 emit_operand(src, dst);
8053 }
8054
8055 void Assembler::addq(Register dst, int32_t imm32) {
8056 (void) prefixq_and_encode(dst->encoding());
8057 emit_arith(0x81, 0xC0, dst, imm32);
8058 }
8059
8060 void Assembler::addq(Register dst, Address src) {
8061 InstructionMark im(this);
8062 prefixq(src, dst);
8063 emit_int8(0x03);
8064 emit_operand(dst, src);
8065 }
8066
8067 void Assembler::addq(Register dst, Register src) {
8068 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8069 emit_arith(0x03, 0xC0, dst, src);
8070 }
8071
8072 void Assembler::adcxq(Register dst, Register src) {
8073 //assert(VM_Version::supports_adx(), "adx instructions not supported");
8074 emit_int8((unsigned char)0x66);
8075 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8076 emit_int8(0x0F);
8077 emit_int8(0x38);
8078 emit_int8((unsigned char)0xF6);
8079 emit_int8((unsigned char)(0xC0 | encode));
8080 }
8081
8082 void Assembler::adoxq(Register dst, Register src) {
8083 //assert(VM_Version::supports_adx(), "adx instructions not supported");
8084 emit_int8((unsigned char)0xF3);
8085 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8086 emit_int8(0x0F);
8087 emit_int8(0x38);
8088 emit_int8((unsigned char)0xF6);
8089 emit_int8((unsigned char)(0xC0 | encode));
8090 }
8091
8092 void Assembler::andq(Address dst, int32_t imm32) {
8093 InstructionMark im(this);
8094 prefixq(dst);
8095 emit_int8((unsigned char)0x81);
8096 emit_operand(rsp, dst, 4);
8097 emit_int32(imm32);
8098 }
8099
8100 void Assembler::andq(Register dst, int32_t imm32) {
8101 (void) prefixq_and_encode(dst->encoding());
8102 emit_arith(0x81, 0xE0, dst, imm32);
8103 }
8104
8105 void Assembler::andq(Register dst, Address src) {
8106 InstructionMark im(this);
8107 prefixq(src, dst);
8108 emit_int8(0x23);
8109 emit_operand(dst, src);
8110 }
8111
8112 void Assembler::andq(Register dst, Register src) {
8113 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8114 emit_arith(0x23, 0xC0, dst, src);
8115 }
8116
8117 void Assembler::andnq(Register dst, Register src1, Register src2) {
8118 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8119 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8120 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8121 emit_int8((unsigned char)0xF2);
8122 emit_int8((unsigned char)(0xC0 | encode));
8123 }
8124
8125 void Assembler::andnq(Register dst, Register src1, Address src2) {
8126 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8127 InstructionMark im(this);
8128 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8129 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8130 emit_int8((unsigned char)0xF2);
8131 emit_operand(dst, src2);
8132 }
8133
8134 void Assembler::bsfq(Register dst, Register src) {
8135 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8136 emit_int8(0x0F);
8137 emit_int8((unsigned char)0xBC);
8138 emit_int8((unsigned char)(0xC0 | encode));
8139 }
8140
8141 void Assembler::bsrq(Register dst, Register src) {
8142 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8143 emit_int8(0x0F);
8144 emit_int8((unsigned char)0xBD);
8145 emit_int8((unsigned char)(0xC0 | encode));
8146 }
8147
8148 void Assembler::bswapq(Register reg) {
8149 int encode = prefixq_and_encode(reg->encoding());
8150 emit_int8(0x0F);
8151 emit_int8((unsigned char)(0xC8 | encode));
8152 }
8153
8154 void Assembler::blsiq(Register dst, Register src) {
8155 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8156 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8157 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8158 emit_int8((unsigned char)0xF3);
8159 emit_int8((unsigned char)(0xC0 | encode));
8160 }
8161
8162 void Assembler::blsiq(Register dst, Address src) {
8163 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8164 InstructionMark im(this);
8165 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8166 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8167 emit_int8((unsigned char)0xF3);
8168 emit_operand(rbx, src);
8169 }
8170
8171 void Assembler::blsmskq(Register dst, Register src) {
8172 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8173 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8174 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8175 emit_int8((unsigned char)0xF3);
8176 emit_int8((unsigned char)(0xC0 | encode));
8177 }
8178
8179 void Assembler::blsmskq(Register dst, Address src) {
8180 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8181 InstructionMark im(this);
8182 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8183 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8184 emit_int8((unsigned char)0xF3);
8185 emit_operand(rdx, src);
8186 }
8187
8188 void Assembler::blsrq(Register dst, Register src) {
8189 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8190 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8191 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8192 emit_int8((unsigned char)0xF3);
8193 emit_int8((unsigned char)(0xC0 | encode));
8194 }
8195
8196 void Assembler::blsrq(Register dst, Address src) {
8197 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
8198 InstructionMark im(this);
8199 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8200 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
8201 emit_int8((unsigned char)0xF3);
8202 emit_operand(rcx, src);
8203 }
8204
8205 void Assembler::cdqq() {
8206 prefix(REX_W);
8207 emit_int8((unsigned char)0x99);
8208 }
8209
8210 void Assembler::clflush(Address adr) {
8211 prefix(adr);
8212 emit_int8(0x0F);
8213 emit_int8((unsigned char)0xAE);
8214 emit_operand(rdi, adr);
8215 }
8216
8217 void Assembler::cmovq(Condition cc, Register dst, Register src) {
8218 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8219 emit_int8(0x0F);
8220 emit_int8(0x40 | cc);
8221 emit_int8((unsigned char)(0xC0 | encode));
8222 }
8223
8224 void Assembler::cmovq(Condition cc, Register dst, Address src) {
8225 InstructionMark im(this);
8226 prefixq(src, dst);
8227 emit_int8(0x0F);
8228 emit_int8(0x40 | cc);
8229 emit_operand(dst, src);
8230 }
8231
8232 void Assembler::cmpq(Address dst, int32_t imm32) {
8233 InstructionMark im(this);
8234 prefixq(dst);
8235 emit_int8((unsigned char)0x81);
8236 emit_operand(rdi, dst, 4);
8237 emit_int32(imm32);
8238 }
8239
8240 void Assembler::cmpq(Register dst, int32_t imm32) {
8241 (void) prefixq_and_encode(dst->encoding());
8242 emit_arith(0x81, 0xF8, dst, imm32);
8243 }
8244
8245 void Assembler::cmpq(Address dst, Register src) {
8246 InstructionMark im(this);
8247 prefixq(dst, src);
8248 emit_int8(0x3B);
8249 emit_operand(src, dst);
8250 }
8251
8252 void Assembler::cmpq(Register dst, Register src) {
8253 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8254 emit_arith(0x3B, 0xC0, dst, src);
8255 }
8256
8257 void Assembler::cmpq(Register dst, Address src) {
8258 InstructionMark im(this);
8259 prefixq(src, dst);
8260 emit_int8(0x3B);
8261 emit_operand(dst, src);
8262 }
8263
8264 void Assembler::cmpxchgq(Register reg, Address adr) {
8265 InstructionMark im(this);
8266 prefixq(adr, reg);
8267 emit_int8(0x0F);
8268 emit_int8((unsigned char)0xB1);
8269 emit_operand(reg, adr);
8270 }
8271
8272 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
8273 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8274 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8275 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8276 emit_int8(0x2A);
8277 emit_int8((unsigned char)(0xC0 | encode));
8278 }
8279
8280 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
8281 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8282 InstructionMark im(this);
8283 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8284 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8285 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8286 emit_int8(0x2A);
8287 emit_operand(dst, src);
8288 }
8289
8290 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
8291 NOT_LP64(assert(VM_Version::supports_sse(), ""));
8292 InstructionMark im(this);
8293 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8294 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
8295 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8296 emit_int8(0x2A);
8297 emit_operand(dst, src);
8298 }
8299
8300 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
8301 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8302 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8303 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
8304 emit_int8(0x2C);
8305 emit_int8((unsigned char)(0xC0 | encode));
8306 }
8307
8308 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
8309 NOT_LP64(assert(VM_Version::supports_sse(), ""));
8310 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8311 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
8312 emit_int8(0x2C);
8313 emit_int8((unsigned char)(0xC0 | encode));
8314 }
8315
8316 void Assembler::decl(Register dst) {
8317 // Don't use it directly. Use MacroAssembler::decrementl() instead.
8318 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
8319 int encode = prefix_and_encode(dst->encoding());
8320 emit_int8((unsigned char)0xFF);
8321 emit_int8((unsigned char)(0xC8 | encode));
8322 }
8323
8324 void Assembler::decq(Register dst) {
8325 // Don't use it directly. Use MacroAssembler::decrementq() instead.
8326 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8327 int encode = prefixq_and_encode(dst->encoding());
8328 emit_int8((unsigned char)0xFF);
8329 emit_int8(0xC8 | encode);
8330 }
8331
8332 void Assembler::decq(Address dst) {
8333 // Don't use it directly. Use MacroAssembler::decrementq() instead.
8334 InstructionMark im(this);
8335 prefixq(dst);
8336 emit_int8((unsigned char)0xFF);
8337 emit_operand(rcx, dst);
8338 }
8339
8340 void Assembler::fxrstor(Address src) {
8341 prefixq(src);
8342 emit_int8(0x0F);
8343 emit_int8((unsigned char)0xAE);
8344 emit_operand(as_Register(1), src);
8345 }
8346
8347 void Assembler::xrstor(Address src) {
8348 prefixq(src);
8349 emit_int8(0x0F);
8350 emit_int8((unsigned char)0xAE);
8351 emit_operand(as_Register(5), src);
8352 }
8353
8354 void Assembler::fxsave(Address dst) {
8355 prefixq(dst);
8356 emit_int8(0x0F);
8357 emit_int8((unsigned char)0xAE);
8358 emit_operand(as_Register(0), dst);
8359 }
8360
8361 void Assembler::xsave(Address dst) {
8362 prefixq(dst);
8363 emit_int8(0x0F);
8364 emit_int8((unsigned char)0xAE);
8365 emit_operand(as_Register(4), dst);
8366 }
8367
8368 void Assembler::idivq(Register src) {
8369 int encode = prefixq_and_encode(src->encoding());
8370 emit_int8((unsigned char)0xF7);
8371 emit_int8((unsigned char)(0xF8 | encode));
8372 }
8373
8374 void Assembler::imulq(Register dst, Register src) {
8375 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8376 emit_int8(0x0F);
8377 emit_int8((unsigned char)0xAF);
8378 emit_int8((unsigned char)(0xC0 | encode));
8379 }
8380
8381 void Assembler::imulq(Register dst, Register src, int value) {
8382 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8383 if (is8bit(value)) {
8384 emit_int8(0x6B);
8385 emit_int8((unsigned char)(0xC0 | encode));
8386 emit_int8(value & 0xFF);
8387 } else {
8388 emit_int8(0x69);
8389 emit_int8((unsigned char)(0xC0 | encode));
8390 emit_int32(value);
8391 }
8392 }
8393
8394 void Assembler::imulq(Register dst, Address src) {
8395 InstructionMark im(this);
8396 prefixq(src, dst);
8397 emit_int8(0x0F);
8398 emit_int8((unsigned char) 0xAF);
8399 emit_operand(dst, src);
8400 }
8401
8402 void Assembler::incl(Register dst) {
8403 // Don't use it directly. Use MacroAssembler::incrementl() instead.
8404 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8405 int encode = prefix_and_encode(dst->encoding());
8406 emit_int8((unsigned char)0xFF);
8407 emit_int8((unsigned char)(0xC0 | encode));
8408 }
8409
8410 void Assembler::incq(Register dst) {
8411 // Don't use it directly. Use MacroAssembler::incrementq() instead.
8412 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
8413 int encode = prefixq_and_encode(dst->encoding());
8414 emit_int8((unsigned char)0xFF);
8415 emit_int8((unsigned char)(0xC0 | encode));
8416 }
8417
8418 void Assembler::incq(Address dst) {
8419 // Don't use it directly. Use MacroAssembler::incrementq() instead.
8420 InstructionMark im(this);
8421 prefixq(dst);
8422 emit_int8((unsigned char)0xFF);
8423 emit_operand(rax, dst);
8424 }
8425
8426 void Assembler::lea(Register dst, Address src) {
8427 leaq(dst, src);
8428 }
8429
8430 void Assembler::leaq(Register dst, Address src) {
8431 InstructionMark im(this);
8432 prefixq(src, dst);
8433 emit_int8((unsigned char)0x8D);
8434 emit_operand(dst, src);
8435 }
8436
8437 void Assembler::mov64(Register dst, int64_t imm64) {
8438 InstructionMark im(this);
8439 int encode = prefixq_and_encode(dst->encoding());
8440 emit_int8((unsigned char)(0xB8 | encode));
8441 emit_int64(imm64);
8442 }
8443
8444 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
8445 InstructionMark im(this);
8446 int encode = prefixq_and_encode(dst->encoding());
8447 emit_int8(0xB8 | encode);
8448 emit_data64(imm64, rspec);
8449 }
8450
8451 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
8452 InstructionMark im(this);
8453 int encode = prefix_and_encode(dst->encoding());
8454 emit_int8((unsigned char)(0xB8 | encode));
8455 emit_data((int)imm32, rspec, narrow_oop_operand);
8456 }
8457
8458 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
8459 InstructionMark im(this);
8460 prefix(dst);
8461 emit_int8((unsigned char)0xC7);
8462 emit_operand(rax, dst, 4);
8463 emit_data((int)imm32, rspec, narrow_oop_operand);
8464 }
8465
8466 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
8467 InstructionMark im(this);
8468 int encode = prefix_and_encode(src1->encoding());
8469 emit_int8((unsigned char)0x81);
8470 emit_int8((unsigned char)(0xF8 | encode));
8471 emit_data((int)imm32, rspec, narrow_oop_operand);
8472 }
8473
8474 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
8475 InstructionMark im(this);
8476 prefix(src1);
8477 emit_int8((unsigned char)0x81);
8478 emit_operand(rax, src1, 4);
8479 emit_data((int)imm32, rspec, narrow_oop_operand);
8480 }
8481
8482 void Assembler::lzcntq(Register dst, Register src) {
8483 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
8484 emit_int8((unsigned char)0xF3);
8485 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8486 emit_int8(0x0F);
8487 emit_int8((unsigned char)0xBD);
8488 emit_int8((unsigned char)(0xC0 | encode));
8489 }
8490
8491 void Assembler::movdq(XMMRegister dst, Register src) {
8492 // table D-1 says MMX/SSE2
8493 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8494 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8495 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8496 emit_int8(0x6E);
8497 emit_int8((unsigned char)(0xC0 | encode));
8498 }
8499
8500 void Assembler::movdq(Register dst, XMMRegister src) {
8501 // table D-1 says MMX/SSE2
8502 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
8503 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
8504 // swap src/dst to get correct prefix
8505 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
8506 emit_int8(0x7E);
8507 emit_int8((unsigned char)(0xC0 | encode));
8508 }
8509
8510 void Assembler::movq(Register dst, Register src) {
8511 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8512 emit_int8((unsigned char)0x8B);
8513 emit_int8((unsigned char)(0xC0 | encode));
8514 }
8515
8516 void Assembler::movq(Register dst, Address src) {
8517 InstructionMark im(this);
8518 prefixq(src, dst);
8519 emit_int8((unsigned char)0x8B);
8520 emit_operand(dst, src);
8521 }
8522
8523 void Assembler::movq(Address dst, Register src) {
8524 InstructionMark im(this);
8525 prefixq(dst, src);
8526 emit_int8((unsigned char)0x89);
8527 emit_operand(src, dst);
8528 }
8529
8530 void Assembler::movsbq(Register dst, Address src) {
8531 InstructionMark im(this);
8532 prefixq(src, dst);
8533 emit_int8(0x0F);
8534 emit_int8((unsigned char)0xBE);
8535 emit_operand(dst, src);
8536 }
8537
8538 void Assembler::movsbq(Register dst, Register src) {
8539 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8540 emit_int8(0x0F);
8541 emit_int8((unsigned char)0xBE);
8542 emit_int8((unsigned char)(0xC0 | encode));
8543 }
8544
8545 void Assembler::movslq(Register dst, int32_t imm32) {
8546 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
8547 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
8548 // as a result we shouldn't use until tested at runtime...
8549 ShouldNotReachHere();
8550 InstructionMark im(this);
8551 int encode = prefixq_and_encode(dst->encoding());
8552 emit_int8((unsigned char)(0xC7 | encode));
8553 emit_int32(imm32);
8554 }
8555
8556 void Assembler::movslq(Address dst, int32_t imm32) {
8557 assert(is_simm32(imm32), "lost bits");
8558 InstructionMark im(this);
8559 prefixq(dst);
8560 emit_int8((unsigned char)0xC7);
8561 emit_operand(rax, dst, 4);
8562 emit_int32(imm32);
8563 }
8564
8565 void Assembler::movslq(Register dst, Address src) {
8566 InstructionMark im(this);
8567 prefixq(src, dst);
8568 emit_int8(0x63);
8569 emit_operand(dst, src);
8570 }
8571
8572 void Assembler::movslq(Register dst, Register src) {
8573 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8574 emit_int8(0x63);
8575 emit_int8((unsigned char)(0xC0 | encode));
8576 }
8577
8578 void Assembler::movswq(Register dst, Address src) {
8579 InstructionMark im(this);
8580 prefixq(src, dst);
8581 emit_int8(0x0F);
8582 emit_int8((unsigned char)0xBF);
8583 emit_operand(dst, src);
8584 }
8585
8586 void Assembler::movswq(Register dst, Register src) {
8587 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8588 emit_int8((unsigned char)0x0F);
8589 emit_int8((unsigned char)0xBF);
8590 emit_int8((unsigned char)(0xC0 | encode));
8591 }
8592
8593 void Assembler::movzbq(Register dst, Address src) {
8594 InstructionMark im(this);
8595 prefixq(src, dst);
8596 emit_int8((unsigned char)0x0F);
8597 emit_int8((unsigned char)0xB6);
8598 emit_operand(dst, src);
8599 }
8600
8601 void Assembler::movzbq(Register dst, Register src) {
8602 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8603 emit_int8(0x0F);
8604 emit_int8((unsigned char)0xB6);
8605 emit_int8(0xC0 | encode);
8606 }
8607
8608 void Assembler::movzwq(Register dst, Address src) {
8609 InstructionMark im(this);
8610 prefixq(src, dst);
8611 emit_int8((unsigned char)0x0F);
8612 emit_int8((unsigned char)0xB7);
8613 emit_operand(dst, src);
8614 }
8615
8616 void Assembler::movzwq(Register dst, Register src) {
8617 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8618 emit_int8((unsigned char)0x0F);
8619 emit_int8((unsigned char)0xB7);
8620 emit_int8((unsigned char)(0xC0 | encode));
8621 }
8622
8623 void Assembler::mulq(Address src) {
8624 InstructionMark im(this);
8625 prefixq(src);
8626 emit_int8((unsigned char)0xF7);
8627 emit_operand(rsp, src);
8628 }
8629
8630 void Assembler::mulq(Register src) {
8631 int encode = prefixq_and_encode(src->encoding());
8632 emit_int8((unsigned char)0xF7);
8633 emit_int8((unsigned char)(0xE0 | encode));
8634 }
8635
8636 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
8637 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8638 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8639 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
8640 emit_int8((unsigned char)0xF6);
8641 emit_int8((unsigned char)(0xC0 | encode));
8642 }
8643
8644 void Assembler::negq(Register dst) {
8645 int encode = prefixq_and_encode(dst->encoding());
8646 emit_int8((unsigned char)0xF7);
8647 emit_int8((unsigned char)(0xD8 | encode));
8648 }
8649
8650 void Assembler::notq(Register dst) {
8651 int encode = prefixq_and_encode(dst->encoding());
8652 emit_int8((unsigned char)0xF7);
8653 emit_int8((unsigned char)(0xD0 | encode));
8654 }
8655
8656 void Assembler::orq(Address dst, int32_t imm32) {
8657 InstructionMark im(this);
8658 prefixq(dst);
8659 emit_int8((unsigned char)0x81);
8660 emit_operand(rcx, dst, 4);
8661 emit_int32(imm32);
8662 }
8663
8664 void Assembler::orq(Register dst, int32_t imm32) {
8665 (void) prefixq_and_encode(dst->encoding());
8666 emit_arith(0x81, 0xC8, dst, imm32);
8667 }
8668
8669 void Assembler::orq(Register dst, Address src) {
8670 InstructionMark im(this);
8671 prefixq(src, dst);
8672 emit_int8(0x0B);
8673 emit_operand(dst, src);
8674 }
8675
8676 void Assembler::orq(Register dst, Register src) {
8677 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8678 emit_arith(0x0B, 0xC0, dst, src);
8679 }
8680
8681 void Assembler::popa() { // 64bit
8682 movq(r15, Address(rsp, 0));
8683 movq(r14, Address(rsp, wordSize));
8684 movq(r13, Address(rsp, 2 * wordSize));
8685 movq(r12, Address(rsp, 3 * wordSize));
8686 movq(r11, Address(rsp, 4 * wordSize));
8687 movq(r10, Address(rsp, 5 * wordSize));
8688 movq(r9, Address(rsp, 6 * wordSize));
8689 movq(r8, Address(rsp, 7 * wordSize));
8690 movq(rdi, Address(rsp, 8 * wordSize));
8691 movq(rsi, Address(rsp, 9 * wordSize));
8692 movq(rbp, Address(rsp, 10 * wordSize));
8693 // skip rsp
8694 movq(rbx, Address(rsp, 12 * wordSize));
8695 movq(rdx, Address(rsp, 13 * wordSize));
8696 movq(rcx, Address(rsp, 14 * wordSize));
8697 movq(rax, Address(rsp, 15 * wordSize));
8698
8699 addq(rsp, 16 * wordSize);
8700 }
8701
8702 void Assembler::popcntq(Register dst, Address src) {
8703 assert(VM_Version::supports_popcnt(), "must support");
8704 InstructionMark im(this);
8705 emit_int8((unsigned char)0xF3);
8706 prefixq(src, dst);
8707 emit_int8((unsigned char)0x0F);
8708 emit_int8((unsigned char)0xB8);
8709 emit_operand(dst, src);
8710 }
8711
8712 void Assembler::popcntq(Register dst, Register src) {
8713 assert(VM_Version::supports_popcnt(), "must support");
8714 emit_int8((unsigned char)0xF3);
8715 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8716 emit_int8((unsigned char)0x0F);
8717 emit_int8((unsigned char)0xB8);
8718 emit_int8((unsigned char)(0xC0 | encode));
8719 }
8720
8721 void Assembler::popq(Address dst) {
8722 InstructionMark im(this);
8723 prefixq(dst);
8724 emit_int8((unsigned char)0x8F);
8725 emit_operand(rax, dst);
8726 }
8727
8728 void Assembler::pusha() { // 64bit
8729 // we have to store original rsp. ABI says that 128 bytes
8730 // below rsp are local scratch.
8731 movq(Address(rsp, -5 * wordSize), rsp);
8732
8733 subq(rsp, 16 * wordSize);
8734
8735 movq(Address(rsp, 15 * wordSize), rax);
8736 movq(Address(rsp, 14 * wordSize), rcx);
8737 movq(Address(rsp, 13 * wordSize), rdx);
8738 movq(Address(rsp, 12 * wordSize), rbx);
8739 // skip rsp
8740 movq(Address(rsp, 10 * wordSize), rbp);
8741 movq(Address(rsp, 9 * wordSize), rsi);
8742 movq(Address(rsp, 8 * wordSize), rdi);
8743 movq(Address(rsp, 7 * wordSize), r8);
8744 movq(Address(rsp, 6 * wordSize), r9);
8745 movq(Address(rsp, 5 * wordSize), r10);
8746 movq(Address(rsp, 4 * wordSize), r11);
8747 movq(Address(rsp, 3 * wordSize), r12);
8748 movq(Address(rsp, 2 * wordSize), r13);
8749 movq(Address(rsp, wordSize), r14);
8750 movq(Address(rsp, 0), r15);
8751 }
8752
8753 void Assembler::pushq(Address src) {
8754 InstructionMark im(this);
8755 prefixq(src);
8756 emit_int8((unsigned char)0xFF);
8757 emit_operand(rsi, src);
8758 }
8759
8760 void Assembler::rclq(Register dst, int imm8) {
8761 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8762 int encode = prefixq_and_encode(dst->encoding());
8763 if (imm8 == 1) {
8764 emit_int8((unsigned char)0xD1);
8765 emit_int8((unsigned char)(0xD0 | encode));
8766 } else {
8767 emit_int8((unsigned char)0xC1);
8768 emit_int8((unsigned char)(0xD0 | encode));
8769 emit_int8(imm8);
8770 }
8771 }
8772
8773 void Assembler::rcrq(Register dst, int imm8) {
8774 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8775 int encode = prefixq_and_encode(dst->encoding());
8776 if (imm8 == 1) {
8777 emit_int8((unsigned char)0xD1);
8778 emit_int8((unsigned char)(0xD8 | encode));
8779 } else {
8780 emit_int8((unsigned char)0xC1);
8781 emit_int8((unsigned char)(0xD8 | encode));
8782 emit_int8(imm8);
8783 }
8784 }
8785
8786 void Assembler::rorq(Register dst, int imm8) {
8787 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8788 int encode = prefixq_and_encode(dst->encoding());
8789 if (imm8 == 1) {
8790 emit_int8((unsigned char)0xD1);
8791 emit_int8((unsigned char)(0xC8 | encode));
8792 } else {
8793 emit_int8((unsigned char)0xC1);
8794 emit_int8((unsigned char)(0xc8 | encode));
8795 emit_int8(imm8);
8796 }
8797 }
8798
8799 void Assembler::rorxq(Register dst, Register src, int imm8) {
8800 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8801 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8802 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8803 emit_int8((unsigned char)0xF0);
8804 emit_int8((unsigned char)(0xC0 | encode));
8805 emit_int8(imm8);
8806 }
8807
8808 void Assembler::rorxd(Register dst, Register src, int imm8) {
8809 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
8810 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
8811 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
8812 emit_int8((unsigned char)0xF0);
8813 emit_int8((unsigned char)(0xC0 | encode));
8814 emit_int8(imm8);
8815 }
8816
8817 void Assembler::sarq(Register dst, int imm8) {
8818 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8819 int encode = prefixq_and_encode(dst->encoding());
8820 if (imm8 == 1) {
8821 emit_int8((unsigned char)0xD1);
8822 emit_int8((unsigned char)(0xF8 | encode));
8823 } else {
8824 emit_int8((unsigned char)0xC1);
8825 emit_int8((unsigned char)(0xF8 | encode));
8826 emit_int8(imm8);
8827 }
8828 }
8829
8830 void Assembler::sarq(Register dst) {
8831 int encode = prefixq_and_encode(dst->encoding());
8832 emit_int8((unsigned char)0xD3);
8833 emit_int8((unsigned char)(0xF8 | encode));
8834 }
8835
8836 void Assembler::sbbq(Address dst, int32_t imm32) {
8837 InstructionMark im(this);
8838 prefixq(dst);
8839 emit_arith_operand(0x81, rbx, dst, imm32);
8840 }
8841
8842 void Assembler::sbbq(Register dst, int32_t imm32) {
8843 (void) prefixq_and_encode(dst->encoding());
8844 emit_arith(0x81, 0xD8, dst, imm32);
8845 }
8846
8847 void Assembler::sbbq(Register dst, Address src) {
8848 InstructionMark im(this);
8849 prefixq(src, dst);
8850 emit_int8(0x1B);
8851 emit_operand(dst, src);
8852 }
8853
8854 void Assembler::sbbq(Register dst, Register src) {
8855 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8856 emit_arith(0x1B, 0xC0, dst, src);
8857 }
8858
8859 void Assembler::shlq(Register dst, int imm8) {
8860 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8861 int encode = prefixq_and_encode(dst->encoding());
8862 if (imm8 == 1) {
8863 emit_int8((unsigned char)0xD1);
8864 emit_int8((unsigned char)(0xE0 | encode));
8865 } else {
8866 emit_int8((unsigned char)0xC1);
8867 emit_int8((unsigned char)(0xE0 | encode));
8868 emit_int8(imm8);
8869 }
8870 }
8871
8872 void Assembler::shlq(Register dst) {
8873 int encode = prefixq_and_encode(dst->encoding());
8874 emit_int8((unsigned char)0xD3);
8875 emit_int8((unsigned char)(0xE0 | encode));
8876 }
8877
8878 void Assembler::shrq(Register dst, int imm8) {
8879 assert(isShiftCount(imm8 >> 1), "illegal shift count");
8880 int encode = prefixq_and_encode(dst->encoding());
8881 emit_int8((unsigned char)0xC1);
8882 emit_int8((unsigned char)(0xE8 | encode));
8883 emit_int8(imm8);
8884 }
8885
8886 void Assembler::shrq(Register dst) {
8887 int encode = prefixq_and_encode(dst->encoding());
8888 emit_int8((unsigned char)0xD3);
8889 emit_int8(0xE8 | encode);
8890 }
8891
8892 void Assembler::subq(Address dst, int32_t imm32) {
8893 InstructionMark im(this);
8894 prefixq(dst);
8895 emit_arith_operand(0x81, rbp, dst, imm32);
8896 }
8897
8898 void Assembler::subq(Address dst, Register src) {
8899 InstructionMark im(this);
8900 prefixq(dst, src);
8901 emit_int8(0x29);
8902 emit_operand(src, dst);
8903 }
8904
8905 void Assembler::subq(Register dst, int32_t imm32) {
8906 (void) prefixq_and_encode(dst->encoding());
8907 emit_arith(0x81, 0xE8, dst, imm32);
8908 }
8909
8910 // Force generation of a 4 byte immediate value even if it fits into 8bit
8911 void Assembler::subq_imm32(Register dst, int32_t imm32) {
8912 (void) prefixq_and_encode(dst->encoding());
8913 emit_arith_imm32(0x81, 0xE8, dst, imm32);
8914 }
8915
8916 void Assembler::subq(Register dst, Address src) {
8917 InstructionMark im(this);
8918 prefixq(src, dst);
8919 emit_int8(0x2B);
8920 emit_operand(dst, src);
8921 }
8922
8923 void Assembler::subq(Register dst, Register src) {
8924 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8925 emit_arith(0x2B, 0xC0, dst, src);
8926 }
8927
8928 void Assembler::testq(Register dst, int32_t imm32) {
8929 // not using emit_arith because test
8930 // doesn't support sign-extension of
8931 // 8bit operands
8932 int encode = dst->encoding();
8933 if (encode == 0) {
8934 prefix(REX_W);
8935 emit_int8((unsigned char)0xA9);
8936 } else {
8937 encode = prefixq_and_encode(encode);
8938 emit_int8((unsigned char)0xF7);
8939 emit_int8((unsigned char)(0xC0 | encode));
8940 }
8941 emit_int32(imm32);
8942 }
8943
8944 void Assembler::testq(Register dst, Register src) {
8945 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8946 emit_arith(0x85, 0xC0, dst, src);
8947 }
8948
8949 void Assembler::xaddq(Address dst, Register src) {
8950 InstructionMark im(this);
8951 prefixq(dst, src);
8952 emit_int8(0x0F);
8953 emit_int8((unsigned char)0xC1);
8954 emit_operand(src, dst);
8955 }
8956
8957 void Assembler::xchgq(Register dst, Address src) {
8958 InstructionMark im(this);
8959 prefixq(src, dst);
8960 emit_int8((unsigned char)0x87);
8961 emit_operand(dst, src);
8962 }
8963
8964 void Assembler::xchgq(Register dst, Register src) {
8965 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
8966 emit_int8((unsigned char)0x87);
8967 emit_int8((unsigned char)(0xc0 | encode));
8968 }
8969
8970 void Assembler::xorq(Register dst, Register src) {
8971 (void) prefixq_and_encode(dst->encoding(), src->encoding());
8972 emit_arith(0x33, 0xC0, dst, src);
8973 }
8974
8975 void Assembler::xorq(Register dst, Address src) {
8976 InstructionMark im(this);
8977 prefixq(src, dst);
8978 emit_int8(0x33);
8979 emit_operand(dst, src);
8980 }
8981
8982 #endif // !LP64