1 /*
2 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableModRefBS.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/biasedLocking.hpp"
34 #include "runtime/interfaceSupport.hpp"
35 #include "runtime/objectMonitor.hpp"
36 #include "runtime/os.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "runtime/stubRoutines.hpp"
39 #include "utilities/macros.hpp"
40 #if INCLUDE_ALL_GCS
41 #include "gc/g1/g1CollectedHeap.inline.hpp"
42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
43 #include "gc/g1/heapRegion.hpp"
44 #endif // INCLUDE_ALL_GCS
45
46 #ifdef PRODUCT
47 #define BLOCK_COMMENT(str) /* nothing */
48 #define STOP(error) stop(error)
49 #else
50 #define BLOCK_COMMENT(str) block_comment(str)
51 #define STOP(error) block_comment(error); stop(error)
52 #endif
53
54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
55 // Implementation of AddressLiteral
56
57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
59 // -----------------Table 4.5 -------------------- //
60 16, 32, 64, // EVEX_FV(0)
61 4, 4, 4, // EVEX_FV(1) - with Evex.b
62 16, 32, 64, // EVEX_FV(2) - with Evex.w
63 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
64 8, 16, 32, // EVEX_HV(0)
65 4, 4, 4, // EVEX_HV(1) - with Evex.b
66 // -----------------Table 4.6 -------------------- //
67 16, 32, 64, // EVEX_FVM(0)
68 1, 1, 1, // EVEX_T1S(0)
69 2, 2, 2, // EVEX_T1S(1)
70 4, 4, 4, // EVEX_T1S(2)
71 8, 8, 8, // EVEX_T1S(3)
72 4, 4, 4, // EVEX_T1F(0)
73 8, 8, 8, // EVEX_T1F(1)
74 8, 8, 8, // EVEX_T2(0)
75 0, 16, 16, // EVEX_T2(1)
76 0, 16, 16, // EVEX_T4(0)
77 0, 0, 32, // EVEX_T4(1)
78 0, 0, 32, // EVEX_T8(0)
79 8, 16, 32, // EVEX_HVM(0)
80 4, 8, 16, // EVEX_QVM(0)
81 2, 4, 8, // EVEX_OVM(0)
82 16, 16, 16, // EVEX_M128(0)
83 8, 32, 64, // EVEX_DUP(0)
84 0, 0, 0 // EVEX_NTUP
85 };
86
87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
88 _is_lval = false;
89 _target = target;
90 switch (rtype) {
91 case relocInfo::oop_type:
92 case relocInfo::metadata_type:
93 // Oops are a special case. Normally they would be their own section
94 // but in cases like icBuffer they are literals in the code stream that
95 // we don't have a section for. We use none so that we get a literal address
96 // which is always patchable.
97 break;
98 case relocInfo::external_word_type:
99 _rspec = external_word_Relocation::spec(target);
100 break;
101 case relocInfo::internal_word_type:
102 _rspec = internal_word_Relocation::spec(target);
103 break;
104 case relocInfo::opt_virtual_call_type:
105 _rspec = opt_virtual_call_Relocation::spec();
106 break;
107 case relocInfo::static_call_type:
108 _rspec = static_call_Relocation::spec();
109 break;
110 case relocInfo::runtime_call_type:
111 _rspec = runtime_call_Relocation::spec();
112 break;
113 case relocInfo::poll_type:
114 case relocInfo::poll_return_type:
115 _rspec = Relocation::spec_simple(rtype);
116 break;
117 case relocInfo::none:
118 break;
119 default:
120 ShouldNotReachHere();
121 break;
122 }
123 }
124
125 // Implementation of Address
126
127 #ifdef _LP64
128
129 Address Address::make_array(ArrayAddress adr) {
130 // Not implementable on 64bit machines
131 // Should have been handled higher up the call chain.
132 ShouldNotReachHere();
133 return Address();
134 }
135
136 // exceedingly dangerous constructor
137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
138 _base = noreg;
139 _index = noreg;
140 _scale = no_scale;
141 _disp = disp;
142 switch (rtype) {
143 case relocInfo::external_word_type:
144 _rspec = external_word_Relocation::spec(loc);
145 break;
146 case relocInfo::internal_word_type:
147 _rspec = internal_word_Relocation::spec(loc);
148 break;
149 case relocInfo::runtime_call_type:
150 // HMM
151 _rspec = runtime_call_Relocation::spec();
152 break;
153 case relocInfo::poll_type:
154 case relocInfo::poll_return_type:
155 _rspec = Relocation::spec_simple(rtype);
156 break;
157 case relocInfo::none:
158 break;
159 default:
160 ShouldNotReachHere();
161 }
162 }
163 #else // LP64
164
165 Address Address::make_array(ArrayAddress adr) {
166 AddressLiteral base = adr.base();
167 Address index = adr.index();
168 assert(index._disp == 0, "must not have disp"); // maybe it can?
169 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
170 array._rspec = base._rspec;
171 return array;
172 }
173
174 // exceedingly dangerous constructor
175 Address::Address(address loc, RelocationHolder spec) {
176 _base = noreg;
177 _index = noreg;
178 _scale = no_scale;
179 _disp = (intptr_t) loc;
180 _rspec = spec;
181 }
182
183 #endif // _LP64
184
185
186
187 // Convert the raw encoding form into the form expected by the constructor for
188 // Address. An index of 4 (rsp) corresponds to having no index, so convert
189 // that to noreg for the Address constructor.
190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
191 RelocationHolder rspec;
192 if (disp_reloc != relocInfo::none) {
193 rspec = Relocation::spec_simple(disp_reloc);
194 }
195 bool valid_index = index != rsp->encoding();
196 if (valid_index) {
197 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
198 madr._rspec = rspec;
199 return madr;
200 } else {
201 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
202 madr._rspec = rspec;
203 return madr;
204 }
205 }
206
207 // Implementation of Assembler
208
209 int AbstractAssembler::code_fill_byte() {
210 return (u_char)'\xF4'; // hlt
211 }
212
213 // make this go away someday
214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
215 if (rtype == relocInfo::none)
216 emit_int32(data);
217 else
218 emit_data(data, Relocation::spec_simple(rtype), format);
219 }
220
221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
222 assert(imm_operand == 0, "default format must be immediate in this file");
223 assert(inst_mark() != NULL, "must be inside InstructionMark");
224 if (rspec.type() != relocInfo::none) {
225 #ifdef ASSERT
226 check_relocation(rspec, format);
227 #endif
228 // Do not use AbstractAssembler::relocate, which is not intended for
229 // embedded words. Instead, relocate to the enclosing instruction.
230
231 // hack. call32 is too wide for mask so use disp32
232 if (format == call32_operand)
233 code_section()->relocate(inst_mark(), rspec, disp32_operand);
234 else
235 code_section()->relocate(inst_mark(), rspec, format);
236 }
237 emit_int32(data);
238 }
239
240 static int encode(Register r) {
241 int enc = r->encoding();
242 if (enc >= 8) {
243 enc -= 8;
244 }
245 return enc;
246 }
247
248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
249 assert(dst->has_byte_register(), "must have byte register");
250 assert(isByte(op1) && isByte(op2), "wrong opcode");
251 assert(isByte(imm8), "not a byte");
252 assert((op1 & 0x01) == 0, "should be 8bit operation");
253 emit_int8(op1);
254 emit_int8(op2 | encode(dst));
255 emit_int8(imm8);
256 }
257
258
259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
260 assert(isByte(op1) && isByte(op2), "wrong opcode");
261 assert((op1 & 0x01) == 1, "should be 32bit operation");
262 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
263 if (is8bit(imm32)) {
264 emit_int8(op1 | 0x02); // set sign bit
265 emit_int8(op2 | encode(dst));
266 emit_int8(imm32 & 0xFF);
267 } else {
268 emit_int8(op1);
269 emit_int8(op2 | encode(dst));
270 emit_int32(imm32);
271 }
272 }
273
274 // Force generation of a 4 byte immediate value even if it fits into 8bit
275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
276 assert(isByte(op1) && isByte(op2), "wrong opcode");
277 assert((op1 & 0x01) == 1, "should be 32bit operation");
278 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
279 emit_int8(op1);
280 emit_int8(op2 | encode(dst));
281 emit_int32(imm32);
282 }
283
284 // immediate-to-memory forms
285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
286 assert((op1 & 0x01) == 1, "should be 32bit operation");
287 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
288 if (is8bit(imm32)) {
289 emit_int8(op1 | 0x02); // set sign bit
290 emit_operand(rm, adr, 1);
291 emit_int8(imm32 & 0xFF);
292 } else {
293 emit_int8(op1);
294 emit_operand(rm, adr, 4);
295 emit_int32(imm32);
296 }
297 }
298
299
300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
301 assert(isByte(op1) && isByte(op2), "wrong opcode");
302 emit_int8(op1);
303 emit_int8(op2 | encode(dst) << 3 | encode(src));
304 }
305
306
307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
308 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
309 int mod_idx = 0;
310 // We will test if the displacement fits the compressed format and if so
311 // apply the compression to the displacment iff the result is8bit.
312 if (VM_Version::supports_evex() && is_evex_inst) {
313 switch (cur_tuple_type) {
314 case EVEX_FV:
315 if ((cur_encoding & VEX_W) == VEX_W) {
316 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
317 } else {
318 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
319 }
320 break;
321
322 case EVEX_HV:
323 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
324 break;
325
326 case EVEX_FVM:
327 break;
328
329 case EVEX_T1S:
330 switch (in_size_in_bits) {
331 case EVEX_8bit:
332 break;
333
334 case EVEX_16bit:
335 mod_idx = 1;
336 break;
337
338 case EVEX_32bit:
339 mod_idx = 2;
340 break;
341
342 case EVEX_64bit:
343 mod_idx = 3;
344 break;
345 }
346 break;
347
348 case EVEX_T1F:
349 case EVEX_T2:
350 case EVEX_T4:
351 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
352 break;
353
354 case EVEX_T8:
355 break;
356
357 case EVEX_HVM:
358 break;
359
360 case EVEX_QVM:
361 break;
362
363 case EVEX_OVM:
364 break;
365
366 case EVEX_M128:
367 break;
368
369 case EVEX_DUP:
370 break;
371
372 default:
373 assert(0, "no valid evex tuple_table entry");
374 break;
375 }
376
377 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
378 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
379 if ((disp % disp_factor) == 0) {
380 int new_disp = disp / disp_factor;
381 if ((-0x80 <= new_disp && new_disp < 0x80)) {
382 disp = new_disp;
383 }
384 } else {
385 return false;
386 }
387 }
388 }
389 return (-0x80 <= disp && disp < 0x80);
390 }
391
392
393 bool Assembler::emit_compressed_disp_byte(int &disp) {
394 int mod_idx = 0;
395 // We will test if the displacement fits the compressed format and if so
396 // apply the compression to the displacment iff the result is8bit.
397 if (VM_Version::supports_evex() && (_attributes != NULL) && _attributes->is_evex_instruction()) {
398 int evex_encoding = _attributes->get_evex_encoding();
399 int tuple_type = _attributes->get_tuple_type();
400 switch (tuple_type) {
401 case EVEX_FV:
402 if ((evex_encoding & VEX_W) == VEX_W) {
403 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 3 : 2;
404 } else {
405 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
406 }
407 break;
408
409 case EVEX_HV:
410 mod_idx = ((evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
411 break;
412
413 case EVEX_FVM:
414 break;
415
416 case EVEX_T1S:
417 switch (_attributes->get_input_size()) {
418 case EVEX_8bit:
419 break;
420
421 case EVEX_16bit:
422 mod_idx = 1;
423 break;
424
425 case EVEX_32bit:
426 mod_idx = 2;
427 break;
428
429 case EVEX_64bit:
430 mod_idx = 3;
431 break;
432 }
433 break;
434
435 case EVEX_T1F:
436 case EVEX_T2:
437 case EVEX_T4:
438 mod_idx = (_attributes->get_input_size() == EVEX_64bit) ? 1 : 0;
439 break;
440
441 case EVEX_T8:
442 break;
443
444 case EVEX_HVM:
445 break;
446
447 case EVEX_QVM:
448 break;
449
450 case EVEX_OVM:
451 break;
452
453 case EVEX_M128:
454 break;
455
456 case EVEX_DUP:
457 break;
458
459 default:
460 assert(0, "no valid evex tuple_table entry");
461 break;
462 }
463
464 int vector_len = _attributes->get_vector_len();
465 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
466 int disp_factor = tuple_table[tuple_type + mod_idx][vector_len];
467 if ((disp % disp_factor) == 0) {
468 int new_disp = disp / disp_factor;
469 if (is8bit(new_disp)) {
470 disp = new_disp;
471 }
472 } else {
473 return false;
474 }
475 }
476 }
477 return is8bit(disp);
478 }
479
480
481 void Assembler::emit_operand(Register reg, Register base, Register index,
482 Address::ScaleFactor scale, int disp,
483 RelocationHolder const& rspec,
484 int rip_relative_correction) {
485 relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
486
487 // Encode the registers as needed in the fields they are used in
488
489 int regenc = encode(reg) << 3;
490 int indexenc = index->is_valid() ? encode(index) << 3 : 0;
491 int baseenc = base->is_valid() ? encode(base) : 0;
492
493 if (base->is_valid()) {
494 if (index->is_valid()) {
495 assert(scale != Address::no_scale, "inconsistent address");
496 // [base + index*scale + disp]
497 if (disp == 0 && rtype == relocInfo::none &&
498 base != rbp LP64_ONLY(&& base != r13)) {
499 // [base + index*scale]
500 // [00 reg 100][ss index base]
501 assert(index != rsp, "illegal addressing mode");
502 emit_int8(0x04 | regenc);
503 emit_int8(scale << 6 | indexenc | baseenc);
504 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
505 // [base + index*scale + imm8]
506 // [01 reg 100][ss index base] imm8
507 assert(index != rsp, "illegal addressing mode");
508 emit_int8(0x44 | regenc);
509 emit_int8(scale << 6 | indexenc | baseenc);
510 emit_int8(disp & 0xFF);
511 } else {
512 // [base + index*scale + disp32]
513 // [10 reg 100][ss index base] disp32
514 assert(index != rsp, "illegal addressing mode");
515 emit_int8(0x84 | regenc);
516 emit_int8(scale << 6 | indexenc | baseenc);
517 emit_data(disp, rspec, disp32_operand);
518 }
519 } else if (base == rsp LP64_ONLY(|| base == r12)) {
520 // [rsp + disp]
521 if (disp == 0 && rtype == relocInfo::none) {
522 // [rsp]
523 // [00 reg 100][00 100 100]
524 emit_int8(0x04 | regenc);
525 emit_int8(0x24);
526 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
527 // [rsp + imm8]
528 // [01 reg 100][00 100 100] disp8
529 emit_int8(0x44 | regenc);
530 emit_int8(0x24);
531 emit_int8(disp & 0xFF);
532 } else {
533 // [rsp + imm32]
534 // [10 reg 100][00 100 100] disp32
535 emit_int8(0x84 | regenc);
536 emit_int8(0x24);
537 emit_data(disp, rspec, disp32_operand);
538 }
539 } else {
540 // [base + disp]
541 assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
542 if (disp == 0 && rtype == relocInfo::none &&
543 base != rbp LP64_ONLY(&& base != r13)) {
544 // [base]
545 // [00 reg base]
546 emit_int8(0x00 | regenc | baseenc);
547 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
548 // [base + disp8]
549 // [01 reg base] disp8
550 emit_int8(0x40 | regenc | baseenc);
551 emit_int8(disp & 0xFF);
552 } else {
553 // [base + disp32]
554 // [10 reg base] disp32
555 emit_int8(0x80 | regenc | baseenc);
556 emit_data(disp, rspec, disp32_operand);
557 }
558 }
559 } else {
560 if (index->is_valid()) {
561 assert(scale != Address::no_scale, "inconsistent address");
562 // [index*scale + disp]
563 // [00 reg 100][ss index 101] disp32
564 assert(index != rsp, "illegal addressing mode");
565 emit_int8(0x04 | regenc);
566 emit_int8(scale << 6 | indexenc | 0x05);
567 emit_data(disp, rspec, disp32_operand);
568 } else if (rtype != relocInfo::none ) {
569 // [disp] (64bit) RIP-RELATIVE (32bit) abs
570 // [00 000 101] disp32
571
572 emit_int8(0x05 | regenc);
573 // Note that the RIP-rel. correction applies to the generated
574 // disp field, but _not_ to the target address in the rspec.
575
576 // disp was created by converting the target address minus the pc
577 // at the start of the instruction. That needs more correction here.
578 // intptr_t disp = target - next_ip;
579 assert(inst_mark() != NULL, "must be inside InstructionMark");
580 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
581 int64_t adjusted = disp;
582 // Do rip-rel adjustment for 64bit
583 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
584 assert(is_simm32(adjusted),
585 "must be 32bit offset (RIP relative address)");
586 emit_data((int32_t) adjusted, rspec, disp32_operand);
587
588 } else {
589 // 32bit never did this, did everything as the rip-rel/disp code above
590 // [disp] ABSOLUTE
591 // [00 reg 100][00 100 101] disp32
592 emit_int8(0x04 | regenc);
593 emit_int8(0x25);
594 emit_data(disp, rspec, disp32_operand);
595 }
596 }
597 }
598
599 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
600 Address::ScaleFactor scale, int disp,
601 RelocationHolder const& rspec) {
602 if (UseAVX > 2) {
603 int xreg_enc = reg->encoding();
604 if (xreg_enc > 15) {
605 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
606 emit_operand((Register)new_reg, base, index, scale, disp, rspec);
607 return;
608 }
609 }
610 emit_operand((Register)reg, base, index, scale, disp, rspec);
611 }
612
613 // Secret local extension to Assembler::WhichOperand:
614 #define end_pc_operand (_WhichOperand_limit)
615
616 address Assembler::locate_operand(address inst, WhichOperand which) {
617 // Decode the given instruction, and return the address of
618 // an embedded 32-bit operand word.
619
620 // If "which" is disp32_operand, selects the displacement portion
621 // of an effective address specifier.
622 // If "which" is imm64_operand, selects the trailing immediate constant.
623 // If "which" is call32_operand, selects the displacement of a call or jump.
624 // Caller is responsible for ensuring that there is such an operand,
625 // and that it is 32/64 bits wide.
626
627 // If "which" is end_pc_operand, find the end of the instruction.
628
629 address ip = inst;
630 bool is_64bit = false;
631
632 debug_only(bool has_disp32 = false);
633 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
634
635 again_after_prefix:
636 switch (0xFF & *ip++) {
637
638 // These convenience macros generate groups of "case" labels for the switch.
639 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
640 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
641 case (x)+4: case (x)+5: case (x)+6: case (x)+7
642 #define REP16(x) REP8((x)+0): \
643 case REP8((x)+8)
644
645 case CS_segment:
646 case SS_segment:
647 case DS_segment:
648 case ES_segment:
649 case FS_segment:
650 case GS_segment:
651 // Seems dubious
652 LP64_ONLY(assert(false, "shouldn't have that prefix"));
653 assert(ip == inst+1, "only one prefix allowed");
654 goto again_after_prefix;
655
656 case 0x67:
657 case REX:
658 case REX_B:
659 case REX_X:
660 case REX_XB:
661 case REX_R:
662 case REX_RB:
663 case REX_RX:
664 case REX_RXB:
665 NOT_LP64(assert(false, "64bit prefixes"));
666 goto again_after_prefix;
667
668 case REX_W:
669 case REX_WB:
670 case REX_WX:
671 case REX_WXB:
672 case REX_WR:
673 case REX_WRB:
674 case REX_WRX:
675 case REX_WRXB:
676 NOT_LP64(assert(false, "64bit prefixes"));
677 is_64bit = true;
678 goto again_after_prefix;
679
680 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
681 case 0x88: // movb a, r
682 case 0x89: // movl a, r
683 case 0x8A: // movb r, a
684 case 0x8B: // movl r, a
685 case 0x8F: // popl a
686 debug_only(has_disp32 = true);
687 break;
688
689 case 0x68: // pushq #32
690 if (which == end_pc_operand) {
691 return ip + 4;
692 }
693 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
694 return ip; // not produced by emit_operand
695
696 case 0x66: // movw ... (size prefix)
697 again_after_size_prefix2:
698 switch (0xFF & *ip++) {
699 case REX:
700 case REX_B:
701 case REX_X:
702 case REX_XB:
703 case REX_R:
704 case REX_RB:
705 case REX_RX:
706 case REX_RXB:
707 case REX_W:
708 case REX_WB:
709 case REX_WX:
710 case REX_WXB:
711 case REX_WR:
712 case REX_WRB:
713 case REX_WRX:
714 case REX_WRXB:
715 NOT_LP64(assert(false, "64bit prefix found"));
716 goto again_after_size_prefix2;
717 case 0x8B: // movw r, a
718 case 0x89: // movw a, r
719 debug_only(has_disp32 = true);
720 break;
721 case 0xC7: // movw a, #16
722 debug_only(has_disp32 = true);
723 tail_size = 2; // the imm16
724 break;
725 case 0x0F: // several SSE/SSE2 variants
726 ip--; // reparse the 0x0F
727 goto again_after_prefix;
728 default:
729 ShouldNotReachHere();
730 }
731 break;
732
733 case REP8(0xB8): // movl/q r, #32/#64(oop?)
734 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
735 // these asserts are somewhat nonsensical
736 #ifndef _LP64
737 assert(which == imm_operand || which == disp32_operand,
738 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
739 #else
740 assert((which == call32_operand || which == imm_operand) && is_64bit ||
741 which == narrow_oop_operand && !is_64bit,
742 "which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
743 #endif // _LP64
744 return ip;
745
746 case 0x69: // imul r, a, #32
747 case 0xC7: // movl a, #32(oop?)
748 tail_size = 4;
749 debug_only(has_disp32 = true); // has both kinds of operands!
750 break;
751
752 case 0x0F: // movx..., etc.
753 switch (0xFF & *ip++) {
754 case 0x3A: // pcmpestri
755 tail_size = 1;
756 case 0x38: // ptest, pmovzxbw
757 ip++; // skip opcode
758 debug_only(has_disp32 = true); // has both kinds of operands!
759 break;
760
761 case 0x70: // pshufd r, r/a, #8
762 debug_only(has_disp32 = true); // has both kinds of operands!
763 case 0x73: // psrldq r, #8
764 tail_size = 1;
765 break;
766
767 case 0x12: // movlps
768 case 0x28: // movaps
769 case 0x2E: // ucomiss
770 case 0x2F: // comiss
771 case 0x54: // andps
772 case 0x55: // andnps
773 case 0x56: // orps
774 case 0x57: // xorps
775 case 0x59: // mulpd
776 case 0x6E: // movd
777 case 0x7E: // movd
778 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
779 debug_only(has_disp32 = true);
780 break;
781
782 case 0xAD: // shrd r, a, %cl
783 case 0xAF: // imul r, a
784 case 0xBE: // movsbl r, a (movsxb)
785 case 0xBF: // movswl r, a (movsxw)
786 case 0xB6: // movzbl r, a (movzxb)
787 case 0xB7: // movzwl r, a (movzxw)
788 case REP16(0x40): // cmovl cc, r, a
789 case 0xB0: // cmpxchgb
790 case 0xB1: // cmpxchg
791 case 0xC1: // xaddl
792 case 0xC7: // cmpxchg8
793 case REP16(0x90): // setcc a
794 debug_only(has_disp32 = true);
795 // fall out of the switch to decode the address
796 break;
797
798 case 0xC4: // pinsrw r, a, #8
799 debug_only(has_disp32 = true);
800 case 0xC5: // pextrw r, r, #8
801 tail_size = 1; // the imm8
802 break;
803
804 case 0xAC: // shrd r, a, #8
805 debug_only(has_disp32 = true);
806 tail_size = 1; // the imm8
807 break;
808
809 case REP16(0x80): // jcc rdisp32
810 if (which == end_pc_operand) return ip + 4;
811 assert(which == call32_operand, "jcc has no disp32 or imm");
812 return ip;
813 default:
814 ShouldNotReachHere();
815 }
816 break;
817
818 case 0x81: // addl a, #32; addl r, #32
819 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
820 // on 32bit in the case of cmpl, the imm might be an oop
821 tail_size = 4;
822 debug_only(has_disp32 = true); // has both kinds of operands!
823 break;
824
825 case 0x83: // addl a, #8; addl r, #8
826 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
827 debug_only(has_disp32 = true); // has both kinds of operands!
828 tail_size = 1;
829 break;
830
831 case 0x9B:
832 switch (0xFF & *ip++) {
833 case 0xD9: // fnstcw a
834 debug_only(has_disp32 = true);
835 break;
836 default:
837 ShouldNotReachHere();
838 }
839 break;
840
841 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
842 case REP4(0x10): // adc...
843 case REP4(0x20): // and...
844 case REP4(0x30): // xor...
845 case REP4(0x08): // or...
846 case REP4(0x18): // sbb...
847 case REP4(0x28): // sub...
848 case 0xF7: // mull a
849 case 0x8D: // lea r, a
850 case 0x87: // xchg r, a
851 case REP4(0x38): // cmp...
852 case 0x85: // test r, a
853 debug_only(has_disp32 = true); // has both kinds of operands!
854 break;
855
856 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
857 case 0xC6: // movb a, #8
858 case 0x80: // cmpb a, #8
859 case 0x6B: // imul r, a, #8
860 debug_only(has_disp32 = true); // has both kinds of operands!
861 tail_size = 1; // the imm8
862 break;
863
864 case 0xC4: // VEX_3bytes
865 case 0xC5: // VEX_2bytes
866 assert((UseAVX > 0), "shouldn't have VEX prefix");
867 assert(ip == inst+1, "no prefixes allowed");
868 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
869 // but they have prefix 0x0F and processed when 0x0F processed above.
870 //
871 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
872 // instructions (these instructions are not supported in 64-bit mode).
873 // To distinguish them bits [7:6] are set in the VEX second byte since
874 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
875 // those VEX bits REX and vvvv bits are inverted.
876 //
877 // Fortunately C2 doesn't generate these instructions so we don't need
878 // to check for them in product version.
879
880 // Check second byte
881 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
882
883 int vex_opcode;
884 // First byte
885 if ((0xFF & *inst) == VEX_3bytes) {
886 vex_opcode = VEX_OPCODE_MASK & *ip;
887 ip++; // third byte
888 is_64bit = ((VEX_W & *ip) == VEX_W);
889 } else {
890 vex_opcode = VEX_OPCODE_0F;
891 }
892 ip++; // opcode
893 // To find the end of instruction (which == end_pc_operand).
894 switch (vex_opcode) {
895 case VEX_OPCODE_0F:
896 switch (0xFF & *ip) {
897 case 0x70: // pshufd r, r/a, #8
898 case 0x71: // ps[rl|ra|ll]w r, #8
899 case 0x72: // ps[rl|ra|ll]d r, #8
900 case 0x73: // ps[rl|ra|ll]q r, #8
901 case 0xC2: // cmp[ps|pd|ss|sd] r, r, r/a, #8
902 case 0xC4: // pinsrw r, r, r/a, #8
903 case 0xC5: // pextrw r/a, r, #8
904 case 0xC6: // shufp[s|d] r, r, r/a, #8
905 tail_size = 1; // the imm8
906 break;
907 }
908 break;
909 case VEX_OPCODE_0F_3A:
910 tail_size = 1;
911 break;
912 }
913 ip++; // skip opcode
914 debug_only(has_disp32 = true); // has both kinds of operands!
915 break;
916
917 case 0x62: // EVEX_4bytes
918 assert((UseAVX > 0), "shouldn't have EVEX prefix");
919 assert(ip == inst+1, "no prefixes allowed");
920 // no EVEX collisions, all instructions that have 0x62 opcodes
921 // have EVEX versions and are subopcodes of 0x66
922 ip++; // skip P0 and exmaine W in P1
923 is_64bit = ((VEX_W & *ip) == VEX_W);
924 ip++; // move to P2
925 ip++; // skip P2, move to opcode
926 // To find the end of instruction (which == end_pc_operand).
927 switch (0xFF & *ip) {
928 case 0x61: // pcmpestri r, r/a, #8
929 case 0x70: // pshufd r, r/a, #8
930 case 0x73: // psrldq r, #8
931 tail_size = 1; // the imm8
932 break;
933 default:
934 break;
935 }
936 ip++; // skip opcode
937 debug_only(has_disp32 = true); // has both kinds of operands!
938 break;
939
940 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
941 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
942 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
943 case 0xDD: // fld_d a; fst_d a; fstp_d a
944 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
945 case 0xDF: // fild_d a; fistp_d a
946 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
947 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
948 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
949 debug_only(has_disp32 = true);
950 break;
951
952 case 0xE8: // call rdisp32
953 case 0xE9: // jmp rdisp32
954 if (which == end_pc_operand) return ip + 4;
955 assert(which == call32_operand, "call has no disp32 or imm");
956 return ip;
957
958 case 0xF0: // Lock
959 assert(os::is_MP(), "only on MP");
960 goto again_after_prefix;
961
962 case 0xF3: // For SSE
963 case 0xF2: // For SSE2
964 switch (0xFF & *ip++) {
965 case REX:
966 case REX_B:
967 case REX_X:
968 case REX_XB:
969 case REX_R:
970 case REX_RB:
971 case REX_RX:
972 case REX_RXB:
973 case REX_W:
974 case REX_WB:
975 case REX_WX:
976 case REX_WXB:
977 case REX_WR:
978 case REX_WRB:
979 case REX_WRX:
980 case REX_WRXB:
981 NOT_LP64(assert(false, "found 64bit prefix"));
982 ip++;
983 default:
984 ip++;
985 }
986 debug_only(has_disp32 = true); // has both kinds of operands!
987 break;
988
989 default:
990 ShouldNotReachHere();
991
992 #undef REP8
993 #undef REP16
994 }
995
996 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
997 #ifdef _LP64
998 assert(which != imm_operand, "instruction is not a movq reg, imm64");
999 #else
1000 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
1001 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
1002 #endif // LP64
1003 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
1004
1005 // parse the output of emit_operand
1006 int op2 = 0xFF & *ip++;
1007 int base = op2 & 0x07;
1008 int op3 = -1;
1009 const int b100 = 4;
1010 const int b101 = 5;
1011 if (base == b100 && (op2 >> 6) != 3) {
1012 op3 = 0xFF & *ip++;
1013 base = op3 & 0x07; // refetch the base
1014 }
1015 // now ip points at the disp (if any)
1016
1017 switch (op2 >> 6) {
1018 case 0:
1019 // [00 reg 100][ss index base]
1020 // [00 reg 100][00 100 esp]
1021 // [00 reg base]
1022 // [00 reg 100][ss index 101][disp32]
1023 // [00 reg 101] [disp32]
1024
1025 if (base == b101) {
1026 if (which == disp32_operand)
1027 return ip; // caller wants the disp32
1028 ip += 4; // skip the disp32
1029 }
1030 break;
1031
1032 case 1:
1033 // [01 reg 100][ss index base][disp8]
1034 // [01 reg 100][00 100 esp][disp8]
1035 // [01 reg base] [disp8]
1036 ip += 1; // skip the disp8
1037 break;
1038
1039 case 2:
1040 // [10 reg 100][ss index base][disp32]
1041 // [10 reg 100][00 100 esp][disp32]
1042 // [10 reg base] [disp32]
1043 if (which == disp32_operand)
1044 return ip; // caller wants the disp32
1045 ip += 4; // skip the disp32
1046 break;
1047
1048 case 3:
1049 // [11 reg base] (not a memory addressing mode)
1050 break;
1051 }
1052
1053 if (which == end_pc_operand) {
1054 return ip + tail_size;
1055 }
1056
1057 #ifdef _LP64
1058 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1059 #else
1060 assert(which == imm_operand, "instruction has only an imm field");
1061 #endif // LP64
1062 return ip;
1063 }
1064
1065 address Assembler::locate_next_instruction(address inst) {
1066 // Secretly share code with locate_operand:
1067 return locate_operand(inst, end_pc_operand);
1068 }
1069
1070
1071 #ifdef ASSERT
1072 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1073 address inst = inst_mark();
1074 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1075 address opnd;
1076
1077 Relocation* r = rspec.reloc();
1078 if (r->type() == relocInfo::none) {
1079 return;
1080 } else if (r->is_call() || format == call32_operand) {
1081 // assert(format == imm32_operand, "cannot specify a nonzero format");
1082 opnd = locate_operand(inst, call32_operand);
1083 } else if (r->is_data()) {
1084 assert(format == imm_operand || format == disp32_operand
1085 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1086 opnd = locate_operand(inst, (WhichOperand)format);
1087 } else {
1088 assert(format == imm_operand, "cannot specify a format");
1089 return;
1090 }
1091 assert(opnd == pc(), "must put operand where relocs can find it");
1092 }
1093 #endif // ASSERT
1094
1095 void Assembler::emit_operand32(Register reg, Address adr) {
1096 assert(reg->encoding() < 8, "no extended registers");
1097 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1098 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1099 adr._rspec);
1100 }
1101
1102 void Assembler::emit_operand(Register reg, Address adr,
1103 int rip_relative_correction) {
1104 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1105 adr._rspec,
1106 rip_relative_correction);
1107 }
1108
1109 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1110 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1111 adr._rspec);
1112 }
1113
1114 // MMX operations
1115 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1116 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1117 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1118 }
1119
1120 // work around gcc (3.2.1-7a) bug
1121 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1122 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1123 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1124 }
1125
1126
1127 void Assembler::emit_farith(int b1, int b2, int i) {
1128 assert(isByte(b1) && isByte(b2), "wrong opcode");
1129 assert(0 <= i && i < 8, "illegal stack offset");
1130 emit_int8(b1);
1131 emit_int8(b2 + i);
1132 }
1133
1134
1135 // Now the Assembler instructions (identical for 32/64 bits)
1136
1137 void Assembler::adcl(Address dst, int32_t imm32) {
1138 InstructionMark im(this);
1139 prefix(dst);
1140 emit_arith_operand(0x81, rdx, dst, imm32);
1141 }
1142
1143 void Assembler::adcl(Address dst, Register src) {
1144 InstructionMark im(this);
1145 prefix(dst, src);
1146 emit_int8(0x11);
1147 emit_operand(src, dst);
1148 }
1149
1150 void Assembler::adcl(Register dst, int32_t imm32) {
1151 prefix(dst);
1152 emit_arith(0x81, 0xD0, dst, imm32);
1153 }
1154
1155 void Assembler::adcl(Register dst, Address src) {
1156 InstructionMark im(this);
1157 prefix(src, dst);
1158 emit_int8(0x13);
1159 emit_operand(dst, src);
1160 }
1161
1162 void Assembler::adcl(Register dst, Register src) {
1163 (void) prefix_and_encode(dst->encoding(), src->encoding());
1164 emit_arith(0x13, 0xC0, dst, src);
1165 }
1166
1167 void Assembler::addl(Address dst, int32_t imm32) {
1168 InstructionMark im(this);
1169 prefix(dst);
1170 emit_arith_operand(0x81, rax, dst, imm32);
1171 }
1172
1173 void Assembler::addl(Address dst, Register src) {
1174 InstructionMark im(this);
1175 prefix(dst, src);
1176 emit_int8(0x01);
1177 emit_operand(src, dst);
1178 }
1179
1180 void Assembler::addl(Register dst, int32_t imm32) {
1181 prefix(dst);
1182 emit_arith(0x81, 0xC0, dst, imm32);
1183 }
1184
1185 void Assembler::addl(Register dst, Address src) {
1186 InstructionMark im(this);
1187 prefix(src, dst);
1188 emit_int8(0x03);
1189 emit_operand(dst, src);
1190 }
1191
1192 void Assembler::addl(Register dst, Register src) {
1193 (void) prefix_and_encode(dst->encoding(), src->encoding());
1194 emit_arith(0x03, 0xC0, dst, src);
1195 }
1196
1197 void Assembler::addr_nop_4() {
1198 assert(UseAddressNop, "no CPU support");
1199 // 4 bytes: NOP DWORD PTR [EAX+0]
1200 emit_int8(0x0F);
1201 emit_int8(0x1F);
1202 emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1203 emit_int8(0); // 8-bits offset (1 byte)
1204 }
1205
1206 void Assembler::addr_nop_5() {
1207 assert(UseAddressNop, "no CPU support");
1208 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1209 emit_int8(0x0F);
1210 emit_int8(0x1F);
1211 emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1212 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1213 emit_int8(0); // 8-bits offset (1 byte)
1214 }
1215
1216 void Assembler::addr_nop_7() {
1217 assert(UseAddressNop, "no CPU support");
1218 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1219 emit_int8(0x0F);
1220 emit_int8(0x1F);
1221 emit_int8((unsigned char)0x80);
1222 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1223 emit_int32(0); // 32-bits offset (4 bytes)
1224 }
1225
1226 void Assembler::addr_nop_8() {
1227 assert(UseAddressNop, "no CPU support");
1228 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1229 emit_int8(0x0F);
1230 emit_int8(0x1F);
1231 emit_int8((unsigned char)0x84);
1232 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1233 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1234 emit_int32(0); // 32-bits offset (4 bytes)
1235 }
1236
1237 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1238 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1239 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1240 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1241 emit_int8(0x58);
1242 emit_int8((unsigned char)(0xC0 | encode));
1243 }
1244
1245 void Assembler::addsd(XMMRegister dst, Address src) {
1246 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1247 InstructionMark im(this);
1248 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1249 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1250 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1251 emit_int8(0x58);
1252 emit_operand(dst, src);
1253 }
1254
1255 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1256 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1257 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1258 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1259 emit_int8(0x58);
1260 emit_int8((unsigned char)(0xC0 | encode));
1261 }
1262
1263 void Assembler::addss(XMMRegister dst, Address src) {
1264 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1265 InstructionMark im(this);
1266 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1267 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1268 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1269 emit_int8(0x58);
1270 emit_operand(dst, src);
1271 }
1272
1273 void Assembler::aesdec(XMMRegister dst, Address src) {
1274 assert(VM_Version::supports_aes(), "");
1275 InstructionMark im(this);
1276 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1277 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1278 emit_int8((unsigned char)0xDE);
1279 emit_operand(dst, src);
1280 }
1281
1282 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1283 assert(VM_Version::supports_aes(), "");
1284 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1285 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1286 emit_int8((unsigned char)0xDE);
1287 emit_int8(0xC0 | encode);
1288 }
1289
1290 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1291 assert(VM_Version::supports_aes(), "");
1292 InstructionMark im(this);
1293 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1294 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1295 emit_int8((unsigned char)0xDF);
1296 emit_operand(dst, src);
1297 }
1298
1299 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1300 assert(VM_Version::supports_aes(), "");
1301 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1302 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1303 emit_int8((unsigned char)0xDF);
1304 emit_int8((unsigned char)(0xC0 | encode));
1305 }
1306
1307 void Assembler::aesenc(XMMRegister dst, Address src) {
1308 assert(VM_Version::supports_aes(), "");
1309 InstructionMark im(this);
1310 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1311 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1312 emit_int8((unsigned char)0xDC);
1313 emit_operand(dst, src);
1314 }
1315
1316 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1317 assert(VM_Version::supports_aes(), "");
1318 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1319 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1320 emit_int8((unsigned char)0xDC);
1321 emit_int8(0xC0 | encode);
1322 }
1323
1324 void Assembler::aesenclast(XMMRegister dst, Address src) {
1325 assert(VM_Version::supports_aes(), "");
1326 InstructionMark im(this);
1327 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1328 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1329 emit_int8((unsigned char)0xDD);
1330 emit_operand(dst, src);
1331 }
1332
1333 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1334 assert(VM_Version::supports_aes(), "");
1335 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1336 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
1337 emit_int8((unsigned char)0xDD);
1338 emit_int8((unsigned char)(0xC0 | encode));
1339 }
1340
1341 void Assembler::andl(Address dst, int32_t imm32) {
1342 InstructionMark im(this);
1343 prefix(dst);
1344 emit_int8((unsigned char)0x81);
1345 emit_operand(rsp, dst, 4);
1346 emit_int32(imm32);
1347 }
1348
1349 void Assembler::andl(Register dst, int32_t imm32) {
1350 prefix(dst);
1351 emit_arith(0x81, 0xE0, dst, imm32);
1352 }
1353
1354 void Assembler::andl(Register dst, Address src) {
1355 InstructionMark im(this);
1356 prefix(src, dst);
1357 emit_int8(0x23);
1358 emit_operand(dst, src);
1359 }
1360
1361 void Assembler::andl(Register dst, Register src) {
1362 (void) prefix_and_encode(dst->encoding(), src->encoding());
1363 emit_arith(0x23, 0xC0, dst, src);
1364 }
1365
1366 void Assembler::andnl(Register dst, Register src1, Register src2) {
1367 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1368 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1369 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1370 emit_int8((unsigned char)0xF2);
1371 emit_int8((unsigned char)(0xC0 | encode));
1372 }
1373
1374 void Assembler::andnl(Register dst, Register src1, Address src2) {
1375 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1376 InstructionMark im(this);
1377 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1378 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1379 emit_int8((unsigned char)0xF2);
1380 emit_operand(dst, src2);
1381 }
1382
1383 void Assembler::bsfl(Register dst, Register src) {
1384 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1385 emit_int8(0x0F);
1386 emit_int8((unsigned char)0xBC);
1387 emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389
1390 void Assembler::bsrl(Register dst, Register src) {
1391 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1392 emit_int8(0x0F);
1393 emit_int8((unsigned char)0xBD);
1394 emit_int8((unsigned char)(0xC0 | encode));
1395 }
1396
1397 void Assembler::bswapl(Register reg) { // bswap
1398 int encode = prefix_and_encode(reg->encoding());
1399 emit_int8(0x0F);
1400 emit_int8((unsigned char)(0xC8 | encode));
1401 }
1402
1403 void Assembler::blsil(Register dst, Register src) {
1404 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1405 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1406 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1407 emit_int8((unsigned char)0xF3);
1408 emit_int8((unsigned char)(0xC0 | encode));
1409 }
1410
1411 void Assembler::blsil(Register dst, Address src) {
1412 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1413 InstructionMark im(this);
1414 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1415 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1416 emit_int8((unsigned char)0xF3);
1417 emit_operand(rbx, src);
1418 }
1419
1420 void Assembler::blsmskl(Register dst, Register src) {
1421 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1422 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1423 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1424 emit_int8((unsigned char)0xF3);
1425 emit_int8((unsigned char)(0xC0 | encode));
1426 }
1427
1428 void Assembler::blsmskl(Register dst, Address src) {
1429 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1430 InstructionMark im(this);
1431 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1432 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1433 emit_int8((unsigned char)0xF3);
1434 emit_operand(rdx, src);
1435 }
1436
1437 void Assembler::blsrl(Register dst, Register src) {
1438 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1439 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1440 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1441 emit_int8((unsigned char)0xF3);
1442 emit_int8((unsigned char)(0xC0 | encode));
1443 }
1444
1445 void Assembler::blsrl(Register dst, Address src) {
1446 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1447 InstructionMark im(this);
1448 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
1449 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
1450 emit_int8((unsigned char)0xF3);
1451 emit_operand(rcx, src);
1452 }
1453
1454 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1455 // suspect disp32 is always good
1456 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1457
1458 if (L.is_bound()) {
1459 const int long_size = 5;
1460 int offs = (int)( target(L) - pc() );
1461 assert(offs <= 0, "assembler error");
1462 InstructionMark im(this);
1463 // 1110 1000 #32-bit disp
1464 emit_int8((unsigned char)0xE8);
1465 emit_data(offs - long_size, rtype, operand);
1466 } else {
1467 InstructionMark im(this);
1468 // 1110 1000 #32-bit disp
1469 L.add_patch_at(code(), locator());
1470
1471 emit_int8((unsigned char)0xE8);
1472 emit_data(int(0), rtype, operand);
1473 }
1474 }
1475
1476 void Assembler::call(Register dst) {
1477 int encode = prefix_and_encode(dst->encoding());
1478 emit_int8((unsigned char)0xFF);
1479 emit_int8((unsigned char)(0xD0 | encode));
1480 }
1481
1482
1483 void Assembler::call(Address adr) {
1484 InstructionMark im(this);
1485 prefix(adr);
1486 emit_int8((unsigned char)0xFF);
1487 emit_operand(rdx, adr);
1488 }
1489
1490 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1491 assert(entry != NULL, "call most probably wrong");
1492 InstructionMark im(this);
1493 emit_int8((unsigned char)0xE8);
1494 intptr_t disp = entry - (pc() + sizeof(int32_t));
1495 assert(is_simm32(disp), "must be 32bit offset (call2)");
1496 // Technically, should use call32_operand, but this format is
1497 // implied by the fact that we're emitting a call instruction.
1498
1499 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1500 emit_data((int) disp, rspec, operand);
1501 }
1502
1503 void Assembler::cdql() {
1504 emit_int8((unsigned char)0x99);
1505 }
1506
1507 void Assembler::cld() {
1508 emit_int8((unsigned char)0xFC);
1509 }
1510
1511 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1512 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1513 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1514 emit_int8(0x0F);
1515 emit_int8(0x40 | cc);
1516 emit_int8((unsigned char)(0xC0 | encode));
1517 }
1518
1519
1520 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1521 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1522 prefix(src, dst);
1523 emit_int8(0x0F);
1524 emit_int8(0x40 | cc);
1525 emit_operand(dst, src);
1526 }
1527
1528 void Assembler::cmpb(Address dst, int imm8) {
1529 InstructionMark im(this);
1530 prefix(dst);
1531 emit_int8((unsigned char)0x80);
1532 emit_operand(rdi, dst, 1);
1533 emit_int8(imm8);
1534 }
1535
1536 void Assembler::cmpl(Address dst, int32_t imm32) {
1537 InstructionMark im(this);
1538 prefix(dst);
1539 emit_int8((unsigned char)0x81);
1540 emit_operand(rdi, dst, 4);
1541 emit_int32(imm32);
1542 }
1543
1544 void Assembler::cmpl(Register dst, int32_t imm32) {
1545 prefix(dst);
1546 emit_arith(0x81, 0xF8, dst, imm32);
1547 }
1548
1549 void Assembler::cmpl(Register dst, Register src) {
1550 (void) prefix_and_encode(dst->encoding(), src->encoding());
1551 emit_arith(0x3B, 0xC0, dst, src);
1552 }
1553
1554 void Assembler::cmpl(Register dst, Address src) {
1555 InstructionMark im(this);
1556 prefix(src, dst);
1557 emit_int8((unsigned char)0x3B);
1558 emit_operand(dst, src);
1559 }
1560
1561 void Assembler::cmpw(Address dst, int imm16) {
1562 InstructionMark im(this);
1563 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1564 emit_int8(0x66);
1565 emit_int8((unsigned char)0x81);
1566 emit_operand(rdi, dst, 2);
1567 emit_int16(imm16);
1568 }
1569
1570 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1571 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1572 // The ZF is set if the compared values were equal, and cleared otherwise.
1573 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1574 InstructionMark im(this);
1575 prefix(adr, reg);
1576 emit_int8(0x0F);
1577 emit_int8((unsigned char)0xB1);
1578 emit_operand(reg, adr);
1579 }
1580
1581 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1582 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1583 // The ZF is set if the compared values were equal, and cleared otherwise.
1584 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1585 InstructionMark im(this);
1586 prefix(adr, reg, true);
1587 emit_int8(0x0F);
1588 emit_int8((unsigned char)0xB0);
1589 emit_operand(reg, adr);
1590 }
1591
1592 void Assembler::comisd(XMMRegister dst, Address src) {
1593 // NOTE: dbx seems to decode this as comiss even though the
1594 // 0x66 is there. Strangly ucomisd comes out correct
1595 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1596 InstructionMark im(this);
1597 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);;
1598 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1599 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1600 emit_int8(0x2F);
1601 emit_operand(dst, src);
1602 }
1603
1604 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1605 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1606 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1607 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
1608 emit_int8(0x2F);
1609 emit_int8((unsigned char)(0xC0 | encode));
1610 }
1611
1612 void Assembler::comiss(XMMRegister dst, Address src) {
1613 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1614 InstructionMark im(this);
1615 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1616 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1617 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1618 emit_int8(0x2F);
1619 emit_operand(dst, src);
1620 }
1621
1622 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1623 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1624 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1625 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1626 emit_int8(0x2F);
1627 emit_int8((unsigned char)(0xC0 | encode));
1628 }
1629
1630 void Assembler::cpuid() {
1631 emit_int8(0x0F);
1632 emit_int8((unsigned char)0xA2);
1633 }
1634
1635 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1636 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1637 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1638 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1639 //
1640 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1641 //
1642 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1643 //
1644 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1645 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1646 assert(VM_Version::supports_sse4_2(), "");
1647 int8_t w = 0x01;
1648 Prefix p = Prefix_EMPTY;
1649
1650 emit_int8((int8_t)0xF2);
1651 switch (sizeInBytes) {
1652 case 1:
1653 w = 0;
1654 break;
1655 case 2:
1656 case 4:
1657 break;
1658 LP64_ONLY(case 8:)
1659 // This instruction is not valid in 32 bits
1660 // Note:
1661 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1662 //
1663 // Page B - 72 Vol. 2C says
1664 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1665 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1666 // F0!!!
1667 // while 3 - 208 Vol. 2A
1668 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1669 //
1670 // the 0 on a last bit is reserved for a different flavor of this instruction :
1671 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1672 p = REX_W;
1673 break;
1674 default:
1675 assert(0, "Unsupported value for a sizeInBytes argument");
1676 break;
1677 }
1678 LP64_ONLY(prefix(crc, v, p);)
1679 emit_int8((int8_t)0x0F);
1680 emit_int8(0x38);
1681 emit_int8((int8_t)(0xF0 | w));
1682 emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1683 }
1684
1685 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1686 assert(VM_Version::supports_sse4_2(), "");
1687 InstructionMark im(this);
1688 int8_t w = 0x01;
1689 Prefix p = Prefix_EMPTY;
1690
1691 emit_int8((int8_t)0xF2);
1692 switch (sizeInBytes) {
1693 case 1:
1694 w = 0;
1695 break;
1696 case 2:
1697 case 4:
1698 break;
1699 LP64_ONLY(case 8:)
1700 // This instruction is not valid in 32 bits
1701 p = REX_W;
1702 break;
1703 default:
1704 assert(0, "Unsupported value for a sizeInBytes argument");
1705 break;
1706 }
1707 LP64_ONLY(prefix(crc, adr, p);)
1708 emit_int8((int8_t)0x0F);
1709 emit_int8(0x38);
1710 emit_int8((int8_t)(0xF0 | w));
1711 emit_operand(crc, adr);
1712 }
1713
1714 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1715 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1716 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1717 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1718 emit_int8((unsigned char)0xE6);
1719 emit_int8((unsigned char)(0xC0 | encode));
1720 }
1721
1722 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1723 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1724 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
1725 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
1726 emit_int8(0x5B);
1727 emit_int8((unsigned char)(0xC0 | encode));
1728 }
1729
1730 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1731 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1732 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1733 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1734 emit_int8(0x5A);
1735 emit_int8((unsigned char)(0xC0 | encode));
1736 }
1737
1738 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1739 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1740 InstructionMark im(this);
1741 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1742 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1743 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1744 emit_int8(0x5A);
1745 emit_operand(dst, src);
1746 }
1747
1748 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1749 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1750 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1751 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1752 emit_int8(0x2A);
1753 emit_int8((unsigned char)(0xC0 | encode));
1754 }
1755
1756 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1757 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1758 InstructionMark im(this);
1759 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1760 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1761 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1762 emit_int8(0x2A);
1763 emit_operand(dst, src);
1764 }
1765
1766 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1767 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1768 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1769 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1770 emit_int8(0x2A);
1771 emit_int8((unsigned char)(0xC0 | encode));
1772 }
1773
1774 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1775 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1776 InstructionMark im(this);
1777 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1778 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1779 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1780 emit_int8(0x2A);
1781 emit_operand(dst, src);
1782 }
1783
1784 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1785 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1786 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1787 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1788 emit_int8(0x2A);
1789 emit_int8((unsigned char)(0xC0 | encode));
1790 }
1791
1792 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1793 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1794 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1795 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1796 emit_int8(0x5A);
1797 emit_int8((unsigned char)(0xC0 | encode));
1798 }
1799
1800 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1801 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1802 InstructionMark im(this);
1803 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1804 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1805 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1806 emit_int8(0x5A);
1807 emit_operand(dst, src);
1808 }
1809
1810
1811 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1812 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1813 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1814 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1815 emit_int8(0x2C);
1816 emit_int8((unsigned char)(0xC0 | encode));
1817 }
1818
1819 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1820 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1821 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
1822 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1823 emit_int8(0x2C);
1824 emit_int8((unsigned char)(0xC0 | encode));
1825 }
1826
1827 void Assembler::decl(Address dst) {
1828 // Don't use it directly. Use MacroAssembler::decrement() instead.
1829 InstructionMark im(this);
1830 prefix(dst);
1831 emit_int8((unsigned char)0xFF);
1832 emit_operand(rcx, dst);
1833 }
1834
1835 void Assembler::divsd(XMMRegister dst, Address src) {
1836 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1837 InstructionMark im(this);
1838 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1839 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
1840 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1841 emit_int8(0x5E);
1842 emit_operand(dst, src);
1843 }
1844
1845 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1846 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1847 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1848 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
1849 emit_int8(0x5E);
1850 emit_int8((unsigned char)(0xC0 | encode));
1851 }
1852
1853 void Assembler::divss(XMMRegister dst, Address src) {
1854 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1855 InstructionMark im(this);
1856 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1857 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
1858 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1859 emit_int8(0x5E);
1860 emit_operand(dst, src);
1861 }
1862
1863 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1864 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1865 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
1866 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
1867 emit_int8(0x5E);
1868 emit_int8((unsigned char)(0xC0 | encode));
1869 }
1870
1871 void Assembler::emms() {
1872 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1873 emit_int8(0x0F);
1874 emit_int8(0x77);
1875 }
1876
1877 void Assembler::hlt() {
1878 emit_int8((unsigned char)0xF4);
1879 }
1880
1881 void Assembler::idivl(Register src) {
1882 int encode = prefix_and_encode(src->encoding());
1883 emit_int8((unsigned char)0xF7);
1884 emit_int8((unsigned char)(0xF8 | encode));
1885 }
1886
1887 void Assembler::divl(Register src) { // Unsigned
1888 int encode = prefix_and_encode(src->encoding());
1889 emit_int8((unsigned char)0xF7);
1890 emit_int8((unsigned char)(0xF0 | encode));
1891 }
1892
1893 void Assembler::imull(Register dst, Register src) {
1894 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1895 emit_int8(0x0F);
1896 emit_int8((unsigned char)0xAF);
1897 emit_int8((unsigned char)(0xC0 | encode));
1898 }
1899
1900
1901 void Assembler::imull(Register dst, Register src, int value) {
1902 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1903 if (is8bit(value)) {
1904 emit_int8(0x6B);
1905 emit_int8((unsigned char)(0xC0 | encode));
1906 emit_int8(value & 0xFF);
1907 } else {
1908 emit_int8(0x69);
1909 emit_int8((unsigned char)(0xC0 | encode));
1910 emit_int32(value);
1911 }
1912 }
1913
1914 void Assembler::imull(Register dst, Address src) {
1915 InstructionMark im(this);
1916 prefix(src, dst);
1917 emit_int8(0x0F);
1918 emit_int8((unsigned char) 0xAF);
1919 emit_operand(dst, src);
1920 }
1921
1922
1923 void Assembler::incl(Address dst) {
1924 // Don't use it directly. Use MacroAssembler::increment() instead.
1925 InstructionMark im(this);
1926 prefix(dst);
1927 emit_int8((unsigned char)0xFF);
1928 emit_operand(rax, dst);
1929 }
1930
1931 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1932 InstructionMark im(this);
1933 assert((0 <= cc) && (cc < 16), "illegal cc");
1934 if (L.is_bound()) {
1935 address dst = target(L);
1936 assert(dst != NULL, "jcc most probably wrong");
1937
1938 const int short_size = 2;
1939 const int long_size = 6;
1940 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1941 if (maybe_short && is8bit(offs - short_size)) {
1942 // 0111 tttn #8-bit disp
1943 emit_int8(0x70 | cc);
1944 emit_int8((offs - short_size) & 0xFF);
1945 } else {
1946 // 0000 1111 1000 tttn #32-bit disp
1947 assert(is_simm32(offs - long_size),
1948 "must be 32bit offset (call4)");
1949 emit_int8(0x0F);
1950 emit_int8((unsigned char)(0x80 | cc));
1951 emit_int32(offs - long_size);
1952 }
1953 } else {
1954 // Note: could eliminate cond. jumps to this jump if condition
1955 // is the same however, seems to be rather unlikely case.
1956 // Note: use jccb() if label to be bound is very close to get
1957 // an 8-bit displacement
1958 L.add_patch_at(code(), locator());
1959 emit_int8(0x0F);
1960 emit_int8((unsigned char)(0x80 | cc));
1961 emit_int32(0);
1962 }
1963 }
1964
1965 void Assembler::jccb(Condition cc, Label& L) {
1966 if (L.is_bound()) {
1967 const int short_size = 2;
1968 address entry = target(L);
1969 #ifdef ASSERT
1970 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1971 intptr_t delta = short_branch_delta();
1972 if (delta != 0) {
1973 dist += (dist < 0 ? (-delta) :delta);
1974 }
1975 assert(is8bit(dist), "Dispacement too large for a short jmp");
1976 #endif
1977 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1978 // 0111 tttn #8-bit disp
1979 emit_int8(0x70 | cc);
1980 emit_int8((offs - short_size) & 0xFF);
1981 } else {
1982 InstructionMark im(this);
1983 L.add_patch_at(code(), locator());
1984 emit_int8(0x70 | cc);
1985 emit_int8(0);
1986 }
1987 }
1988
1989 void Assembler::jmp(Address adr) {
1990 InstructionMark im(this);
1991 prefix(adr);
1992 emit_int8((unsigned char)0xFF);
1993 emit_operand(rsp, adr);
1994 }
1995
1996 void Assembler::jmp(Label& L, bool maybe_short) {
1997 if (L.is_bound()) {
1998 address entry = target(L);
1999 assert(entry != NULL, "jmp most probably wrong");
2000 InstructionMark im(this);
2001 const int short_size = 2;
2002 const int long_size = 5;
2003 intptr_t offs = entry - pc();
2004 if (maybe_short && is8bit(offs - short_size)) {
2005 emit_int8((unsigned char)0xEB);
2006 emit_int8((offs - short_size) & 0xFF);
2007 } else {
2008 emit_int8((unsigned char)0xE9);
2009 emit_int32(offs - long_size);
2010 }
2011 } else {
2012 // By default, forward jumps are always 32-bit displacements, since
2013 // we can't yet know where the label will be bound. If you're sure that
2014 // the forward jump will not run beyond 256 bytes, use jmpb to
2015 // force an 8-bit displacement.
2016 InstructionMark im(this);
2017 L.add_patch_at(code(), locator());
2018 emit_int8((unsigned char)0xE9);
2019 emit_int32(0);
2020 }
2021 }
2022
2023 void Assembler::jmp(Register entry) {
2024 int encode = prefix_and_encode(entry->encoding());
2025 emit_int8((unsigned char)0xFF);
2026 emit_int8((unsigned char)(0xE0 | encode));
2027 }
2028
2029 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
2030 InstructionMark im(this);
2031 emit_int8((unsigned char)0xE9);
2032 assert(dest != NULL, "must have a target");
2033 intptr_t disp = dest - (pc() + sizeof(int32_t));
2034 assert(is_simm32(disp), "must be 32bit offset (jmp)");
2035 emit_data(disp, rspec.reloc(), call32_operand);
2036 }
2037
2038 void Assembler::jmpb(Label& L) {
2039 if (L.is_bound()) {
2040 const int short_size = 2;
2041 address entry = target(L);
2042 assert(entry != NULL, "jmp most probably wrong");
2043 #ifdef ASSERT
2044 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2045 intptr_t delta = short_branch_delta();
2046 if (delta != 0) {
2047 dist += (dist < 0 ? (-delta) :delta);
2048 }
2049 assert(is8bit(dist), "Dispacement too large for a short jmp");
2050 #endif
2051 intptr_t offs = entry - pc();
2052 emit_int8((unsigned char)0xEB);
2053 emit_int8((offs - short_size) & 0xFF);
2054 } else {
2055 InstructionMark im(this);
2056 L.add_patch_at(code(), locator());
2057 emit_int8((unsigned char)0xEB);
2058 emit_int8(0);
2059 }
2060 }
2061
2062 void Assembler::ldmxcsr( Address src) {
2063 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2064 InstructionMark im(this);
2065 prefix(src);
2066 emit_int8(0x0F);
2067 emit_int8((unsigned char)0xAE);
2068 emit_operand(as_Register(2), src);
2069 }
2070
2071 void Assembler::leal(Register dst, Address src) {
2072 InstructionMark im(this);
2073 #ifdef _LP64
2074 emit_int8(0x67); // addr32
2075 prefix(src, dst);
2076 #endif // LP64
2077 emit_int8((unsigned char)0x8D);
2078 emit_operand(dst, src);
2079 }
2080
2081 void Assembler::lfence() {
2082 emit_int8(0x0F);
2083 emit_int8((unsigned char)0xAE);
2084 emit_int8((unsigned char)0xE8);
2085 }
2086
2087 void Assembler::lock() {
2088 emit_int8((unsigned char)0xF0);
2089 }
2090
2091 void Assembler::lzcntl(Register dst, Register src) {
2092 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2093 emit_int8((unsigned char)0xF3);
2094 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2095 emit_int8(0x0F);
2096 emit_int8((unsigned char)0xBD);
2097 emit_int8((unsigned char)(0xC0 | encode));
2098 }
2099
2100 // Emit mfence instruction
2101 void Assembler::mfence() {
2102 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2103 emit_int8(0x0F);
2104 emit_int8((unsigned char)0xAE);
2105 emit_int8((unsigned char)0xF0);
2106 }
2107
2108 void Assembler::mov(Register dst, Register src) {
2109 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2110 }
2111
2112 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2113 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2114 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2115 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2116 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2117 emit_int8(0x28);
2118 emit_int8((unsigned char)(0xC0 | encode));
2119 }
2120
2121 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2122 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2123 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2124 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2125 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2126 emit_int8(0x28);
2127 emit_int8((unsigned char)(0xC0 | encode));
2128 }
2129
2130 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2131 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2132 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2133 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2134 emit_int8(0x16);
2135 emit_int8((unsigned char)(0xC0 | encode));
2136 }
2137
2138 void Assembler::movb(Register dst, Address src) {
2139 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2140 InstructionMark im(this);
2141 prefix(src, dst, true);
2142 emit_int8((unsigned char)0x8A);
2143 emit_operand(dst, src);
2144 }
2145
2146 void Assembler::movddup(XMMRegister dst, XMMRegister src) {
2147 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
2148 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
2149 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2150 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2151 emit_int8(0x12);
2152 emit_int8(0xC0 | encode);
2153 }
2154
2155 void Assembler::kmovql(KRegister dst, KRegister src) {
2156 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2157 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2158 int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2159 emit_int8((unsigned char)0x90);
2160 emit_int8((unsigned char)(0xC0 | encode));
2161 }
2162
2163 void Assembler::kmovql(KRegister dst, Address src) {
2164 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2165 InstructionMark im(this);
2166 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2167 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2168 emit_int8((unsigned char)0x90);
2169 emit_operand((Register)dst, src);
2170 }
2171
2172 void Assembler::kmovql(Address dst, KRegister src) {
2173 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2174 InstructionMark im(this);
2175 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2176 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2177 emit_int8((unsigned char)0x90);
2178 emit_operand((Register)src, dst);
2179 }
2180
2181 void Assembler::kmovql(KRegister dst, Register src) {
2182 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2183 VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2184 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_bw, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2185 int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
2186 emit_int8((unsigned char)0x92);
2187 emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189
2190 void Assembler::kmovdl(KRegister dst, Register src) {
2191 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2192 VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2193 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2194 int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, VEX_OPCODE_0F, &attributes);
2195 emit_int8((unsigned char)0x92);
2196 emit_int8((unsigned char)(0xC0 | encode));
2197 }
2198
2199 void Assembler::kmovwl(KRegister dst, Register src) {
2200 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2201 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
2202 int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
2203 emit_int8((unsigned char)0x92);
2204 emit_int8((unsigned char)(0xC0 | encode));
2205 }
2206
2207 void Assembler::movb(Address dst, int imm8) {
2208 InstructionMark im(this);
2209 prefix(dst);
2210 emit_int8((unsigned char)0xC6);
2211 emit_operand(rax, dst, 1);
2212 emit_int8(imm8);
2213 }
2214
2215
2216 void Assembler::movb(Address dst, Register src) {
2217 assert(src->has_byte_register(), "must have byte register");
2218 InstructionMark im(this);
2219 prefix(dst, src, true);
2220 emit_int8((unsigned char)0x88);
2221 emit_operand(src, dst);
2222 }
2223
2224 void Assembler::movdl(XMMRegister dst, Register src) {
2225 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2226 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2227 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2228 emit_int8(0x6E);
2229 emit_int8((unsigned char)(0xC0 | encode));
2230 }
2231
2232 void Assembler::movdl(Register dst, XMMRegister src) {
2233 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2234 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2235 // swap src/dst to get correct prefix
2236 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2237 emit_int8(0x7E);
2238 emit_int8((unsigned char)(0xC0 | encode));
2239 }
2240
2241 void Assembler::movdl(XMMRegister dst, Address src) {
2242 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2243 InstructionMark im(this);
2244 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2245 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2246 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2247 emit_int8(0x6E);
2248 emit_operand(dst, src);
2249 }
2250
2251 void Assembler::movdl(Address dst, XMMRegister src) {
2252 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2253 InstructionMark im(this);
2254 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2255 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2256 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2257 emit_int8(0x7E);
2258 emit_operand(src, dst);
2259 }
2260
2261 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2262 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2263 int vector_len = VM_Version::supports_avx512novl() ? AVX_512bit : AVX_128bit;
2264 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2265 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2266 emit_int8(0x6F);
2267 emit_int8((unsigned char)(0xC0 | encode));
2268 }
2269
2270 void Assembler::movdqa(XMMRegister dst, Address src) {
2271 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2272 InstructionMark im(this);
2273 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2274 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2275 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2276 emit_int8(0x6F);
2277 emit_operand(dst, src);
2278 }
2279
2280 void Assembler::movdqu(XMMRegister dst, Address src) {
2281 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2282 InstructionMark im(this);
2283 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2284 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2285 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2286 emit_int8(0x6F);
2287 emit_operand(dst, src);
2288 }
2289
2290 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2291 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2292 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2293 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2294 emit_int8(0x6F);
2295 emit_int8((unsigned char)(0xC0 | encode));
2296 }
2297
2298 void Assembler::movdqu(Address dst, XMMRegister src) {
2299 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2300 InstructionMark im(this);
2301 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2302 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2303 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2304 emit_int8(0x7F);
2305 emit_operand(src, dst);
2306 }
2307
2308 // Move Unaligned 256bit Vector
2309 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2310 assert(UseAVX > 0, "");
2311 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2312 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2313 emit_int8(0x6F);
2314 emit_int8((unsigned char)(0xC0 | encode));
2315 }
2316
2317 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2318 assert(UseAVX > 0, "");
2319 InstructionMark im(this);
2320 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2321 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2322 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2323 emit_int8(0x6F);
2324 emit_operand(dst, src);
2325 }
2326
2327 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2328 assert(UseAVX > 0, "");
2329 InstructionMark im(this);
2330 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2331 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2332 // swap src<->dst for encoding
2333 assert(src != xnoreg, "sanity");
2334 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2335 emit_int8(0x7F);
2336 emit_operand(src, dst);
2337 }
2338
2339 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2340 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2341 assert(VM_Version::supports_evex(), "");
2342 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2343 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2344 emit_int8(0x6F);
2345 emit_int8((unsigned char)(0xC0 | encode));
2346 }
2347
2348 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2349 assert(VM_Version::supports_evex(), "");
2350 InstructionMark im(this);
2351 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2352 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2353 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2354 emit_int8(0x6F);
2355 emit_operand(dst, src);
2356 }
2357
2358 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2359 assert(VM_Version::supports_evex(), "");
2360 assert(src != xnoreg, "sanity");
2361 InstructionMark im(this);
2362 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2363 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2364 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2365 emit_int8(0x7F);
2366 emit_operand(src, dst);
2367 }
2368
2369 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2370 assert(VM_Version::supports_evex(), "");
2371 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2372 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2373 emit_int8(0x6F);
2374 emit_int8((unsigned char)(0xC0 | encode));
2375 }
2376
2377 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2378 assert(VM_Version::supports_evex(), "");
2379 InstructionMark im(this);
2380 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true);
2381 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2382 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2383 emit_int8(0x6F);
2384 emit_operand(dst, src);
2385 }
2386
2387 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2388 assert(VM_Version::supports_evex(), "");
2389 assert(src != xnoreg, "sanity");
2390 InstructionMark im(this);
2391 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
2392 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
2393 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2394 emit_int8(0x7F);
2395 emit_operand(src, dst);
2396 }
2397
2398 // Uses zero extension on 64bit
2399
2400 void Assembler::movl(Register dst, int32_t imm32) {
2401 int encode = prefix_and_encode(dst->encoding());
2402 emit_int8((unsigned char)(0xB8 | encode));
2403 emit_int32(imm32);
2404 }
2405
2406 void Assembler::movl(Register dst, Register src) {
2407 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2408 emit_int8((unsigned char)0x8B);
2409 emit_int8((unsigned char)(0xC0 | encode));
2410 }
2411
2412 void Assembler::movl(Register dst, Address src) {
2413 InstructionMark im(this);
2414 prefix(src, dst);
2415 emit_int8((unsigned char)0x8B);
2416 emit_operand(dst, src);
2417 }
2418
2419 void Assembler::movl(Address dst, int32_t imm32) {
2420 InstructionMark im(this);
2421 prefix(dst);
2422 emit_int8((unsigned char)0xC7);
2423 emit_operand(rax, dst, 4);
2424 emit_int32(imm32);
2425 }
2426
2427 void Assembler::movl(Address dst, Register src) {
2428 InstructionMark im(this);
2429 prefix(dst, src);
2430 emit_int8((unsigned char)0x89);
2431 emit_operand(src, dst);
2432 }
2433
2434 // New cpus require to use movsd and movss to avoid partial register stall
2435 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2436 // The selection is done in MacroAssembler::movdbl() and movflt().
2437 void Assembler::movlpd(XMMRegister dst, Address src) {
2438 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2439 InstructionMark im(this);
2440 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2441 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2442 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2443 emit_int8(0x12);
2444 emit_operand(dst, src);
2445 }
2446
2447 void Assembler::movq( MMXRegister dst, Address src ) {
2448 assert( VM_Version::supports_mmx(), "" );
2449 emit_int8(0x0F);
2450 emit_int8(0x6F);
2451 emit_operand(dst, src);
2452 }
2453
2454 void Assembler::movq( Address dst, MMXRegister src ) {
2455 assert( VM_Version::supports_mmx(), "" );
2456 emit_int8(0x0F);
2457 emit_int8(0x7F);
2458 // workaround gcc (3.2.1-7a) bug
2459 // In that version of gcc with only an emit_operand(MMX, Address)
2460 // gcc will tail jump and try and reverse the parameters completely
2461 // obliterating dst in the process. By having a version available
2462 // that doesn't need to swap the args at the tail jump the bug is
2463 // avoided.
2464 emit_operand(dst, src);
2465 }
2466
2467 void Assembler::movq(XMMRegister dst, Address src) {
2468 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2469 InstructionMark im(this);
2470 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2471 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2472 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2473 emit_int8(0x7E);
2474 emit_operand(dst, src);
2475 }
2476
2477 void Assembler::movq(Address dst, XMMRegister src) {
2478 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2479 InstructionMark im(this);
2480 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2481 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2482 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2483 emit_int8((unsigned char)0xD6);
2484 emit_operand(src, dst);
2485 }
2486
2487 void Assembler::movsbl(Register dst, Address src) { // movsxb
2488 InstructionMark im(this);
2489 prefix(src, dst);
2490 emit_int8(0x0F);
2491 emit_int8((unsigned char)0xBE);
2492 emit_operand(dst, src);
2493 }
2494
2495 void Assembler::movsbl(Register dst, Register src) { // movsxb
2496 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2497 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2498 emit_int8(0x0F);
2499 emit_int8((unsigned char)0xBE);
2500 emit_int8((unsigned char)(0xC0 | encode));
2501 }
2502
2503 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2504 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2505 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2506 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2507 emit_int8(0x10);
2508 emit_int8((unsigned char)(0xC0 | encode));
2509 }
2510
2511 void Assembler::movsd(XMMRegister dst, Address src) {
2512 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2513 InstructionMark im(this);
2514 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2515 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2516 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2517 emit_int8(0x10);
2518 emit_operand(dst, src);
2519 }
2520
2521 void Assembler::movsd(Address dst, XMMRegister src) {
2522 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2523 InstructionMark im(this);
2524 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2525 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2526 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2527 emit_int8(0x11);
2528 emit_operand(src, dst);
2529 }
2530
2531 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2532 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2533 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2534 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2535 emit_int8(0x10);
2536 emit_int8((unsigned char)(0xC0 | encode));
2537 }
2538
2539 void Assembler::movss(XMMRegister dst, Address src) {
2540 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2541 InstructionMark im(this);
2542 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2543 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2544 simd_prefix(dst, xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2545 emit_int8(0x10);
2546 emit_operand(dst, src);
2547 }
2548
2549 void Assembler::movss(Address dst, XMMRegister src) {
2550 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2551 InstructionMark im(this);
2552 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
2553 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2554 simd_prefix(src, xnoreg, dst, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2555 emit_int8(0x11);
2556 emit_operand(src, dst);
2557 }
2558
2559 void Assembler::movswl(Register dst, Address src) { // movsxw
2560 InstructionMark im(this);
2561 prefix(src, dst);
2562 emit_int8(0x0F);
2563 emit_int8((unsigned char)0xBF);
2564 emit_operand(dst, src);
2565 }
2566
2567 void Assembler::movswl(Register dst, Register src) { // movsxw
2568 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2569 emit_int8(0x0F);
2570 emit_int8((unsigned char)0xBF);
2571 emit_int8((unsigned char)(0xC0 | encode));
2572 }
2573
2574 void Assembler::movw(Address dst, int imm16) {
2575 InstructionMark im(this);
2576
2577 emit_int8(0x66); // switch to 16-bit mode
2578 prefix(dst);
2579 emit_int8((unsigned char)0xC7);
2580 emit_operand(rax, dst, 2);
2581 emit_int16(imm16);
2582 }
2583
2584 void Assembler::movw(Register dst, Address src) {
2585 InstructionMark im(this);
2586 emit_int8(0x66);
2587 prefix(src, dst);
2588 emit_int8((unsigned char)0x8B);
2589 emit_operand(dst, src);
2590 }
2591
2592 void Assembler::movw(Address dst, Register src) {
2593 InstructionMark im(this);
2594 emit_int8(0x66);
2595 prefix(dst, src);
2596 emit_int8((unsigned char)0x89);
2597 emit_operand(src, dst);
2598 }
2599
2600 void Assembler::movzbl(Register dst, Address src) { // movzxb
2601 InstructionMark im(this);
2602 prefix(src, dst);
2603 emit_int8(0x0F);
2604 emit_int8((unsigned char)0xB6);
2605 emit_operand(dst, src);
2606 }
2607
2608 void Assembler::movzbl(Register dst, Register src) { // movzxb
2609 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2610 int encode = prefix_and_encode(dst->encoding(), false, src->encoding(), true);
2611 emit_int8(0x0F);
2612 emit_int8((unsigned char)0xB6);
2613 emit_int8(0xC0 | encode);
2614 }
2615
2616 void Assembler::movzwl(Register dst, Address src) { // movzxw
2617 InstructionMark im(this);
2618 prefix(src, dst);
2619 emit_int8(0x0F);
2620 emit_int8((unsigned char)0xB7);
2621 emit_operand(dst, src);
2622 }
2623
2624 void Assembler::movzwl(Register dst, Register src) { // movzxw
2625 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2626 emit_int8(0x0F);
2627 emit_int8((unsigned char)0xB7);
2628 emit_int8(0xC0 | encode);
2629 }
2630
2631 void Assembler::mull(Address src) {
2632 InstructionMark im(this);
2633 prefix(src);
2634 emit_int8((unsigned char)0xF7);
2635 emit_operand(rsp, src);
2636 }
2637
2638 void Assembler::mull(Register src) {
2639 int encode = prefix_and_encode(src->encoding());
2640 emit_int8((unsigned char)0xF7);
2641 emit_int8((unsigned char)(0xE0 | encode));
2642 }
2643
2644 void Assembler::mulsd(XMMRegister dst, Address src) {
2645 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2646 InstructionMark im(this);
2647 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2648 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
2649 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2650 emit_int8(0x59);
2651 emit_operand(dst, src);
2652 }
2653
2654 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2655 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2656 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2657 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
2658 emit_int8(0x59);
2659 emit_int8((unsigned char)(0xC0 | encode));
2660 }
2661
2662 void Assembler::mulss(XMMRegister dst, Address src) {
2663 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2664 InstructionMark im(this);
2665 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2666 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
2667 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2668 emit_int8(0x59);
2669 emit_operand(dst, src);
2670 }
2671
2672 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2673 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2674 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
2675 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
2676 emit_int8(0x59);
2677 emit_int8((unsigned char)(0xC0 | encode));
2678 }
2679
2680 void Assembler::negl(Register dst) {
2681 int encode = prefix_and_encode(dst->encoding());
2682 emit_int8((unsigned char)0xF7);
2683 emit_int8((unsigned char)(0xD8 | encode));
2684 }
2685
2686 void Assembler::nop(int i) {
2687 #ifdef ASSERT
2688 assert(i > 0, " ");
2689 // The fancy nops aren't currently recognized by debuggers making it a
2690 // pain to disassemble code while debugging. If asserts are on clearly
2691 // speed is not an issue so simply use the single byte traditional nop
2692 // to do alignment.
2693
2694 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2695 return;
2696
2697 #endif // ASSERT
2698
2699 if (UseAddressNop && VM_Version::is_intel()) {
2700 //
2701 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2702 // 1: 0x90
2703 // 2: 0x66 0x90
2704 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2705 // 4: 0x0F 0x1F 0x40 0x00
2706 // 5: 0x0F 0x1F 0x44 0x00 0x00
2707 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2708 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2709 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2710 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2711 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2712 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2713
2714 // The rest coding is Intel specific - don't use consecutive address nops
2715
2716 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2717 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2718 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2719 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2720
2721 while(i >= 15) {
2722 // For Intel don't generate consecutive addess nops (mix with regular nops)
2723 i -= 15;
2724 emit_int8(0x66); // size prefix
2725 emit_int8(0x66); // size prefix
2726 emit_int8(0x66); // size prefix
2727 addr_nop_8();
2728 emit_int8(0x66); // size prefix
2729 emit_int8(0x66); // size prefix
2730 emit_int8(0x66); // size prefix
2731 emit_int8((unsigned char)0x90);
2732 // nop
2733 }
2734 switch (i) {
2735 case 14:
2736 emit_int8(0x66); // size prefix
2737 case 13:
2738 emit_int8(0x66); // size prefix
2739 case 12:
2740 addr_nop_8();
2741 emit_int8(0x66); // size prefix
2742 emit_int8(0x66); // size prefix
2743 emit_int8(0x66); // size prefix
2744 emit_int8((unsigned char)0x90);
2745 // nop
2746 break;
2747 case 11:
2748 emit_int8(0x66); // size prefix
2749 case 10:
2750 emit_int8(0x66); // size prefix
2751 case 9:
2752 emit_int8(0x66); // size prefix
2753 case 8:
2754 addr_nop_8();
2755 break;
2756 case 7:
2757 addr_nop_7();
2758 break;
2759 case 6:
2760 emit_int8(0x66); // size prefix
2761 case 5:
2762 addr_nop_5();
2763 break;
2764 case 4:
2765 addr_nop_4();
2766 break;
2767 case 3:
2768 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2769 emit_int8(0x66); // size prefix
2770 case 2:
2771 emit_int8(0x66); // size prefix
2772 case 1:
2773 emit_int8((unsigned char)0x90);
2774 // nop
2775 break;
2776 default:
2777 assert(i == 0, " ");
2778 }
2779 return;
2780 }
2781 if (UseAddressNop && VM_Version::is_amd()) {
2782 //
2783 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2784 // 1: 0x90
2785 // 2: 0x66 0x90
2786 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2787 // 4: 0x0F 0x1F 0x40 0x00
2788 // 5: 0x0F 0x1F 0x44 0x00 0x00
2789 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2790 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2791 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2792 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2793 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2794 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2795
2796 // The rest coding is AMD specific - use consecutive address nops
2797
2798 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2799 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2800 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2801 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2802 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2803 // Size prefixes (0x66) are added for larger sizes
2804
2805 while(i >= 22) {
2806 i -= 11;
2807 emit_int8(0x66); // size prefix
2808 emit_int8(0x66); // size prefix
2809 emit_int8(0x66); // size prefix
2810 addr_nop_8();
2811 }
2812 // Generate first nop for size between 21-12
2813 switch (i) {
2814 case 21:
2815 i -= 1;
2816 emit_int8(0x66); // size prefix
2817 case 20:
2818 case 19:
2819 i -= 1;
2820 emit_int8(0x66); // size prefix
2821 case 18:
2822 case 17:
2823 i -= 1;
2824 emit_int8(0x66); // size prefix
2825 case 16:
2826 case 15:
2827 i -= 8;
2828 addr_nop_8();
2829 break;
2830 case 14:
2831 case 13:
2832 i -= 7;
2833 addr_nop_7();
2834 break;
2835 case 12:
2836 i -= 6;
2837 emit_int8(0x66); // size prefix
2838 addr_nop_5();
2839 break;
2840 default:
2841 assert(i < 12, " ");
2842 }
2843
2844 // Generate second nop for size between 11-1
2845 switch (i) {
2846 case 11:
2847 emit_int8(0x66); // size prefix
2848 case 10:
2849 emit_int8(0x66); // size prefix
2850 case 9:
2851 emit_int8(0x66); // size prefix
2852 case 8:
2853 addr_nop_8();
2854 break;
2855 case 7:
2856 addr_nop_7();
2857 break;
2858 case 6:
2859 emit_int8(0x66); // size prefix
2860 case 5:
2861 addr_nop_5();
2862 break;
2863 case 4:
2864 addr_nop_4();
2865 break;
2866 case 3:
2867 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2868 emit_int8(0x66); // size prefix
2869 case 2:
2870 emit_int8(0x66); // size prefix
2871 case 1:
2872 emit_int8((unsigned char)0x90);
2873 // nop
2874 break;
2875 default:
2876 assert(i == 0, " ");
2877 }
2878 return;
2879 }
2880
2881 // Using nops with size prefixes "0x66 0x90".
2882 // From AMD Optimization Guide:
2883 // 1: 0x90
2884 // 2: 0x66 0x90
2885 // 3: 0x66 0x66 0x90
2886 // 4: 0x66 0x66 0x66 0x90
2887 // 5: 0x66 0x66 0x90 0x66 0x90
2888 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
2889 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2890 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2891 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2892 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2893 //
2894 while(i > 12) {
2895 i -= 4;
2896 emit_int8(0x66); // size prefix
2897 emit_int8(0x66);
2898 emit_int8(0x66);
2899 emit_int8((unsigned char)0x90);
2900 // nop
2901 }
2902 // 1 - 12 nops
2903 if(i > 8) {
2904 if(i > 9) {
2905 i -= 1;
2906 emit_int8(0x66);
2907 }
2908 i -= 3;
2909 emit_int8(0x66);
2910 emit_int8(0x66);
2911 emit_int8((unsigned char)0x90);
2912 }
2913 // 1 - 8 nops
2914 if(i > 4) {
2915 if(i > 6) {
2916 i -= 1;
2917 emit_int8(0x66);
2918 }
2919 i -= 3;
2920 emit_int8(0x66);
2921 emit_int8(0x66);
2922 emit_int8((unsigned char)0x90);
2923 }
2924 switch (i) {
2925 case 4:
2926 emit_int8(0x66);
2927 case 3:
2928 emit_int8(0x66);
2929 case 2:
2930 emit_int8(0x66);
2931 case 1:
2932 emit_int8((unsigned char)0x90);
2933 break;
2934 default:
2935 assert(i == 0, " ");
2936 }
2937 }
2938
2939 void Assembler::notl(Register dst) {
2940 int encode = prefix_and_encode(dst->encoding());
2941 emit_int8((unsigned char)0xF7);
2942 emit_int8((unsigned char)(0xD0 | encode));
2943 }
2944
2945 void Assembler::orl(Address dst, int32_t imm32) {
2946 InstructionMark im(this);
2947 prefix(dst);
2948 emit_arith_operand(0x81, rcx, dst, imm32);
2949 }
2950
2951 void Assembler::orl(Register dst, int32_t imm32) {
2952 prefix(dst);
2953 emit_arith(0x81, 0xC8, dst, imm32);
2954 }
2955
2956 void Assembler::orl(Register dst, Address src) {
2957 InstructionMark im(this);
2958 prefix(src, dst);
2959 emit_int8(0x0B);
2960 emit_operand(dst, src);
2961 }
2962
2963 void Assembler::orl(Register dst, Register src) {
2964 (void) prefix_and_encode(dst->encoding(), src->encoding());
2965 emit_arith(0x0B, 0xC0, dst, src);
2966 }
2967
2968 void Assembler::orl(Address dst, Register src) {
2969 InstructionMark im(this);
2970 prefix(dst, src);
2971 emit_int8(0x09);
2972 emit_operand(src, dst);
2973 }
2974
2975 void Assembler::packuswb(XMMRegister dst, Address src) {
2976 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2977 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2978 InstructionMark im(this);
2979 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2980 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
2981 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2982 emit_int8(0x67);
2983 emit_operand(dst, src);
2984 }
2985
2986 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2987 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2988 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2989 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2990 emit_int8(0x67);
2991 emit_int8((unsigned char)(0xC0 | encode));
2992 }
2993
2994 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2995 assert(UseAVX > 0, "some form of AVX must be enabled");
2996 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
2997 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
2998 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
2999 emit_int8(0x67);
3000 emit_int8((unsigned char)(0xC0 | encode));
3001 }
3002
3003 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
3004 assert(VM_Version::supports_avx2(), "");
3005 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3006 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3007 emit_int8(0x00);
3008 emit_int8(0xC0 | encode);
3009 emit_int8(imm8);
3010 }
3011
3012 void Assembler::pause() {
3013 emit_int8((unsigned char)0xF3);
3014 emit_int8((unsigned char)0x90);
3015 }
3016
3017 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3018 assert(VM_Version::supports_sse4_2(), "");
3019 InstructionMark im(this);
3020 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3021 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3022 emit_int8(0x61);
3023 emit_operand(dst, src);
3024 emit_int8(imm8);
3025 }
3026
3027 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3028 assert(VM_Version::supports_sse4_2(), "");
3029 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3030 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3031 emit_int8(0x61);
3032 emit_int8((unsigned char)(0xC0 | encode));
3033 emit_int8(imm8);
3034 }
3035
3036 void Assembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3037 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3038 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3039 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3040 emit_int8(0x75);
3041 emit_int8((unsigned char)(0xC0 | encode));
3042 }
3043
3044 void Assembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3045 assert(VM_Version::supports_avx(), "");
3046 assert(!VM_Version::supports_evex(), "");
3047 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3048 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3049 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3050 emit_int8(0x75);
3051 emit_int8((unsigned char)(0xC0 | encode));
3052 }
3053
3054 void Assembler::pmovmskb(Register dst, XMMRegister src) {
3055 assert(VM_Version::supports_sse2(), "");
3056 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3057 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3058 emit_int8((unsigned char)0xD7);
3059 emit_int8((unsigned char)(0xC0 | encode));
3060 }
3061
3062 void Assembler::vpmovmskb(Register dst, XMMRegister src) {
3063 assert(VM_Version::supports_avx2(), "");
3064 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3065 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3066 emit_int8((unsigned char)0xD7);
3067 emit_int8((unsigned char)(0xC0 | encode));
3068 }
3069
3070 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3071 assert(VM_Version::supports_sse4_1(), "");
3072 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3073 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3074 emit_int8(0x16);
3075 emit_int8((unsigned char)(0xC0 | encode));
3076 emit_int8(imm8);
3077 }
3078
3079 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3080 assert(VM_Version::supports_sse4_1(), "");
3081 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3082 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3083 emit_int8(0x16);
3084 emit_int8((unsigned char)(0xC0 | encode));
3085 emit_int8(imm8);
3086 }
3087
3088 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3089 assert(VM_Version::supports_sse2(), "");
3090 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3091 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3092 emit_int8((unsigned char)0xC5);
3093 emit_int8((unsigned char)(0xC0 | encode));
3094 emit_int8(imm8);
3095 }
3096
3097 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3098 assert(VM_Version::supports_sse4_1(), "");
3099 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3100 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3101 emit_int8(0x22);
3102 emit_int8((unsigned char)(0xC0 | encode));
3103 emit_int8(imm8);
3104 }
3105
3106 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3107 assert(VM_Version::supports_sse4_1(), "");
3108 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ true, /* uses_vl */ false);
3109 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
3110 emit_int8(0x22);
3111 emit_int8((unsigned char)(0xC0 | encode));
3112 emit_int8(imm8);
3113 }
3114
3115 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3116 assert(VM_Version::supports_sse2(), "");
3117 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3118 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3119 emit_int8((unsigned char)0xC4);
3120 emit_int8((unsigned char)(0xC0 | encode));
3121 emit_int8(imm8);
3122 }
3123
3124 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3125 assert(VM_Version::supports_sse4_1(), "");
3126 InstructionMark im(this);
3127 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3128 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3129 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3130 emit_int8(0x30);
3131 emit_operand(dst, src);
3132 }
3133
3134 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3135 assert(VM_Version::supports_sse4_1(), "");
3136 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3137 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3138 emit_int8(0x30);
3139 emit_int8((unsigned char)(0xC0 | encode));
3140 }
3141
3142 void Assembler::vpmovzxbw(XMMRegister dst, Address src) {
3143 assert(VM_Version::supports_avx(), "");
3144 InstructionMark im(this);
3145 assert(dst != xnoreg, "sanity");
3146 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3147 attributes.set_address_attributes(/* tuple_type */ EVEX_HVM, /* input_size_in_bits */ EVEX_NObit);
3148 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3149 emit_int8(0x30);
3150 emit_operand(dst, src);
3151 }
3152
3153 // generic
3154 void Assembler::pop(Register dst) {
3155 int encode = prefix_and_encode(dst->encoding());
3156 emit_int8(0x58 | encode);
3157 }
3158
3159 void Assembler::popcntl(Register dst, Address src) {
3160 assert(VM_Version::supports_popcnt(), "must support");
3161 InstructionMark im(this);
3162 emit_int8((unsigned char)0xF3);
3163 prefix(src, dst);
3164 emit_int8(0x0F);
3165 emit_int8((unsigned char)0xB8);
3166 emit_operand(dst, src);
3167 }
3168
3169 void Assembler::popcntl(Register dst, Register src) {
3170 assert(VM_Version::supports_popcnt(), "must support");
3171 emit_int8((unsigned char)0xF3);
3172 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3173 emit_int8(0x0F);
3174 emit_int8((unsigned char)0xB8);
3175 emit_int8((unsigned char)(0xC0 | encode));
3176 }
3177
3178 void Assembler::popf() {
3179 emit_int8((unsigned char)0x9D);
3180 }
3181
3182 #ifndef _LP64 // no 32bit push/pop on amd64
3183 void Assembler::popl(Address dst) {
3184 // NOTE: this will adjust stack by 8byte on 64bits
3185 InstructionMark im(this);
3186 prefix(dst);
3187 emit_int8((unsigned char)0x8F);
3188 emit_operand(rax, dst);
3189 }
3190 #endif
3191
3192 void Assembler::prefetch_prefix(Address src) {
3193 prefix(src);
3194 emit_int8(0x0F);
3195 }
3196
3197 void Assembler::prefetchnta(Address src) {
3198 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3199 InstructionMark im(this);
3200 prefetch_prefix(src);
3201 emit_int8(0x18);
3202 emit_operand(rax, src); // 0, src
3203 }
3204
3205 void Assembler::prefetchr(Address src) {
3206 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3207 InstructionMark im(this);
3208 prefetch_prefix(src);
3209 emit_int8(0x0D);
3210 emit_operand(rax, src); // 0, src
3211 }
3212
3213 void Assembler::prefetcht0(Address src) {
3214 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3215 InstructionMark im(this);
3216 prefetch_prefix(src);
3217 emit_int8(0x18);
3218 emit_operand(rcx, src); // 1, src
3219 }
3220
3221 void Assembler::prefetcht1(Address src) {
3222 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3223 InstructionMark im(this);
3224 prefetch_prefix(src);
3225 emit_int8(0x18);
3226 emit_operand(rdx, src); // 2, src
3227 }
3228
3229 void Assembler::prefetcht2(Address src) {
3230 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3231 InstructionMark im(this);
3232 prefetch_prefix(src);
3233 emit_int8(0x18);
3234 emit_operand(rbx, src); // 3, src
3235 }
3236
3237 void Assembler::prefetchw(Address src) {
3238 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3239 InstructionMark im(this);
3240 prefetch_prefix(src);
3241 emit_int8(0x0D);
3242 emit_operand(rcx, src); // 1, src
3243 }
3244
3245 void Assembler::prefix(Prefix p) {
3246 emit_int8(p);
3247 }
3248
3249 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3250 assert(VM_Version::supports_ssse3(), "");
3251 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3252 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3253 emit_int8(0x00);
3254 emit_int8((unsigned char)(0xC0 | encode));
3255 }
3256
3257 void Assembler::pshufb(XMMRegister dst, Address src) {
3258 assert(VM_Version::supports_ssse3(), "");
3259 InstructionMark im(this);
3260 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3261 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3262 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3263 emit_int8(0x00);
3264 emit_operand(dst, src);
3265 }
3266
3267 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3268 assert(isByte(mode), "invalid value");
3269 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3270 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_128bit;
3271 InstructionAttr attributes(vector_len, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3272 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3273 emit_int8(0x70);
3274 emit_int8((unsigned char)(0xC0 | encode));
3275 emit_int8(mode & 0xFF);
3276 }
3277
3278 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3279 assert(isByte(mode), "invalid value");
3280 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3281 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3282 InstructionMark im(this);
3283 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3284 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3285 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3286 emit_int8(0x70);
3287 emit_operand(dst, src);
3288 emit_int8(mode & 0xFF);
3289 }
3290
3291 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3292 assert(isByte(mode), "invalid value");
3293 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3294 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3295 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3296 emit_int8(0x70);
3297 emit_int8((unsigned char)(0xC0 | encode));
3298 emit_int8(mode & 0xFF);
3299 }
3300
3301 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3302 assert(isByte(mode), "invalid value");
3303 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3304 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3305 InstructionMark im(this);
3306 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ false);
3307 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3308 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3309 emit_int8(0x70);
3310 emit_operand(dst, src);
3311 emit_int8(mode & 0xFF);
3312 }
3313
3314 void Assembler::psrldq(XMMRegister dst, int shift) {
3315 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3316 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3317 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3318 // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3319 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3320 emit_int8(0x73);
3321 emit_int8((unsigned char)(0xC0 | encode));
3322 emit_int8(shift);
3323 }
3324
3325 void Assembler::pslldq(XMMRegister dst, int shift) {
3326 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3327 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3328 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ true, /* uses_vl */ false);
3329 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3330 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3331 emit_int8(0x73);
3332 emit_int8((unsigned char)(0xC0 | encode));
3333 emit_int8(shift);
3334 }
3335
3336 void Assembler::ptest(XMMRegister dst, Address src) {
3337 assert(VM_Version::supports_sse4_1(), "");
3338 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3339 InstructionMark im(this);
3340 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3341 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3342 emit_int8(0x17);
3343 emit_operand(dst, src);
3344 }
3345
3346 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3347 assert(VM_Version::supports_sse4_1(), "");
3348 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3349 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3350 emit_int8(0x17);
3351 emit_int8((unsigned char)(0xC0 | encode));
3352 }
3353
3354 void Assembler::vptest(XMMRegister dst, Address src) {
3355 assert(VM_Version::supports_avx(), "");
3356 InstructionMark im(this);
3357 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3358 assert(dst != xnoreg, "sanity");
3359 // swap src<->dst for encoding
3360 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3361 emit_int8(0x17);
3362 emit_operand(dst, src);
3363 }
3364
3365 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3366 assert(VM_Version::supports_avx(), "");
3367 InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3368 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
3369 emit_int8(0x17);
3370 emit_int8((unsigned char)(0xC0 | encode));
3371 }
3372
3373 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3374 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3375 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3376 InstructionMark im(this);
3377 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3378 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
3379 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3380 emit_int8(0x60);
3381 emit_operand(dst, src);
3382 }
3383
3384 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3385 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3386 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_vlbw, /* no_mask_reg */ false, /* uses_vl */ true);
3387 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3388 emit_int8(0x60);
3389 emit_int8((unsigned char)(0xC0 | encode));
3390 }
3391
3392 void Assembler::punpckldq(XMMRegister dst, Address src) {
3393 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3394 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3395 InstructionMark im(this);
3396 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3397 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
3398 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3399 emit_int8(0x62);
3400 emit_operand(dst, src);
3401 }
3402
3403 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3404 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3405 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3406 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3407 emit_int8(0x62);
3408 emit_int8((unsigned char)(0xC0 | encode));
3409 }
3410
3411 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3412 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3413 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
3414 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3415 emit_int8(0x6C);
3416 emit_int8((unsigned char)(0xC0 | encode));
3417 }
3418
3419 void Assembler::push(int32_t imm32) {
3420 // in 64bits we push 64bits onto the stack but only
3421 // take a 32bit immediate
3422 emit_int8(0x68);
3423 emit_int32(imm32);
3424 }
3425
3426 void Assembler::push(Register src) {
3427 int encode = prefix_and_encode(src->encoding());
3428
3429 emit_int8(0x50 | encode);
3430 }
3431
3432 void Assembler::pushf() {
3433 emit_int8((unsigned char)0x9C);
3434 }
3435
3436 #ifndef _LP64 // no 32bit push/pop on amd64
3437 void Assembler::pushl(Address src) {
3438 // Note this will push 64bit on 64bit
3439 InstructionMark im(this);
3440 prefix(src);
3441 emit_int8((unsigned char)0xFF);
3442 emit_operand(rsi, src);
3443 }
3444 #endif
3445
3446 void Assembler::rcll(Register dst, int imm8) {
3447 assert(isShiftCount(imm8), "illegal shift count");
3448 int encode = prefix_and_encode(dst->encoding());
3449 if (imm8 == 1) {
3450 emit_int8((unsigned char)0xD1);
3451 emit_int8((unsigned char)(0xD0 | encode));
3452 } else {
3453 emit_int8((unsigned char)0xC1);
3454 emit_int8((unsigned char)0xD0 | encode);
3455 emit_int8(imm8);
3456 }
3457 }
3458
3459 void Assembler::rcpps(XMMRegister dst, XMMRegister src) {
3460 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3461 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3462 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3463 emit_int8(0x53);
3464 emit_int8((unsigned char)(0xC0 | encode));
3465 }
3466
3467 void Assembler::rcpss(XMMRegister dst, XMMRegister src) {
3468 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3469 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
3470 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3471 emit_int8(0x53);
3472 emit_int8((unsigned char)(0xC0 | encode));
3473 }
3474
3475 void Assembler::rdtsc() {
3476 emit_int8((unsigned char)0x0F);
3477 emit_int8((unsigned char)0x31);
3478 }
3479
3480 // copies data from [esi] to [edi] using rcx pointer sized words
3481 // generic
3482 void Assembler::rep_mov() {
3483 emit_int8((unsigned char)0xF3);
3484 // MOVSQ
3485 LP64_ONLY(prefix(REX_W));
3486 emit_int8((unsigned char)0xA5);
3487 }
3488
3489 // sets rcx bytes with rax, value at [edi]
3490 void Assembler::rep_stosb() {
3491 emit_int8((unsigned char)0xF3); // REP
3492 LP64_ONLY(prefix(REX_W));
3493 emit_int8((unsigned char)0xAA); // STOSB
3494 }
3495
3496 // sets rcx pointer sized words with rax, value at [edi]
3497 // generic
3498 void Assembler::rep_stos() {
3499 emit_int8((unsigned char)0xF3); // REP
3500 LP64_ONLY(prefix(REX_W)); // LP64:STOSQ, LP32:STOSD
3501 emit_int8((unsigned char)0xAB);
3502 }
3503
3504 // scans rcx pointer sized words at [edi] for occurance of rax,
3505 // generic
3506 void Assembler::repne_scan() { // repne_scan
3507 emit_int8((unsigned char)0xF2);
3508 // SCASQ
3509 LP64_ONLY(prefix(REX_W));
3510 emit_int8((unsigned char)0xAF);
3511 }
3512
3513 #ifdef _LP64
3514 // scans rcx 4 byte words at [edi] for occurance of rax,
3515 // generic
3516 void Assembler::repne_scanl() { // repne_scan
3517 emit_int8((unsigned char)0xF2);
3518 // SCASL
3519 emit_int8((unsigned char)0xAF);
3520 }
3521 #endif
3522
3523 void Assembler::ret(int imm16) {
3524 if (imm16 == 0) {
3525 emit_int8((unsigned char)0xC3);
3526 } else {
3527 emit_int8((unsigned char)0xC2);
3528 emit_int16(imm16);
3529 }
3530 }
3531
3532 void Assembler::sahf() {
3533 #ifdef _LP64
3534 // Not supported in 64bit mode
3535 ShouldNotReachHere();
3536 #endif
3537 emit_int8((unsigned char)0x9E);
3538 }
3539
3540 void Assembler::sarl(Register dst, int imm8) {
3541 int encode = prefix_and_encode(dst->encoding());
3542 assert(isShiftCount(imm8), "illegal shift count");
3543 if (imm8 == 1) {
3544 emit_int8((unsigned char)0xD1);
3545 emit_int8((unsigned char)(0xF8 | encode));
3546 } else {
3547 emit_int8((unsigned char)0xC1);
3548 emit_int8((unsigned char)(0xF8 | encode));
3549 emit_int8(imm8);
3550 }
3551 }
3552
3553 void Assembler::sarl(Register dst) {
3554 int encode = prefix_and_encode(dst->encoding());
3555 emit_int8((unsigned char)0xD3);
3556 emit_int8((unsigned char)(0xF8 | encode));
3557 }
3558
3559 void Assembler::sbbl(Address dst, int32_t imm32) {
3560 InstructionMark im(this);
3561 prefix(dst);
3562 emit_arith_operand(0x81, rbx, dst, imm32);
3563 }
3564
3565 void Assembler::sbbl(Register dst, int32_t imm32) {
3566 prefix(dst);
3567 emit_arith(0x81, 0xD8, dst, imm32);
3568 }
3569
3570
3571 void Assembler::sbbl(Register dst, Address src) {
3572 InstructionMark im(this);
3573 prefix(src, dst);
3574 emit_int8(0x1B);
3575 emit_operand(dst, src);
3576 }
3577
3578 void Assembler::sbbl(Register dst, Register src) {
3579 (void) prefix_and_encode(dst->encoding(), src->encoding());
3580 emit_arith(0x1B, 0xC0, dst, src);
3581 }
3582
3583 void Assembler::setb(Condition cc, Register dst) {
3584 assert(0 <= cc && cc < 16, "illegal cc");
3585 int encode = prefix_and_encode(dst->encoding(), true);
3586 emit_int8(0x0F);
3587 emit_int8((unsigned char)0x90 | cc);
3588 emit_int8((unsigned char)(0xC0 | encode));
3589 }
3590
3591 void Assembler::shll(Register dst, int imm8) {
3592 assert(isShiftCount(imm8), "illegal shift count");
3593 int encode = prefix_and_encode(dst->encoding());
3594 if (imm8 == 1 ) {
3595 emit_int8((unsigned char)0xD1);
3596 emit_int8((unsigned char)(0xE0 | encode));
3597 } else {
3598 emit_int8((unsigned char)0xC1);
3599 emit_int8((unsigned char)(0xE0 | encode));
3600 emit_int8(imm8);
3601 }
3602 }
3603
3604 void Assembler::shll(Register dst) {
3605 int encode = prefix_and_encode(dst->encoding());
3606 emit_int8((unsigned char)0xD3);
3607 emit_int8((unsigned char)(0xE0 | encode));
3608 }
3609
3610 void Assembler::shrl(Register dst, int imm8) {
3611 assert(isShiftCount(imm8), "illegal shift count");
3612 int encode = prefix_and_encode(dst->encoding());
3613 emit_int8((unsigned char)0xC1);
3614 emit_int8((unsigned char)(0xE8 | encode));
3615 emit_int8(imm8);
3616 }
3617
3618 void Assembler::shrl(Register dst) {
3619 int encode = prefix_and_encode(dst->encoding());
3620 emit_int8((unsigned char)0xD3);
3621 emit_int8((unsigned char)(0xE8 | encode));
3622 }
3623
3624 // copies a single word from [esi] to [edi]
3625 void Assembler::smovl() {
3626 emit_int8((unsigned char)0xA5);
3627 }
3628
3629 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3630 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3631 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3632 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3633 emit_int8(0x51);
3634 emit_int8((unsigned char)(0xC0 | encode));
3635 }
3636
3637 void Assembler::sqrtsd(XMMRegister dst, Address src) {
3638 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3639 InstructionMark im(this);
3640 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3641 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3642 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3643 emit_int8(0x51);
3644 emit_operand(dst, src);
3645 }
3646
3647 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3648 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3649 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3650 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3651 emit_int8(0x51);
3652 emit_int8((unsigned char)(0xC0 | encode));
3653 }
3654
3655 void Assembler::std() {
3656 emit_int8((unsigned char)0xFD);
3657 }
3658
3659 void Assembler::sqrtss(XMMRegister dst, Address src) {
3660 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3661 InstructionMark im(this);
3662 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3663 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3664 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3665 emit_int8(0x51);
3666 emit_operand(dst, src);
3667 }
3668
3669 void Assembler::stmxcsr( Address dst) {
3670 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3671 InstructionMark im(this);
3672 prefix(dst);
3673 emit_int8(0x0F);
3674 emit_int8((unsigned char)0xAE);
3675 emit_operand(as_Register(3), dst);
3676 }
3677
3678 void Assembler::subl(Address dst, int32_t imm32) {
3679 InstructionMark im(this);
3680 prefix(dst);
3681 emit_arith_operand(0x81, rbp, dst, imm32);
3682 }
3683
3684 void Assembler::subl(Address dst, Register src) {
3685 InstructionMark im(this);
3686 prefix(dst, src);
3687 emit_int8(0x29);
3688 emit_operand(src, dst);
3689 }
3690
3691 void Assembler::subl(Register dst, int32_t imm32) {
3692 prefix(dst);
3693 emit_arith(0x81, 0xE8, dst, imm32);
3694 }
3695
3696 // Force generation of a 4 byte immediate value even if it fits into 8bit
3697 void Assembler::subl_imm32(Register dst, int32_t imm32) {
3698 prefix(dst);
3699 emit_arith_imm32(0x81, 0xE8, dst, imm32);
3700 }
3701
3702 void Assembler::subl(Register dst, Address src) {
3703 InstructionMark im(this);
3704 prefix(src, dst);
3705 emit_int8(0x2B);
3706 emit_operand(dst, src);
3707 }
3708
3709 void Assembler::subl(Register dst, Register src) {
3710 (void) prefix_and_encode(dst->encoding(), src->encoding());
3711 emit_arith(0x2B, 0xC0, dst, src);
3712 }
3713
3714 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3715 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3716 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3717 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3718 emit_int8(0x5C);
3719 emit_int8((unsigned char)(0xC0 | encode));
3720 }
3721
3722 void Assembler::subsd(XMMRegister dst, Address src) {
3723 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3724 InstructionMark im(this);
3725 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3726 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3727 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3728 emit_int8(0x5C);
3729 emit_operand(dst, src);
3730 }
3731
3732 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3733 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3734 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3735 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3736 emit_int8(0x5C);
3737 emit_int8((unsigned char)(0xC0 | encode));
3738 }
3739
3740 void Assembler::subss(XMMRegister dst, Address src) {
3741 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3742 InstructionMark im(this);
3743 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3744 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3745 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3746 emit_int8(0x5C);
3747 emit_operand(dst, src);
3748 }
3749
3750 void Assembler::testb(Register dst, int imm8) {
3751 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
3752 (void) prefix_and_encode(dst->encoding(), true);
3753 emit_arith_b(0xF6, 0xC0, dst, imm8);
3754 }
3755
3756 void Assembler::testl(Register dst, int32_t imm32) {
3757 // not using emit_arith because test
3758 // doesn't support sign-extension of
3759 // 8bit operands
3760 int encode = dst->encoding();
3761 if (encode == 0) {
3762 emit_int8((unsigned char)0xA9);
3763 } else {
3764 encode = prefix_and_encode(encode);
3765 emit_int8((unsigned char)0xF7);
3766 emit_int8((unsigned char)(0xC0 | encode));
3767 }
3768 emit_int32(imm32);
3769 }
3770
3771 void Assembler::testl(Register dst, Register src) {
3772 (void) prefix_and_encode(dst->encoding(), src->encoding());
3773 emit_arith(0x85, 0xC0, dst, src);
3774 }
3775
3776 void Assembler::testl(Register dst, Address src) {
3777 InstructionMark im(this);
3778 prefix(src, dst);
3779 emit_int8((unsigned char)0x85);
3780 emit_operand(dst, src);
3781 }
3782
3783 void Assembler::tzcntl(Register dst, Register src) {
3784 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3785 emit_int8((unsigned char)0xF3);
3786 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3787 emit_int8(0x0F);
3788 emit_int8((unsigned char)0xBC);
3789 emit_int8((unsigned char)0xC0 | encode);
3790 }
3791
3792 void Assembler::tzcntq(Register dst, Register src) {
3793 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3794 emit_int8((unsigned char)0xF3);
3795 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3796 emit_int8(0x0F);
3797 emit_int8((unsigned char)0xBC);
3798 emit_int8((unsigned char)(0xC0 | encode));
3799 }
3800
3801 void Assembler::ucomisd(XMMRegister dst, Address src) {
3802 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3803 InstructionMark im(this);
3804 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3805 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3806 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3807 emit_int8(0x2E);
3808 emit_operand(dst, src);
3809 }
3810
3811 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3812 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3813 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3814 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
3815 emit_int8(0x2E);
3816 emit_int8((unsigned char)(0xC0 | encode));
3817 }
3818
3819 void Assembler::ucomiss(XMMRegister dst, Address src) {
3820 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3821 InstructionMark im(this);
3822 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3823 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3824 simd_prefix(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3825 emit_int8(0x2E);
3826 emit_operand(dst, src);
3827 }
3828
3829 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3830 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3831 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3832 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
3833 emit_int8(0x2E);
3834 emit_int8((unsigned char)(0xC0 | encode));
3835 }
3836
3837 void Assembler::xabort(int8_t imm8) {
3838 emit_int8((unsigned char)0xC6);
3839 emit_int8((unsigned char)0xF8);
3840 emit_int8((unsigned char)(imm8 & 0xFF));
3841 }
3842
3843 void Assembler::xaddl(Address dst, Register src) {
3844 InstructionMark im(this);
3845 prefix(dst, src);
3846 emit_int8(0x0F);
3847 emit_int8((unsigned char)0xC1);
3848 emit_operand(src, dst);
3849 }
3850
3851 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3852 InstructionMark im(this);
3853 relocate(rtype);
3854 if (abort.is_bound()) {
3855 address entry = target(abort);
3856 assert(entry != NULL, "abort entry NULL");
3857 intptr_t offset = entry - pc();
3858 emit_int8((unsigned char)0xC7);
3859 emit_int8((unsigned char)0xF8);
3860 emit_int32(offset - 6); // 2 opcode + 4 address
3861 } else {
3862 abort.add_patch_at(code(), locator());
3863 emit_int8((unsigned char)0xC7);
3864 emit_int8((unsigned char)0xF8);
3865 emit_int32(0);
3866 }
3867 }
3868
3869 void Assembler::xchgl(Register dst, Address src) { // xchg
3870 InstructionMark im(this);
3871 prefix(src, dst);
3872 emit_int8((unsigned char)0x87);
3873 emit_operand(dst, src);
3874 }
3875
3876 void Assembler::xchgl(Register dst, Register src) {
3877 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3878 emit_int8((unsigned char)0x87);
3879 emit_int8((unsigned char)(0xC0 | encode));
3880 }
3881
3882 void Assembler::xend() {
3883 emit_int8((unsigned char)0x0F);
3884 emit_int8((unsigned char)0x01);
3885 emit_int8((unsigned char)0xD5);
3886 }
3887
3888 void Assembler::xgetbv() {
3889 emit_int8(0x0F);
3890 emit_int8(0x01);
3891 emit_int8((unsigned char)0xD0);
3892 }
3893
3894 void Assembler::xorl(Register dst, int32_t imm32) {
3895 prefix(dst);
3896 emit_arith(0x81, 0xF0, dst, imm32);
3897 }
3898
3899 void Assembler::xorl(Register dst, Address src) {
3900 InstructionMark im(this);
3901 prefix(src, dst);
3902 emit_int8(0x33);
3903 emit_operand(dst, src);
3904 }
3905
3906 void Assembler::xorl(Register dst, Register src) {
3907 (void) prefix_and_encode(dst->encoding(), src->encoding());
3908 emit_arith(0x33, 0xC0, dst, src);
3909 }
3910
3911
3912 // AVX 3-operands scalar float-point arithmetic instructions
3913
3914 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3915 assert(VM_Version::supports_avx(), "");
3916 InstructionMark im(this);
3917 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3918 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3919 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3920 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3921 emit_int8(0x58);
3922 emit_operand(dst, src);
3923 }
3924
3925 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3926 assert(VM_Version::supports_avx(), "");
3927 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3928 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3929 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3930 emit_int8(0x58);
3931 emit_int8((unsigned char)(0xC0 | encode));
3932 }
3933
3934 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3935 assert(VM_Version::supports_avx(), "");
3936 InstructionMark im(this);
3937 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3938 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3939 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3940 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3941 emit_int8(0x58);
3942 emit_operand(dst, src);
3943 }
3944
3945 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3946 assert(VM_Version::supports_avx(), "");
3947 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3948 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3949 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3950 emit_int8(0x58);
3951 emit_int8((unsigned char)(0xC0 | encode));
3952 }
3953
3954 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3955 assert(VM_Version::supports_avx(), "");
3956 InstructionMark im(this);
3957 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3958 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3959 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3960 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3961 emit_int8(0x5E);
3962 emit_operand(dst, src);
3963 }
3964
3965 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3966 assert(VM_Version::supports_avx(), "");
3967 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3968 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3969 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
3970 emit_int8(0x5E);
3971 emit_int8((unsigned char)(0xC0 | encode));
3972 }
3973
3974 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3975 assert(VM_Version::supports_avx(), "");
3976 InstructionMark im(this);
3977 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3978 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
3979 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3980 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3981 emit_int8(0x5E);
3982 emit_operand(dst, src);
3983 }
3984
3985 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3986 assert(VM_Version::supports_avx(), "");
3987 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3988 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3989 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
3990 emit_int8(0x5E);
3991 emit_int8((unsigned char)(0xC0 | encode));
3992 }
3993
3994 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3995 assert(VM_Version::supports_avx(), "");
3996 InstructionMark im(this);
3997 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
3998 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
3999 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4000 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4001 emit_int8(0x59);
4002 emit_operand(dst, src);
4003 }
4004
4005 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4006 assert(VM_Version::supports_avx(), "");
4007 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4008 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4009 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4010 emit_int8(0x59);
4011 emit_int8((unsigned char)(0xC0 | encode));
4012 }
4013
4014 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
4015 assert(VM_Version::supports_avx(), "");
4016 InstructionMark im(this);
4017 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4018 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4019 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4020 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4021 emit_int8(0x59);
4022 emit_operand(dst, src);
4023 }
4024
4025 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4026 assert(VM_Version::supports_avx(), "");
4027 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4028 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4029 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4030 emit_int8(0x59);
4031 emit_int8((unsigned char)(0xC0 | encode));
4032 }
4033
4034 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
4035 assert(VM_Version::supports_avx(), "");
4036 InstructionMark im(this);
4037 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4038 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
4039 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4040 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4041 emit_int8(0x5C);
4042 emit_operand(dst, src);
4043 }
4044
4045 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4046 assert(VM_Version::supports_avx(), "");
4047 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4048 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4049 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
4050 emit_int8(0x5C);
4051 emit_int8((unsigned char)(0xC0 | encode));
4052 }
4053
4054 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
4055 assert(VM_Version::supports_avx(), "");
4056 InstructionMark im(this);
4057 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4058 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
4059 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4060 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4061 emit_int8(0x5C);
4062 emit_operand(dst, src);
4063 }
4064
4065 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
4066 assert(VM_Version::supports_avx(), "");
4067 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
4068 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4069 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
4070 emit_int8(0x5C);
4071 emit_int8((unsigned char)(0xC0 | encode));
4072 }
4073
4074 //====================VECTOR ARITHMETIC=====================================
4075
4076 // Float-point vector arithmetic
4077
4078 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
4079 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4080 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4081 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4082 emit_int8(0x58);
4083 emit_int8((unsigned char)(0xC0 | encode));
4084 }
4085
4086 void Assembler::addps(XMMRegister dst, XMMRegister src) {
4087 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4088 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4089 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4090 emit_int8(0x58);
4091 emit_int8((unsigned char)(0xC0 | encode));
4092 }
4093
4094 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4095 assert(VM_Version::supports_avx(), "");
4096 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4097 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4098 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4099 emit_int8(0x58);
4100 emit_int8((unsigned char)(0xC0 | encode));
4101 }
4102
4103 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4104 assert(VM_Version::supports_avx(), "");
4105 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4106 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4107 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4108 emit_int8(0x58);
4109 emit_int8((unsigned char)(0xC0 | encode));
4110 }
4111
4112 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4113 assert(VM_Version::supports_avx(), "");
4114 InstructionMark im(this);
4115 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4116 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4117 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4118 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4119 emit_int8(0x58);
4120 emit_operand(dst, src);
4121 }
4122
4123 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4124 assert(VM_Version::supports_avx(), "");
4125 InstructionMark im(this);
4126 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4127 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4128 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4129 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4130 emit_int8(0x58);
4131 emit_operand(dst, src);
4132 }
4133
4134 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4135 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4136 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4137 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4138 emit_int8(0x5C);
4139 emit_int8((unsigned char)(0xC0 | encode));
4140 }
4141
4142 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4143 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4144 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4145 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4146 emit_int8(0x5C);
4147 emit_int8((unsigned char)(0xC0 | encode));
4148 }
4149
4150 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4151 assert(VM_Version::supports_avx(), "");
4152 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4153 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4154 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4155 emit_int8(0x5C);
4156 emit_int8((unsigned char)(0xC0 | encode));
4157 }
4158
4159 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4160 assert(VM_Version::supports_avx(), "");
4161 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4162 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4163 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4164 emit_int8(0x5C);
4165 emit_int8((unsigned char)(0xC0 | encode));
4166 }
4167
4168 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4169 assert(VM_Version::supports_avx(), "");
4170 InstructionMark im(this);
4171 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4172 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4173 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4174 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4175 emit_int8(0x5C);
4176 emit_operand(dst, src);
4177 }
4178
4179 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4180 assert(VM_Version::supports_avx(), "");
4181 InstructionMark im(this);
4182 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4183 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4184 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4185 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4186 emit_int8(0x5C);
4187 emit_operand(dst, src);
4188 }
4189
4190 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4191 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4192 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4193 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4194 emit_int8(0x59);
4195 emit_int8((unsigned char)(0xC0 | encode));
4196 }
4197
4198 void Assembler::mulpd(XMMRegister dst, Address src) {
4199 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4200 InstructionMark im(this);
4201 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4202 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4203 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4204 emit_int8(0x59);
4205 emit_operand(dst, src);
4206 }
4207
4208 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4209 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4210 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4211 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4212 emit_int8(0x59);
4213 emit_int8((unsigned char)(0xC0 | encode));
4214 }
4215
4216 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4217 assert(VM_Version::supports_avx(), "");
4218 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4219 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4220 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4221 emit_int8(0x59);
4222 emit_int8((unsigned char)(0xC0 | encode));
4223 }
4224
4225 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4226 assert(VM_Version::supports_avx(), "");
4227 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4228 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4229 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4230 emit_int8(0x59);
4231 emit_int8((unsigned char)(0xC0 | encode));
4232 }
4233
4234 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4235 assert(VM_Version::supports_avx(), "");
4236 InstructionMark im(this);
4237 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4238 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4239 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4240 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4241 emit_int8(0x59);
4242 emit_operand(dst, src);
4243 }
4244
4245 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4246 assert(VM_Version::supports_avx(), "");
4247 InstructionMark im(this);
4248 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4249 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4250 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4251 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4252 emit_int8(0x59);
4253 emit_operand(dst, src);
4254 }
4255
4256 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4257 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4258 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4259 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4260 emit_int8(0x5E);
4261 emit_int8((unsigned char)(0xC0 | encode));
4262 }
4263
4264 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4265 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4266 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4267 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4268 emit_int8(0x5E);
4269 emit_int8((unsigned char)(0xC0 | encode));
4270 }
4271
4272 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4273 assert(VM_Version::supports_avx(), "");
4274 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4275 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4276 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4277 emit_int8(0x5E);
4278 emit_int8((unsigned char)(0xC0 | encode));
4279 }
4280
4281 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4282 assert(VM_Version::supports_avx(), "");
4283 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4284 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4285 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4286 emit_int8(0x5E);
4287 emit_int8((unsigned char)(0xC0 | encode));
4288 }
4289
4290 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4291 assert(VM_Version::supports_avx(), "");
4292 InstructionMark im(this);
4293 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4294 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4295 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4296 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4297 emit_int8(0x5E);
4298 emit_operand(dst, src);
4299 }
4300
4301 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4302 assert(VM_Version::supports_avx(), "");
4303 InstructionMark im(this);
4304 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4305 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4306 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4307 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4308 emit_int8(0x5E);
4309 emit_operand(dst, src);
4310 }
4311
4312 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4313 assert(VM_Version::supports_avx(), "");
4314 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4315 int nds_enc = 0;
4316 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4317 emit_int8(0x51);
4318 emit_int8((unsigned char)(0xC0 | encode));
4319 }
4320
4321 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4322 assert(VM_Version::supports_avx(), "");
4323 InstructionMark im(this);
4324 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4325 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4326 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4327 emit_int8(0x51);
4328 emit_operand(dst, src);
4329 }
4330
4331 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4332 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4333 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4334 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4335 emit_int8(0x54);
4336 emit_int8((unsigned char)(0xC0 | encode));
4337 }
4338
4339 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4340 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4341 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4342 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4343 emit_int8(0x54);
4344 emit_int8((unsigned char)(0xC0 | encode));
4345 }
4346
4347 void Assembler::andps(XMMRegister dst, Address src) {
4348 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4349 InstructionMark im(this);
4350 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4351 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4352 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4353 emit_int8(0x54);
4354 emit_operand(dst, src);
4355 }
4356
4357 void Assembler::andpd(XMMRegister dst, Address src) {
4358 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4359 InstructionMark im(this);
4360 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4361 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4362 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4363 emit_int8(0x54);
4364 emit_operand(dst, src);
4365 }
4366
4367 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4368 assert(VM_Version::supports_avx(), "");
4369 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4370 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4371 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4372 emit_int8(0x54);
4373 emit_int8((unsigned char)(0xC0 | encode));
4374 }
4375
4376 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4377 assert(VM_Version::supports_avx(), "");
4378 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4379 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4380 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4381 emit_int8(0x54);
4382 emit_int8((unsigned char)(0xC0 | encode));
4383 }
4384
4385 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4386 assert(VM_Version::supports_avx(), "");
4387 InstructionMark im(this);
4388 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4389 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4390 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4391 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4392 emit_int8(0x54);
4393 emit_operand(dst, src);
4394 }
4395
4396 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4397 assert(VM_Version::supports_avx(), "");
4398 InstructionMark im(this);
4399 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4400 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4401 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4402 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4403 emit_int8(0x54);
4404 emit_operand(dst, src);
4405 }
4406
4407 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4408 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4409 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4410 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4411 emit_int8(0x15);
4412 emit_int8((unsigned char)(0xC0 | encode));
4413 }
4414
4415 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4416 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4417 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4418 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4419 emit_int8(0x14);
4420 emit_int8((unsigned char)(0xC0 | encode));
4421 }
4422
4423 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4424 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4425 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4426 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4427 emit_int8(0x57);
4428 emit_int8((unsigned char)(0xC0 | encode));
4429 }
4430
4431 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4432 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4433 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4434 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4435 emit_int8(0x57);
4436 emit_int8((unsigned char)(0xC0 | encode));
4437 }
4438
4439 void Assembler::xorpd(XMMRegister dst, Address src) {
4440 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4441 InstructionMark im(this);
4442 InstructionAttr attributes(AVX_128bit, /* rex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4443 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4444 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4445 emit_int8(0x57);
4446 emit_operand(dst, src);
4447 }
4448
4449 void Assembler::xorps(XMMRegister dst, Address src) {
4450 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4451 InstructionMark im(this);
4452 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4453 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4454 simd_prefix(dst, dst, src, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4455 emit_int8(0x57);
4456 emit_operand(dst, src);
4457 }
4458
4459 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4460 assert(VM_Version::supports_avx(), "");
4461 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4462 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4463 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4464 emit_int8(0x57);
4465 emit_int8((unsigned char)(0xC0 | encode));
4466 }
4467
4468 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4469 assert(VM_Version::supports_avx(), "");
4470 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4471 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4472 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4473 emit_int8(0x57);
4474 emit_int8((unsigned char)(0xC0 | encode));
4475 }
4476
4477 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4478 assert(VM_Version::supports_avx(), "");
4479 InstructionMark im(this);
4480 InstructionAttr attributes(vector_len, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4481 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4482 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4483 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4484 emit_int8(0x57);
4485 emit_operand(dst, src);
4486 }
4487
4488 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4489 assert(VM_Version::supports_avx(), "");
4490 InstructionMark im(this);
4491 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4492 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4493 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4494 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
4495 emit_int8(0x57);
4496 emit_operand(dst, src);
4497 }
4498
4499 // Integer vector arithmetic
4500 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4501 assert(VM_Version::supports_avx() && (vector_len == 0) ||
4502 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4503 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4504 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4505 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4506 emit_int8(0x01);
4507 emit_int8((unsigned char)(0xC0 | encode));
4508 }
4509
4510 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4511 assert(VM_Version::supports_avx() && (vector_len == 0) ||
4512 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4513 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4514 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4515 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4516 emit_int8(0x02);
4517 emit_int8((unsigned char)(0xC0 | encode));
4518 }
4519
4520 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4521 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4522 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4523 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4524 emit_int8((unsigned char)0xFC);
4525 emit_int8((unsigned char)(0xC0 | encode));
4526 }
4527
4528 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4529 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4530 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4531 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4532 emit_int8((unsigned char)0xFD);
4533 emit_int8((unsigned char)(0xC0 | encode));
4534 }
4535
4536 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
4537 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4538 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4539 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4540 emit_int8((unsigned char)0xFE);
4541 emit_int8((unsigned char)(0xC0 | encode));
4542 }
4543
4544 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
4545 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4546 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4547 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4548 emit_int8((unsigned char)0xD4);
4549 emit_int8((unsigned char)(0xC0 | encode));
4550 }
4551
4552 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
4553 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4554 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4555 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4556 emit_int8(0x01);
4557 emit_int8((unsigned char)(0xC0 | encode));
4558 }
4559
4560 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
4561 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4562 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
4563 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4564 emit_int8(0x02);
4565 emit_int8((unsigned char)(0xC0 | encode));
4566 }
4567
4568 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4569 assert(UseAVX > 0, "requires some form of AVX");
4570 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4571 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4572 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4573 emit_int8((unsigned char)0xFC);
4574 emit_int8((unsigned char)(0xC0 | encode));
4575 }
4576
4577 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4578 assert(UseAVX > 0, "requires some form of AVX");
4579 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4580 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4581 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4582 emit_int8((unsigned char)0xFD);
4583 emit_int8((unsigned char)(0xC0 | encode));
4584 }
4585
4586 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4587 assert(UseAVX > 0, "requires some form of AVX");
4588 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4589 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4590 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4591 emit_int8((unsigned char)0xFE);
4592 emit_int8((unsigned char)(0xC0 | encode));
4593 }
4594
4595 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4596 assert(UseAVX > 0, "requires some form of AVX");
4597 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4598 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4599 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4600 emit_int8((unsigned char)0xD4);
4601 emit_int8((unsigned char)(0xC0 | encode));
4602 }
4603
4604 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4605 assert(UseAVX > 0, "requires some form of AVX");
4606 InstructionMark im(this);
4607 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4608 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4609 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4610 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4611 emit_int8((unsigned char)0xFC);
4612 emit_operand(dst, src);
4613 }
4614
4615 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4616 assert(UseAVX > 0, "requires some form of AVX");
4617 InstructionMark im(this);
4618 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4619 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4620 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4621 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4622 emit_int8((unsigned char)0xFD);
4623 emit_operand(dst, src);
4624 }
4625
4626 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4627 assert(UseAVX > 0, "requires some form of AVX");
4628 InstructionMark im(this);
4629 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4630 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4631 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4632 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4633 emit_int8((unsigned char)0xFE);
4634 emit_operand(dst, src);
4635 }
4636
4637 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4638 assert(UseAVX > 0, "requires some form of AVX");
4639 InstructionMark im(this);
4640 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4641 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4642 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4643 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4644 emit_int8((unsigned char)0xD4);
4645 emit_operand(dst, src);
4646 }
4647
4648 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
4649 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4650 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4651 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4652 emit_int8((unsigned char)0xF8);
4653 emit_int8((unsigned char)(0xC0 | encode));
4654 }
4655
4656 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
4657 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4658 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4659 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4660 emit_int8((unsigned char)0xF9);
4661 emit_int8((unsigned char)(0xC0 | encode));
4662 }
4663
4664 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
4665 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4666 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4667 emit_int8((unsigned char)0xFA);
4668 emit_int8((unsigned char)(0xC0 | encode));
4669 }
4670
4671 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
4672 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4673 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4674 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4675 emit_int8((unsigned char)0xFB);
4676 emit_int8((unsigned char)(0xC0 | encode));
4677 }
4678
4679 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4680 assert(UseAVX > 0, "requires some form of AVX");
4681 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4682 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4683 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4684 emit_int8((unsigned char)0xF8);
4685 emit_int8((unsigned char)(0xC0 | encode));
4686 }
4687
4688 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4689 assert(UseAVX > 0, "requires some form of AVX");
4690 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4691 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4692 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4693 emit_int8((unsigned char)0xF9);
4694 emit_int8((unsigned char)(0xC0 | encode));
4695 }
4696
4697 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4698 assert(UseAVX > 0, "requires some form of AVX");
4699 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4700 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4701 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4702 emit_int8((unsigned char)0xFA);
4703 emit_int8((unsigned char)(0xC0 | encode));
4704 }
4705
4706 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4707 assert(UseAVX > 0, "requires some form of AVX");
4708 InstructionAttr attributes(vector_len, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4709 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4710 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4711 emit_int8((unsigned char)0xFB);
4712 emit_int8((unsigned char)(0xC0 | encode));
4713 }
4714
4715 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4716 assert(UseAVX > 0, "requires some form of AVX");
4717 InstructionMark im(this);
4718 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4719 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4720 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4721 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4722 emit_int8((unsigned char)0xF8);
4723 emit_operand(dst, src);
4724 }
4725
4726 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4727 assert(UseAVX > 0, "requires some form of AVX");
4728 InstructionMark im(this);
4729 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4730 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4731 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4732 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4733 emit_int8((unsigned char)0xF9);
4734 emit_operand(dst, src);
4735 }
4736
4737 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4738 assert(UseAVX > 0, "requires some form of AVX");
4739 InstructionMark im(this);
4740 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4741 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4742 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4743 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4744 emit_int8((unsigned char)0xFA);
4745 emit_operand(dst, src);
4746 }
4747
4748 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4749 assert(UseAVX > 0, "requires some form of AVX");
4750 InstructionMark im(this);
4751 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4752 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4753 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4754 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4755 emit_int8((unsigned char)0xFB);
4756 emit_operand(dst, src);
4757 }
4758
4759 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
4760 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4761 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4762 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4763 emit_int8((unsigned char)0xD5);
4764 emit_int8((unsigned char)(0xC0 | encode));
4765 }
4766
4767 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
4768 assert(VM_Version::supports_sse4_1(), "");
4769 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4770 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4771 emit_int8(0x40);
4772 emit_int8((unsigned char)(0xC0 | encode));
4773 }
4774
4775 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4776 assert(UseAVX > 0, "requires some form of AVX");
4777 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4778 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4779 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4780 emit_int8((unsigned char)0xD5);
4781 emit_int8((unsigned char)(0xC0 | encode));
4782 }
4783
4784 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4785 assert(UseAVX > 0, "requires some form of AVX");
4786 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4787 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4788 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4789 emit_int8(0x40);
4790 emit_int8((unsigned char)(0xC0 | encode));
4791 }
4792
4793 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4794 assert(UseAVX > 2, "requires some form of AVX");
4795 InstructionAttr attributes(vector_len, /* rex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4796 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4797 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4798 emit_int8(0x40);
4799 emit_int8((unsigned char)(0xC0 | encode));
4800 }
4801
4802 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4803 assert(UseAVX > 0, "requires some form of AVX");
4804 InstructionMark im(this);
4805 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4806 attributes.set_address_attributes(/* tuple_type */ EVEX_FVM, /* input_size_in_bits */ EVEX_NObit);
4807 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4808 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4809 emit_int8((unsigned char)0xD5);
4810 emit_operand(dst, src);
4811 }
4812
4813 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4814 assert(UseAVX > 0, "requires some form of AVX");
4815 InstructionMark im(this);
4816 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4817 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
4818 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4819 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4820 emit_int8(0x40);
4821 emit_operand(dst, src);
4822 }
4823
4824 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4825 assert(UseAVX > 0, "requires some form of AVX");
4826 InstructionMark im(this);
4827 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false, /* uses_vl */ true);
4828 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_64bit);
4829 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4830 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
4831 emit_int8(0x40);
4832 emit_operand(dst, src);
4833 }
4834
4835 // Shift packed integers left by specified number of bits.
4836 void Assembler::psllw(XMMRegister dst, int shift) {
4837 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4838 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4839 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4840 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4841 emit_int8(0x71);
4842 emit_int8((unsigned char)(0xC0 | encode));
4843 emit_int8(shift & 0xFF);
4844 }
4845
4846 void Assembler::pslld(XMMRegister dst, int shift) {
4847 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4848 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4849 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4850 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4851 emit_int8(0x72);
4852 emit_int8((unsigned char)(0xC0 | encode));
4853 emit_int8(shift & 0xFF);
4854 }
4855
4856 void Assembler::psllq(XMMRegister dst, int shift) {
4857 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4858 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4859 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4860 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4861 emit_int8(0x73);
4862 emit_int8((unsigned char)(0xC0 | encode));
4863 emit_int8(shift & 0xFF);
4864 }
4865
4866 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
4867 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4868 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4869 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4870 emit_int8((unsigned char)0xF1);
4871 emit_int8((unsigned char)(0xC0 | encode));
4872 }
4873
4874 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
4875 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4876 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4877 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4878 emit_int8((unsigned char)0xF2);
4879 emit_int8((unsigned char)(0xC0 | encode));
4880 }
4881
4882 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
4883 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4884 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4885 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4886 emit_int8((unsigned char)0xF3);
4887 emit_int8((unsigned char)(0xC0 | encode));
4888 }
4889
4890 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4891 assert(UseAVX > 0, "requires some form of AVX");
4892 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4893 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4894 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4895 emit_int8(0x71);
4896 emit_int8((unsigned char)(0xC0 | encode));
4897 emit_int8(shift & 0xFF);
4898 }
4899
4900 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4901 assert(UseAVX > 0, "requires some form of AVX");
4902 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4903 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4904 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4905 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4906 emit_int8(0x72);
4907 emit_int8((unsigned char)(0xC0 | encode));
4908 emit_int8(shift & 0xFF);
4909 }
4910
4911 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4912 assert(UseAVX > 0, "requires some form of AVX");
4913 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4914 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4915 int encode = vex_prefix_and_encode(xmm6->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4916 emit_int8(0x73);
4917 emit_int8((unsigned char)(0xC0 | encode));
4918 emit_int8(shift & 0xFF);
4919 }
4920
4921 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4922 assert(UseAVX > 0, "requires some form of AVX");
4923 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4924 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4925 emit_int8((unsigned char)0xF1);
4926 emit_int8((unsigned char)(0xC0 | encode));
4927 }
4928
4929 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4930 assert(UseAVX > 0, "requires some form of AVX");
4931 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4932 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4933 emit_int8((unsigned char)0xF2);
4934 emit_int8((unsigned char)(0xC0 | encode));
4935 }
4936
4937 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4938 assert(UseAVX > 0, "requires some form of AVX");
4939 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4940 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4941 emit_int8((unsigned char)0xF3);
4942 emit_int8((unsigned char)(0xC0 | encode));
4943 }
4944
4945 // Shift packed integers logically right by specified number of bits.
4946 void Assembler::psrlw(XMMRegister dst, int shift) {
4947 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4948 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4949 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4950 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4951 emit_int8(0x71);
4952 emit_int8((unsigned char)(0xC0 | encode));
4953 emit_int8(shift & 0xFF);
4954 }
4955
4956 void Assembler::psrld(XMMRegister dst, int shift) {
4957 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4958 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4959 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4960 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4961 emit_int8(0x72);
4962 emit_int8((unsigned char)(0xC0 | encode));
4963 emit_int8(shift & 0xFF);
4964 }
4965
4966 void Assembler::psrlq(XMMRegister dst, int shift) {
4967 // Do not confuse it with psrldq SSE2 instruction which
4968 // shifts 128 bit value in xmm register by number of bytes.
4969 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4970 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4971 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4972 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4973 emit_int8(0x73);
4974 emit_int8((unsigned char)(0xC0 | encode));
4975 emit_int8(shift & 0xFF);
4976 }
4977
4978 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
4979 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4980 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
4981 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4982 emit_int8((unsigned char)0xD1);
4983 emit_int8((unsigned char)(0xC0 | encode));
4984 }
4985
4986 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
4987 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4988 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4989 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4990 emit_int8((unsigned char)0xD2);
4991 emit_int8((unsigned char)(0xC0 | encode));
4992 }
4993
4994 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
4995 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4996 InstructionAttr attributes(AVX_128bit, /* rex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
4997 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
4998 emit_int8((unsigned char)0xD3);
4999 emit_int8((unsigned char)(0xC0 | encode));
5000 }
5001
5002 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5003 assert(UseAVX > 0, "requires some form of AVX");
5004 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5005 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
5006 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5007 emit_int8(0x71);
5008 emit_int8((unsigned char)(0xC0 | encode));
5009 emit_int8(shift & 0xFF);
5010 }
5011
5012 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5013 assert(UseAVX > 0, "requires some form of AVX");
5014 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5015 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
5016 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5017 emit_int8(0x72);
5018 emit_int8((unsigned char)(0xC0 | encode));
5019 emit_int8(shift & 0xFF);
5020 }
5021
5022 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5023 assert(UseAVX > 0, "requires some form of AVX");
5024 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5025 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
5026 int encode = vex_prefix_and_encode(xmm2->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5027 emit_int8(0x73);
5028 emit_int8((unsigned char)(0xC0 | encode));
5029 emit_int8(shift & 0xFF);
5030 }
5031
5032 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5033 assert(UseAVX > 0, "requires some form of AVX");
5034 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5035 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5036 emit_int8((unsigned char)0xD1);
5037 emit_int8((unsigned char)(0xC0 | encode));
5038 }
5039
5040 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5041 assert(UseAVX > 0, "requires some form of AVX");
5042 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5043 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5044 emit_int8((unsigned char)0xD2);
5045 emit_int8((unsigned char)(0xC0 | encode));
5046 }
5047
5048 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5049 assert(UseAVX > 0, "requires some form of AVX");
5050 InstructionAttr attributes(vector_len, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5051 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5052 emit_int8((unsigned char)0xD3);
5053 emit_int8((unsigned char)(0xC0 | encode));
5054 }
5055
5056 // Shift packed integers arithmetically right by specified number of bits.
5057 void Assembler::psraw(XMMRegister dst, int shift) {
5058 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5059 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5060 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5061 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5062 emit_int8(0x71);
5063 emit_int8((unsigned char)(0xC0 | encode));
5064 emit_int8(shift & 0xFF);
5065 }
5066
5067 void Assembler::psrad(XMMRegister dst, int shift) {
5068 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5069 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5070 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
5071 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5072 emit_int8(0x72);
5073 emit_int8((unsigned char)(0xC0 | encode));
5074 emit_int8(shift & 0xFF);
5075 }
5076
5077 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
5078 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5079 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5080 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5081 emit_int8((unsigned char)0xE1);
5082 emit_int8((unsigned char)(0xC0 | encode));
5083 }
5084
5085 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
5086 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5087 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5088 int encode = simd_prefix_and_encode(dst, dst, shift, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5089 emit_int8((unsigned char)0xE2);
5090 emit_int8((unsigned char)(0xC0 | encode));
5091 }
5092
5093 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5094 assert(UseAVX > 0, "requires some form of AVX");
5095 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5096 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5097 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5098 emit_int8(0x71);
5099 emit_int8((unsigned char)(0xC0 | encode));
5100 emit_int8(shift & 0xFF);
5101 }
5102
5103 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
5104 assert(UseAVX > 0, "requires some form of AVX");
5105 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5106 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
5107 int encode = vex_prefix_and_encode(xmm4->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5108 emit_int8(0x72);
5109 emit_int8((unsigned char)(0xC0 | encode));
5110 emit_int8(shift & 0xFF);
5111 }
5112
5113 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5114 assert(UseAVX > 0, "requires some form of AVX");
5115 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5116 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5117 emit_int8((unsigned char)0xE1);
5118 emit_int8((unsigned char)(0xC0 | encode));
5119 }
5120
5121 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
5122 assert(UseAVX > 0, "requires some form of AVX");
5123 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5124 int encode = vex_prefix_and_encode(dst->encoding(), src->encoding(), shift->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5125 emit_int8((unsigned char)0xE2);
5126 emit_int8((unsigned char)(0xC0 | encode));
5127 }
5128
5129
5130 // logical operations packed integers
5131 void Assembler::pand(XMMRegister dst, XMMRegister src) {
5132 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5133 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5134 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5135 emit_int8((unsigned char)0xDB);
5136 emit_int8((unsigned char)(0xC0 | encode));
5137 }
5138
5139 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5140 assert(UseAVX > 0, "requires some form of AVX");
5141 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5142 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5143 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5144 emit_int8((unsigned char)0xDB);
5145 emit_int8((unsigned char)(0xC0 | encode));
5146 }
5147
5148 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5149 assert(UseAVX > 0, "requires some form of AVX");
5150 InstructionMark im(this);
5151 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5152 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5153 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5154 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5155 emit_int8((unsigned char)0xDB);
5156 emit_operand(dst, src);
5157 }
5158
5159 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
5160 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5161 InstructionAttr attributes(AVX_128bit, /* vex_w */ VM_Version::supports_evex(), /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5162 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5163 emit_int8((unsigned char)0xDF);
5164 emit_int8((unsigned char)(0xC0 | encode));
5165 }
5166
5167 void Assembler::por(XMMRegister dst, XMMRegister src) {
5168 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5169 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5170 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5171 emit_int8((unsigned char)0xEB);
5172 emit_int8((unsigned char)(0xC0 | encode));
5173 }
5174
5175 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5176 assert(UseAVX > 0, "requires some form of AVX");
5177 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5178 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5179 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5180 emit_int8((unsigned char)0xEB);
5181 emit_int8((unsigned char)(0xC0 | encode));
5182 }
5183
5184 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5185 assert(UseAVX > 0, "requires some form of AVX");
5186 InstructionMark im(this);
5187 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5188 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5189 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5190 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5191 emit_int8((unsigned char)0xEB);
5192 emit_operand(dst, src);
5193 }
5194
5195 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
5196 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5197 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5198 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5199 emit_int8((unsigned char)0xEF);
5200 emit_int8((unsigned char)(0xC0 | encode));
5201 }
5202
5203 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
5204 assert(UseAVX > 0, "requires some form of AVX");
5205 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5206 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5207 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5208 emit_int8((unsigned char)0xEF);
5209 emit_int8((unsigned char)(0xC0 | encode));
5210 }
5211
5212 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
5213 assert(UseAVX > 0, "requires some form of AVX");
5214 InstructionMark im(this);
5215 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5216 attributes.set_address_attributes(/* tuple_type */ EVEX_FV, /* input_size_in_bits */ EVEX_32bit);
5217 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5218 vex_prefix(src, nds_enc, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
5219 emit_int8((unsigned char)0xEF);
5220 emit_operand(dst, src);
5221 }
5222
5223
5224 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5225 assert(VM_Version::supports_avx(), "");
5226 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5227 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5228 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5229 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5230 emit_int8(0x18);
5231 emit_int8((unsigned char)(0xC0 | encode));
5232 // 0x00 - insert into lower 128 bits
5233 // 0x01 - insert into upper 128 bits
5234 emit_int8(0x01);
5235 }
5236
5237 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5238 assert(VM_Version::supports_evex(), "");
5239 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5240 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5241 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5242 emit_int8(0x1A);
5243 emit_int8((unsigned char)(0xC0 | encode));
5244 // 0x00 - insert into lower 256 bits
5245 // 0x01 - insert into upper 256 bits
5246 emit_int8(value & 0x01);
5247 }
5248
5249 void Assembler::vinsertf64x4h(XMMRegister dst, Address src, int value) {
5250 assert(VM_Version::supports_evex(), "");
5251 assert(dst != xnoreg, "sanity");
5252 InstructionMark im(this);
5253 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5254 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_64bit);
5255 // swap src<->dst for encoding
5256 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5257 emit_int8(0x1A);
5258 emit_operand(dst, src);
5259 // 0x00 - insert into lower 256 bits
5260 // 0x01 - insert into upper 128 bits
5261 emit_int8(value & 0x01);
5262 }
5263
5264 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5265 assert(VM_Version::supports_evex(), "");
5266 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5267 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5268 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5269 emit_int8(0x18);
5270 emit_int8((unsigned char)(0xC0 | encode));
5271 // 0x00 - insert into q0 128 bits (0..127)
5272 // 0x01 - insert into q1 128 bits (128..255)
5273 // 0x02 - insert into q2 128 bits (256..383)
5274 // 0x03 - insert into q3 128 bits (384..511)
5275 emit_int8(value & 0x3);
5276 }
5277
5278 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
5279 assert(VM_Version::supports_avx(), "");
5280 assert(dst != xnoreg, "sanity");
5281 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5282 InstructionMark im(this);
5283 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5284 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5285 // swap src<->dst for encoding
5286 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5287 emit_int8(0x18);
5288 emit_operand(dst, src);
5289 // 0x00 - insert into q0 128 bits (0..127)
5290 // 0x01 - insert into q1 128 bits (128..255)
5291 // 0x02 - insert into q2 128 bits (256..383)
5292 // 0x03 - insert into q3 128 bits (384..511)
5293 emit_int8(value & 0x3);
5294 }
5295
5296 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
5297 assert(VM_Version::supports_avx(), "");
5298 assert(dst != xnoreg, "sanity");
5299 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5300 InstructionMark im(this);
5301 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5302 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5303 // swap src<->dst for encoding
5304 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5305 emit_int8(0x18);
5306 emit_operand(dst, src);
5307 // 0x01 - insert into upper 128 bits
5308 emit_int8(0x01);
5309 }
5310
5311 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
5312 assert(VM_Version::supports_avx(), "");
5313 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5314 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5315 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5316 emit_int8(0x19);
5317 emit_int8((unsigned char)(0xC0 | encode));
5318 // 0x00 - insert into lower 128 bits
5319 // 0x01 - insert into upper 128 bits
5320 emit_int8(0x01);
5321 }
5322
5323 void Assembler::vextractf128h(Address dst, XMMRegister src) {
5324 assert(VM_Version::supports_avx(), "");
5325 assert(src != xnoreg, "sanity");
5326 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5327 InstructionMark im(this);
5328 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5329 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5330 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5331 emit_int8(0x19);
5332 emit_operand(src, dst);
5333 // 0x01 - extract from upper 128 bits
5334 emit_int8(0x01);
5335 }
5336
5337 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5338 assert(VM_Version::supports_avx2(), "");
5339 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5340 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5341 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5342 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5343 emit_int8(0x38);
5344 emit_int8((unsigned char)(0xC0 | encode));
5345 // 0x00 - insert into lower 128 bits
5346 // 0x01 - insert into upper 128 bits
5347 emit_int8(0x01);
5348 }
5349
5350 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5351 assert(VM_Version::supports_evex(), "");
5352 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5353 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5354 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5355 emit_int8(0x38);
5356 emit_int8((unsigned char)(0xC0 | encode));
5357 // 0x00 - insert into lower 256 bits
5358 // 0x01 - insert into upper 256 bits
5359 emit_int8(value & 0x01);
5360 }
5361
5362 void Assembler::vinserti128h(XMMRegister dst, Address src) {
5363 assert(VM_Version::supports_avx2(), "");
5364 assert(dst != xnoreg, "sanity");
5365 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5366 InstructionMark im(this);
5367 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5368 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5369 // swap src<->dst for encoding
5370 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5371 emit_int8(0x38);
5372 emit_operand(dst, src);
5373 // 0x01 - insert into upper 128 bits
5374 emit_int8(0x01);
5375 }
5376
5377 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
5378 assert(VM_Version::supports_avx(), "");
5379 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5380 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5381 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5382 emit_int8(0x39);
5383 emit_int8((unsigned char)(0xC0 | encode));
5384 // 0x00 - insert into lower 128 bits
5385 // 0x01 - insert into upper 128 bits
5386 emit_int8(0x01);
5387 }
5388
5389 void Assembler::vextracti128h(Address dst, XMMRegister src) {
5390 assert(VM_Version::supports_avx2(), "");
5391 assert(src != xnoreg, "sanity");
5392 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5393 InstructionMark im(this);
5394 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5395 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5396 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5397 emit_int8(0x39);
5398 emit_operand(src, dst);
5399 // 0x01 - extract from upper 128 bits
5400 emit_int8(0x01);
5401 }
5402
5403 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src, int value) {
5404 assert(VM_Version::supports_evex(), "");
5405 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5406 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5407 emit_int8(0x3B);
5408 emit_int8((unsigned char)(0xC0 | encode));
5409 // 0x00 - extract from lower 256 bits
5410 // 0x01 - extract from upper 256 bits
5411 emit_int8(value & 0x01);
5412 }
5413
5414 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
5415 assert(VM_Version::supports_evex(), "");
5416 InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5417 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5418 emit_int8(0x39);
5419 emit_int8((unsigned char)(0xC0 | encode));
5420 // 0x01 - extract from bits 255:128
5421 // 0x02 - extract from bits 383:256
5422 // 0x03 - extract from bits 511:384
5423 emit_int8(value & 0x3);
5424 }
5425
5426 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src, int value) {
5427 assert(VM_Version::supports_evex(), "");
5428 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5429 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5430 emit_int8(0x1B);
5431 emit_int8((unsigned char)(0xC0 | encode));
5432 // 0x00 - extract from lower 256 bits
5433 // 0x01 - extract from upper 256 bits
5434 emit_int8(value & 0x1);
5435 }
5436
5437 void Assembler::vextractf64x4h(Address dst, XMMRegister src, int value) {
5438 assert(VM_Version::supports_evex(), "");
5439 assert(src != xnoreg, "sanity");
5440 InstructionMark im(this);
5441 InstructionAttr attributes(AVX_512bit, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5442 attributes.set_address_attributes(/* tuple_type */ EVEX_T4,/* input_size_in_bits */ EVEX_64bit);
5443 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5444 emit_int8(0x1B);
5445 emit_operand(src, dst);
5446 // 0x00 - extract from lower 256 bits
5447 // 0x01 - extract from upper 256 bits
5448 emit_int8(value & 0x01);
5449 }
5450
5451 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
5452 assert(VM_Version::supports_avx(), "");
5453 int vector_len = VM_Version::supports_evex() ? AVX_512bit : AVX_256bit;
5454 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5455 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5456 emit_int8(0x19);
5457 emit_int8((unsigned char)(0xC0 | encode));
5458 // 0x00 - extract from bits 127:0
5459 // 0x01 - extract from bits 255:128
5460 // 0x02 - extract from bits 383:256
5461 // 0x03 - extract from bits 511:384
5462 emit_int8(value & 0x3);
5463 }
5464
5465 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
5466 assert(VM_Version::supports_evex(), "");
5467 assert(src != xnoreg, "sanity");
5468 InstructionMark im(this);
5469 InstructionAttr attributes(AVX_512bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5470 attributes.set_address_attributes(/* tuple_type */ EVEX_T4, /* input_size_in_bits */ EVEX_32bit);
5471 vex_prefix(dst, 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5472 emit_int8(0x19);
5473 emit_operand(src, dst);
5474 // 0x00 - extract from bits 127:0
5475 // 0x01 - extract from bits 255:128
5476 // 0x02 - extract from bits 383:256
5477 // 0x03 - extract from bits 511:384
5478 emit_int8(value & 0x3);
5479 }
5480
5481 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
5482 assert(VM_Version::supports_evex(), "");
5483 InstructionAttr attributes(AVX_512bit, /* vex_w */ !_legacy_mode_dq, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ false);
5484 int encode = vex_prefix_and_encode(src->encoding(), 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5485 emit_int8(0x19);
5486 emit_int8((unsigned char)(0xC0 | encode));
5487 // 0x01 - extract from bits 255:128
5488 // 0x02 - extract from bits 383:256
5489 // 0x03 - extract from bits 511:384
5490 emit_int8(value & 0x3);
5491 }
5492
5493 // duplicate 4-bytes integer data from src into 8 locations in dest
5494 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
5495 assert(VM_Version::supports_avx2(), "");
5496 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5497 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5498 emit_int8(0x58);
5499 emit_int8((unsigned char)(0xC0 | encode));
5500 }
5501
5502 // duplicate 2-bytes integer data from src into 16 locations in dest
5503 void Assembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
5504 assert(VM_Version::supports_avx2(), "");
5505 InstructionAttr attributes(AVX_256bit, /* vex_w */ false, /* legacy_mode */ _legacy_mode_bw, /* no_mask_reg */ false, /* uses_vl */ true);
5506 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5507 emit_int8(0x79);
5508 emit_int8((unsigned char)(0xC0 | encode));
5509 }
5510
5511 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5512 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
5513 assert(VM_Version::supports_evex(), "");
5514 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5515 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5516 emit_int8(0x78);
5517 emit_int8((unsigned char)(0xC0 | encode));
5518 }
5519
5520 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
5521 assert(VM_Version::supports_evex(), "");
5522 assert(dst != xnoreg, "sanity");
5523 InstructionMark im(this);
5524 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5525 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_8bit);
5526 // swap src<->dst for encoding
5527 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5528 emit_int8(0x78);
5529 emit_operand(dst, src);
5530 }
5531
5532 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5533 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5534 assert(VM_Version::supports_evex(), "");
5535 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5536 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5537 emit_int8(0x79);
5538 emit_int8((unsigned char)(0xC0 | encode));
5539 }
5540
5541 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5542 assert(VM_Version::supports_evex(), "");
5543 assert(dst != xnoreg, "sanity");
5544 InstructionMark im(this);
5545 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5546 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_16bit);
5547 // swap src<->dst for encoding
5548 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5549 emit_int8(0x79);
5550 emit_operand(dst, src);
5551 }
5552
5553 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5554 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5555 assert(VM_Version::supports_evex(), "");
5556 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5557 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5558 emit_int8(0x58);
5559 emit_int8((unsigned char)(0xC0 | encode));
5560 }
5561
5562 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5563 assert(VM_Version::supports_evex(), "");
5564 assert(dst != xnoreg, "sanity");
5565 InstructionMark im(this);
5566 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5567 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5568 // swap src<->dst for encoding
5569 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5570 emit_int8(0x58);
5571 emit_operand(dst, src);
5572 }
5573
5574 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5575 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5576 assert(VM_Version::supports_evex(), "");
5577 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5578 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5579 emit_int8(0x59);
5580 emit_int8((unsigned char)(0xC0 | encode));
5581 }
5582
5583 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5584 assert(VM_Version::supports_evex(), "");
5585 assert(dst != xnoreg, "sanity");
5586 InstructionMark im(this);
5587 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5588 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5589 // swap src<->dst for encoding
5590 vex_prefix(src, dst->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5591 emit_int8(0x59);
5592 emit_operand(dst, src);
5593 }
5594
5595 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5596 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5597 assert(VM_Version::supports_evex(), "");
5598 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5599 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5600 emit_int8(0x18);
5601 emit_int8((unsigned char)(0xC0 | encode));
5602 }
5603
5604 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5605 assert(VM_Version::supports_evex(), "");
5606 assert(dst != xnoreg, "sanity");
5607 InstructionMark im(this);
5608 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5609 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_32bit);
5610 // swap src<->dst for encoding
5611 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5612 emit_int8(0x18);
5613 emit_operand(dst, src);
5614 }
5615
5616 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5617 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5618 assert(VM_Version::supports_evex(), "");
5619 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5620 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5621 emit_int8(0x19);
5622 emit_int8((unsigned char)(0xC0 | encode));
5623 }
5624
5625 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5626 assert(VM_Version::supports_evex(), "");
5627 assert(dst != xnoreg, "sanity");
5628 InstructionMark im(this);
5629 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5630 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
5631 // swap src<->dst for encoding
5632 vex_prefix(src, 0, dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5633 emit_int8(0x19);
5634 emit_operand(dst, src);
5635 }
5636
5637 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5638 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
5639 assert(VM_Version::supports_evex(), "");
5640 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5641 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5642 emit_int8(0x7A);
5643 emit_int8((unsigned char)(0xC0 | encode));
5644 }
5645
5646 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5647 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
5648 assert(VM_Version::supports_evex(), "");
5649 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5650 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5651 emit_int8(0x7B);
5652 emit_int8((unsigned char)(0xC0 | encode));
5653 }
5654
5655 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5656 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
5657 assert(VM_Version::supports_evex(), "");
5658 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5659 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5660 emit_int8(0x7C);
5661 emit_int8((unsigned char)(0xC0 | encode));
5662 }
5663
5664 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5665 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
5666 assert(VM_Version::supports_evex(), "");
5667 InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ false, /* uses_vl */ true);
5668 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
5669 emit_int8(0x7C);
5670 emit_int8((unsigned char)(0xC0 | encode));
5671 }
5672
5673 // Carry-Less Multiplication Quadword
5674 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5675 assert(VM_Version::supports_clmul(), "");
5676 InstructionAttr attributes(AVX_128bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5677 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5678 emit_int8(0x44);
5679 emit_int8((unsigned char)(0xC0 | encode));
5680 emit_int8((unsigned char)mask);
5681 }
5682
5683 // Carry-Less Multiplication Quadword
5684 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5685 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5686 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5687 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5688 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
5689 emit_int8(0x44);
5690 emit_int8((unsigned char)(0xC0 | encode));
5691 emit_int8((unsigned char)mask);
5692 }
5693
5694 void Assembler::vzeroupper() {
5695 assert(VM_Version::supports_avx(), "");
5696 InstructionAttr attributes(AVX_128bit, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
5697 (void)vex_prefix_and_encode(0, 0, 0, VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
5698 emit_int8(0x77);
5699 }
5700
5701
5702 #ifndef _LP64
5703 // 32bit only pieces of the assembler
5704
5705 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5706 // NO PREFIX AS NEVER 64BIT
5707 InstructionMark im(this);
5708 emit_int8((unsigned char)0x81);
5709 emit_int8((unsigned char)(0xF8 | src1->encoding()));
5710 emit_data(imm32, rspec, 0);
5711 }
5712
5713 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5714 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
5715 InstructionMark im(this);
5716 emit_int8((unsigned char)0x81);
5717 emit_operand(rdi, src1);
5718 emit_data(imm32, rspec, 0);
5719 }
5720
5721 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
5722 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
5723 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
5724 void Assembler::cmpxchg8(Address adr) {
5725 InstructionMark im(this);
5726 emit_int8(0x0F);
5727 emit_int8((unsigned char)0xC7);
5728 emit_operand(rcx, adr);
5729 }
5730
5731 void Assembler::decl(Register dst) {
5732 // Don't use it directly. Use MacroAssembler::decrementl() instead.
5733 emit_int8(0x48 | dst->encoding());
5734 }
5735
5736 #endif // _LP64
5737
5738 // 64bit typically doesn't use the x87 but needs to for the trig funcs
5739
5740 void Assembler::fabs() {
5741 emit_int8((unsigned char)0xD9);
5742 emit_int8((unsigned char)0xE1);
5743 }
5744
5745 void Assembler::fadd(int i) {
5746 emit_farith(0xD8, 0xC0, i);
5747 }
5748
5749 void Assembler::fadd_d(Address src) {
5750 InstructionMark im(this);
5751 emit_int8((unsigned char)0xDC);
5752 emit_operand32(rax, src);
5753 }
5754
5755 void Assembler::fadd_s(Address src) {
5756 InstructionMark im(this);
5757 emit_int8((unsigned char)0xD8);
5758 emit_operand32(rax, src);
5759 }
5760
5761 void Assembler::fadda(int i) {
5762 emit_farith(0xDC, 0xC0, i);
5763 }
5764
5765 void Assembler::faddp(int i) {
5766 emit_farith(0xDE, 0xC0, i);
5767 }
5768
5769 void Assembler::fchs() {
5770 emit_int8((unsigned char)0xD9);
5771 emit_int8((unsigned char)0xE0);
5772 }
5773
5774 void Assembler::fcom(int i) {
5775 emit_farith(0xD8, 0xD0, i);
5776 }
5777
5778 void Assembler::fcomp(int i) {
5779 emit_farith(0xD8, 0xD8, i);
5780 }
5781
5782 void Assembler::fcomp_d(Address src) {
5783 InstructionMark im(this);
5784 emit_int8((unsigned char)0xDC);
5785 emit_operand32(rbx, src);
5786 }
5787
5788 void Assembler::fcomp_s(Address src) {
5789 InstructionMark im(this);
5790 emit_int8((unsigned char)0xD8);
5791 emit_operand32(rbx, src);
5792 }
5793
5794 void Assembler::fcompp() {
5795 emit_int8((unsigned char)0xDE);
5796 emit_int8((unsigned char)0xD9);
5797 }
5798
5799 void Assembler::fcos() {
5800 emit_int8((unsigned char)0xD9);
5801 emit_int8((unsigned char)0xFF);
5802 }
5803
5804 void Assembler::fdecstp() {
5805 emit_int8((unsigned char)0xD9);
5806 emit_int8((unsigned char)0xF6);
5807 }
5808
5809 void Assembler::fdiv(int i) {
5810 emit_farith(0xD8, 0xF0, i);
5811 }
5812
5813 void Assembler::fdiv_d(Address src) {
5814 InstructionMark im(this);
5815 emit_int8((unsigned char)0xDC);
5816 emit_operand32(rsi, src);
5817 }
5818
5819 void Assembler::fdiv_s(Address src) {
5820 InstructionMark im(this);
5821 emit_int8((unsigned char)0xD8);
5822 emit_operand32(rsi, src);
5823 }
5824
5825 void Assembler::fdiva(int i) {
5826 emit_farith(0xDC, 0xF8, i);
5827 }
5828
5829 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
5830 // is erroneous for some of the floating-point instructions below.
5831
5832 void Assembler::fdivp(int i) {
5833 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
5834 }
5835
5836 void Assembler::fdivr(int i) {
5837 emit_farith(0xD8, 0xF8, i);
5838 }
5839
5840 void Assembler::fdivr_d(Address src) {
5841 InstructionMark im(this);
5842 emit_int8((unsigned char)0xDC);
5843 emit_operand32(rdi, src);
5844 }
5845
5846 void Assembler::fdivr_s(Address src) {
5847 InstructionMark im(this);
5848 emit_int8((unsigned char)0xD8);
5849 emit_operand32(rdi, src);
5850 }
5851
5852 void Assembler::fdivra(int i) {
5853 emit_farith(0xDC, 0xF0, i);
5854 }
5855
5856 void Assembler::fdivrp(int i) {
5857 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
5858 }
5859
5860 void Assembler::ffree(int i) {
5861 emit_farith(0xDD, 0xC0, i);
5862 }
5863
5864 void Assembler::fild_d(Address adr) {
5865 InstructionMark im(this);
5866 emit_int8((unsigned char)0xDF);
5867 emit_operand32(rbp, adr);
5868 }
5869
5870 void Assembler::fild_s(Address adr) {
5871 InstructionMark im(this);
5872 emit_int8((unsigned char)0xDB);
5873 emit_operand32(rax, adr);
5874 }
5875
5876 void Assembler::fincstp() {
5877 emit_int8((unsigned char)0xD9);
5878 emit_int8((unsigned char)0xF7);
5879 }
5880
5881 void Assembler::finit() {
5882 emit_int8((unsigned char)0x9B);
5883 emit_int8((unsigned char)0xDB);
5884 emit_int8((unsigned char)0xE3);
5885 }
5886
5887 void Assembler::fist_s(Address adr) {
5888 InstructionMark im(this);
5889 emit_int8((unsigned char)0xDB);
5890 emit_operand32(rdx, adr);
5891 }
5892
5893 void Assembler::fistp_d(Address adr) {
5894 InstructionMark im(this);
5895 emit_int8((unsigned char)0xDF);
5896 emit_operand32(rdi, adr);
5897 }
5898
5899 void Assembler::fistp_s(Address adr) {
5900 InstructionMark im(this);
5901 emit_int8((unsigned char)0xDB);
5902 emit_operand32(rbx, adr);
5903 }
5904
5905 void Assembler::fld1() {
5906 emit_int8((unsigned char)0xD9);
5907 emit_int8((unsigned char)0xE8);
5908 }
5909
5910 void Assembler::fld_d(Address adr) {
5911 InstructionMark im(this);
5912 emit_int8((unsigned char)0xDD);
5913 emit_operand32(rax, adr);
5914 }
5915
5916 void Assembler::fld_s(Address adr) {
5917 InstructionMark im(this);
5918 emit_int8((unsigned char)0xD9);
5919 emit_operand32(rax, adr);
5920 }
5921
5922
5923 void Assembler::fld_s(int index) {
5924 emit_farith(0xD9, 0xC0, index);
5925 }
5926
5927 void Assembler::fld_x(Address adr) {
5928 InstructionMark im(this);
5929 emit_int8((unsigned char)0xDB);
5930 emit_operand32(rbp, adr);
5931 }
5932
5933 void Assembler::fldcw(Address src) {
5934 InstructionMark im(this);
5935 emit_int8((unsigned char)0xD9);
5936 emit_operand32(rbp, src);
5937 }
5938
5939 void Assembler::fldenv(Address src) {
5940 InstructionMark im(this);
5941 emit_int8((unsigned char)0xD9);
5942 emit_operand32(rsp, src);
5943 }
5944
5945 void Assembler::fldlg2() {
5946 emit_int8((unsigned char)0xD9);
5947 emit_int8((unsigned char)0xEC);
5948 }
5949
5950 void Assembler::fldln2() {
5951 emit_int8((unsigned char)0xD9);
5952 emit_int8((unsigned char)0xED);
5953 }
5954
5955 void Assembler::fldz() {
5956 emit_int8((unsigned char)0xD9);
5957 emit_int8((unsigned char)0xEE);
5958 }
5959
5960 void Assembler::flog() {
5961 fldln2();
5962 fxch();
5963 fyl2x();
5964 }
5965
5966 void Assembler::flog10() {
5967 fldlg2();
5968 fxch();
5969 fyl2x();
5970 }
5971
5972 void Assembler::fmul(int i) {
5973 emit_farith(0xD8, 0xC8, i);
5974 }
5975
5976 void Assembler::fmul_d(Address src) {
5977 InstructionMark im(this);
5978 emit_int8((unsigned char)0xDC);
5979 emit_operand32(rcx, src);
5980 }
5981
5982 void Assembler::fmul_s(Address src) {
5983 InstructionMark im(this);
5984 emit_int8((unsigned char)0xD8);
5985 emit_operand32(rcx, src);
5986 }
5987
5988 void Assembler::fmula(int i) {
5989 emit_farith(0xDC, 0xC8, i);
5990 }
5991
5992 void Assembler::fmulp(int i) {
5993 emit_farith(0xDE, 0xC8, i);
5994 }
5995
5996 void Assembler::fnsave(Address dst) {
5997 InstructionMark im(this);
5998 emit_int8((unsigned char)0xDD);
5999 emit_operand32(rsi, dst);
6000 }
6001
6002 void Assembler::fnstcw(Address src) {
6003 InstructionMark im(this);
6004 emit_int8((unsigned char)0x9B);
6005 emit_int8((unsigned char)0xD9);
6006 emit_operand32(rdi, src);
6007 }
6008
6009 void Assembler::fnstsw_ax() {
6010 emit_int8((unsigned char)0xDF);
6011 emit_int8((unsigned char)0xE0);
6012 }
6013
6014 void Assembler::fprem() {
6015 emit_int8((unsigned char)0xD9);
6016 emit_int8((unsigned char)0xF8);
6017 }
6018
6019 void Assembler::fprem1() {
6020 emit_int8((unsigned char)0xD9);
6021 emit_int8((unsigned char)0xF5);
6022 }
6023
6024 void Assembler::frstor(Address src) {
6025 InstructionMark im(this);
6026 emit_int8((unsigned char)0xDD);
6027 emit_operand32(rsp, src);
6028 }
6029
6030 void Assembler::fsin() {
6031 emit_int8((unsigned char)0xD9);
6032 emit_int8((unsigned char)0xFE);
6033 }
6034
6035 void Assembler::fsqrt() {
6036 emit_int8((unsigned char)0xD9);
6037 emit_int8((unsigned char)0xFA);
6038 }
6039
6040 void Assembler::fst_d(Address adr) {
6041 InstructionMark im(this);
6042 emit_int8((unsigned char)0xDD);
6043 emit_operand32(rdx, adr);
6044 }
6045
6046 void Assembler::fst_s(Address adr) {
6047 InstructionMark im(this);
6048 emit_int8((unsigned char)0xD9);
6049 emit_operand32(rdx, adr);
6050 }
6051
6052 void Assembler::fstp_d(Address adr) {
6053 InstructionMark im(this);
6054 emit_int8((unsigned char)0xDD);
6055 emit_operand32(rbx, adr);
6056 }
6057
6058 void Assembler::fstp_d(int index) {
6059 emit_farith(0xDD, 0xD8, index);
6060 }
6061
6062 void Assembler::fstp_s(Address adr) {
6063 InstructionMark im(this);
6064 emit_int8((unsigned char)0xD9);
6065 emit_operand32(rbx, adr);
6066 }
6067
6068 void Assembler::fstp_x(Address adr) {
6069 InstructionMark im(this);
6070 emit_int8((unsigned char)0xDB);
6071 emit_operand32(rdi, adr);
6072 }
6073
6074 void Assembler::fsub(int i) {
6075 emit_farith(0xD8, 0xE0, i);
6076 }
6077
6078 void Assembler::fsub_d(Address src) {
6079 InstructionMark im(this);
6080 emit_int8((unsigned char)0xDC);
6081 emit_operand32(rsp, src);
6082 }
6083
6084 void Assembler::fsub_s(Address src) {
6085 InstructionMark im(this);
6086 emit_int8((unsigned char)0xD8);
6087 emit_operand32(rsp, src);
6088 }
6089
6090 void Assembler::fsuba(int i) {
6091 emit_farith(0xDC, 0xE8, i);
6092 }
6093
6094 void Assembler::fsubp(int i) {
6095 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
6096 }
6097
6098 void Assembler::fsubr(int i) {
6099 emit_farith(0xD8, 0xE8, i);
6100 }
6101
6102 void Assembler::fsubr_d(Address src) {
6103 InstructionMark im(this);
6104 emit_int8((unsigned char)0xDC);
6105 emit_operand32(rbp, src);
6106 }
6107
6108 void Assembler::fsubr_s(Address src) {
6109 InstructionMark im(this);
6110 emit_int8((unsigned char)0xD8);
6111 emit_operand32(rbp, src);
6112 }
6113
6114 void Assembler::fsubra(int i) {
6115 emit_farith(0xDC, 0xE0, i);
6116 }
6117
6118 void Assembler::fsubrp(int i) {
6119 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
6120 }
6121
6122 void Assembler::ftan() {
6123 emit_int8((unsigned char)0xD9);
6124 emit_int8((unsigned char)0xF2);
6125 emit_int8((unsigned char)0xDD);
6126 emit_int8((unsigned char)0xD8);
6127 }
6128
6129 void Assembler::ftst() {
6130 emit_int8((unsigned char)0xD9);
6131 emit_int8((unsigned char)0xE4);
6132 }
6133
6134 void Assembler::fucomi(int i) {
6135 // make sure the instruction is supported (introduced for P6, together with cmov)
6136 guarantee(VM_Version::supports_cmov(), "illegal instruction");
6137 emit_farith(0xDB, 0xE8, i);
6138 }
6139
6140 void Assembler::fucomip(int i) {
6141 // make sure the instruction is supported (introduced for P6, together with cmov)
6142 guarantee(VM_Version::supports_cmov(), "illegal instruction");
6143 emit_farith(0xDF, 0xE8, i);
6144 }
6145
6146 void Assembler::fwait() {
6147 emit_int8((unsigned char)0x9B);
6148 }
6149
6150 void Assembler::fxch(int i) {
6151 emit_farith(0xD9, 0xC8, i);
6152 }
6153
6154 void Assembler::fyl2x() {
6155 emit_int8((unsigned char)0xD9);
6156 emit_int8((unsigned char)0xF1);
6157 }
6158
6159 void Assembler::frndint() {
6160 emit_int8((unsigned char)0xD9);
6161 emit_int8((unsigned char)0xFC);
6162 }
6163
6164 void Assembler::f2xm1() {
6165 emit_int8((unsigned char)0xD9);
6166 emit_int8((unsigned char)0xF0);
6167 }
6168
6169 void Assembler::fldl2e() {
6170 emit_int8((unsigned char)0xD9);
6171 emit_int8((unsigned char)0xEA);
6172 }
6173
6174 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6175 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6176 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6177 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
6178
6179 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6180 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6181 if (pre > 0) {
6182 emit_int8(simd_pre[pre]);
6183 }
6184 if (rex_w) {
6185 prefixq(adr, xreg);
6186 } else {
6187 prefix(adr, xreg);
6188 }
6189 if (opc > 0) {
6190 emit_int8(0x0F);
6191 int opc2 = simd_opc[opc];
6192 if (opc2 > 0) {
6193 emit_int8(opc2);
6194 }
6195 }
6196 }
6197
6198 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6199 if (pre > 0) {
6200 emit_int8(simd_pre[pre]);
6201 }
6202 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) : prefix_and_encode(dst_enc, src_enc);
6203 if (opc > 0) {
6204 emit_int8(0x0F);
6205 int opc2 = simd_opc[opc];
6206 if (opc2 > 0) {
6207 emit_int8(opc2);
6208 }
6209 }
6210 return encode;
6211 }
6212
6213
6214 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, int nds_enc, VexSimdPrefix pre, VexOpcode opc) {
6215 int vector_len = _attributes->get_vector_len();
6216 bool vex_w = _attributes->is_rex_vex_w();
6217 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6218 prefix(VEX_3bytes);
6219
6220 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6221 byte1 = (~byte1) & 0xE0;
6222 byte1 |= opc;
6223 emit_int8(byte1);
6224
6225 int byte2 = ((~nds_enc) & 0xf) << 3;
6226 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6227 emit_int8(byte2);
6228 } else {
6229 prefix(VEX_2bytes);
6230
6231 int byte1 = vex_r ? VEX_R : 0;
6232 byte1 = (~byte1) & 0x80;
6233 byte1 |= ((~nds_enc) & 0xf) << 3;
6234 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6235 emit_int8(byte1);
6236 }
6237 }
6238
6239 // This is a 4 byte encoding
6240 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool evex_r, bool evex_v, int nds_enc, VexSimdPrefix pre, VexOpcode opc){
6241 // EVEX 0x62 prefix
6242 prefix(EVEX_4bytes);
6243 bool vex_w = _attributes->is_rex_vex_w();
6244 int evex_encoding = (vex_w ? VEX_W : 0);
6245 // EVEX.b is not currently used for broadcast of single element or data rounding modes
6246 _attributes->set_evex_encoding(evex_encoding);
6247
6248 // P0: byte 2, initialized to RXBR`00mm
6249 // instead of not'd
6250 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6251 byte2 = (~byte2) & 0xF0;
6252 // confine opc opcode extensions in mm bits to lower two bits
6253 // of form {0F, 0F_38, 0F_3A}
6254 byte2 |= opc;
6255 emit_int8(byte2);
6256
6257 // P1: byte 3 as Wvvvv1pp
6258 int byte3 = ((~nds_enc) & 0xf) << 3;
6259 // p[10] is always 1
6260 byte3 |= EVEX_F;
6261 byte3 |= (vex_w & 1) << 7;
6262 // confine pre opcode extensions in pp bits to lower two bits
6263 // of form {66, F3, F2}
6264 byte3 |= pre;
6265 emit_int8(byte3);
6266
6267 // P2: byte 4 as zL'Lbv'aaa
6268 int byte4 = (_attributes->is_no_reg_mask()) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6269 // EVEX.v` for extending EVEX.vvvv or VIDX
6270 byte4 |= (evex_v ? 0: EVEX_V);
6271 // third EXEC.b for broadcast actions
6272 byte4 |= (_attributes->is_extended_context() ? EVEX_Rb : 0);
6273 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6274 byte4 |= ((_attributes->get_vector_len())& 0x3) << 5;
6275 // last is EVEX.z for zero/merge actions
6276 byte4 |= (_attributes->is_clear_context() ? EVEX_Z : 0);
6277 emit_int8(byte4);
6278 }
6279
6280 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6281 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6282 bool vex_b = adr.base_needs_rex();
6283 bool vex_x = adr.index_needs_rex();
6284 set_attributes(attributes);
6285 attributes->set_current_assembler(this);
6286
6287 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6288 if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
6289 switch (attributes->get_vector_len()) {
6290 case AVX_128bit:
6291 case AVX_256bit:
6292 attributes->set_is_legacy_mode();
6293 break;
6294 }
6295 }
6296
6297 if ((UseAVX > 2) && !attributes->is_legacy_mode())
6298 {
6299 bool evex_r = (xreg_enc >= 16);
6300 bool evex_v = (nds_enc >= 16);
6301 attributes->set_is_evex_instruction();
6302 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6303 } else {
6304 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6305 }
6306 }
6307
6308 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, InstructionAttr *attributes) {
6309 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6310 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6311 bool vex_x = false;
6312 set_attributes(attributes);
6313 attributes->set_current_assembler(this);
6314
6315 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6316 if ((UseAVX > 2) && _legacy_mode_vl && attributes->uses_vl()) {
6317 switch (attributes->get_vector_len()) {
6318 case AVX_128bit:
6319 case AVX_256bit:
6320 if ((dst_enc >= 16) | (nds_enc >= 16) | (src_enc >= 16)) {
6321 // up propagate arithmetic instructions to meet RA requirements
6322 attributes->set_vector_len(AVX_512bit);
6323 } else {
6324 attributes->set_is_legacy_mode();
6325 }
6326 break;
6327 }
6328 }
6329
6330 if ((UseAVX > 2) && !attributes->is_legacy_mode())
6331 {
6332 bool evex_r = (dst_enc >= 16);
6333 bool evex_v = (nds_enc >= 16);
6334 // can use vex_x as bank extender on rm encoding
6335 vex_x = (src_enc >= 16);
6336 attributes->set_is_evex_instruction();
6337 evex_prefix(vex_r, vex_b, vex_x, evex_r, evex_v, nds_enc, pre, opc);
6338 } else {
6339 vex_prefix(vex_r, vex_b, vex_x, nds_enc, pre, opc);
6340 }
6341
6342 // return modrm byte components for operands
6343 return (((dst_enc & 7) << 3) | (src_enc & 7));
6344 }
6345
6346
6347 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6348 VexOpcode opc, InstructionAttr *attributes) {
6349 if (UseAVX > 0) {
6350 int xreg_enc = xreg->encoding();
6351 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6352 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, attributes);
6353 } else {
6354 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6355 rex_prefix(adr, xreg, pre, opc, attributes->is_rex_vex_w());
6356 }
6357 }
6358
6359 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6360 VexOpcode opc, InstructionAttr *attributes) {
6361 int dst_enc = dst->encoding();
6362 int src_enc = src->encoding();
6363 if (UseAVX > 0) {
6364 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6365 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, attributes);
6366 } else {
6367 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6368 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, attributes->is_rex_vex_w());
6369 }
6370 }
6371
6372 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
6373 VexOpcode opc, InstructionAttr *attributes) {
6374 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6375 return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
6376 }
6377
6378 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
6379 VexOpcode opc, InstructionAttr *attributes) {
6380 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6381 return vex_prefix_and_encode(dst->encoding(), nds_enc, src->encoding(), pre, opc, attributes);
6382 }
6383
6384 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
6385 assert(VM_Version::supports_avx(), "");
6386 assert(!VM_Version::supports_evex(), "");
6387 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6388 int encode = simd_prefix_and_encode(dst, nds, src, VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
6389 emit_int8((unsigned char)0xC2);
6390 emit_int8((unsigned char)(0xC0 | encode));
6391 emit_int8((unsigned char)(0xF & cop));
6392 }
6393
6394 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
6395 assert(VM_Version::supports_avx(), "");
6396 assert(!VM_Version::supports_evex(), "");
6397 InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6398 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6399 int encode = vex_prefix_and_encode(dst->encoding(), nds_enc, src1->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_3A, &attributes);
6400 emit_int8((unsigned char)0x4B);
6401 emit_int8((unsigned char)(0xC0 | encode));
6402 int src2_enc = src2->encoding();
6403 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
6404 }
6405
6406
6407 #ifndef _LP64
6408
6409 void Assembler::incl(Register dst) {
6410 // Don't use it directly. Use MacroAssembler::incrementl() instead.
6411 emit_int8(0x40 | dst->encoding());
6412 }
6413
6414 void Assembler::lea(Register dst, Address src) {
6415 leal(dst, src);
6416 }
6417
6418 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
6419 InstructionMark im(this);
6420 emit_int8((unsigned char)0xC7);
6421 emit_operand(rax, dst);
6422 emit_data((int)imm32, rspec, 0);
6423 }
6424
6425 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
6426 InstructionMark im(this);
6427 int encode = prefix_and_encode(dst->encoding());
6428 emit_int8((unsigned char)(0xB8 | encode));
6429 emit_data((int)imm32, rspec, 0);
6430 }
6431
6432 void Assembler::popa() { // 32bit
6433 emit_int8(0x61);
6434 }
6435
6436 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
6437 InstructionMark im(this);
6438 emit_int8(0x68);
6439 emit_data(imm32, rspec, 0);
6440 }
6441
6442 void Assembler::pusha() { // 32bit
6443 emit_int8(0x60);
6444 }
6445
6446 void Assembler::set_byte_if_not_zero(Register dst) {
6447 emit_int8(0x0F);
6448 emit_int8((unsigned char)0x95);
6449 emit_int8((unsigned char)(0xE0 | dst->encoding()));
6450 }
6451
6452 void Assembler::shldl(Register dst, Register src) {
6453 emit_int8(0x0F);
6454 emit_int8((unsigned char)0xA5);
6455 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6456 }
6457
6458 // 0F A4 / r ib
6459 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
6460 emit_int8(0x0F);
6461 emit_int8((unsigned char)0xA4);
6462 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6463 emit_int8(imm8);
6464 }
6465
6466 void Assembler::shrdl(Register dst, Register src) {
6467 emit_int8(0x0F);
6468 emit_int8((unsigned char)0xAD);
6469 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6470 }
6471
6472 #else // LP64
6473
6474 void Assembler::set_byte_if_not_zero(Register dst) {
6475 int enc = prefix_and_encode(dst->encoding(), true);
6476 emit_int8(0x0F);
6477 emit_int8((unsigned char)0x95);
6478 emit_int8((unsigned char)(0xE0 | enc));
6479 }
6480
6481 // 64bit only pieces of the assembler
6482 // This should only be used by 64bit instructions that can use rip-relative
6483 // it cannot be used by instructions that want an immediate value.
6484
6485 bool Assembler::reachable(AddressLiteral adr) {
6486 int64_t disp;
6487 // None will force a 64bit literal to the code stream. Likely a placeholder
6488 // for something that will be patched later and we need to certain it will
6489 // always be reachable.
6490 if (adr.reloc() == relocInfo::none) {
6491 return false;
6492 }
6493 if (adr.reloc() == relocInfo::internal_word_type) {
6494 // This should be rip relative and easily reachable.
6495 return true;
6496 }
6497 if (adr.reloc() == relocInfo::virtual_call_type ||
6498 adr.reloc() == relocInfo::opt_virtual_call_type ||
6499 adr.reloc() == relocInfo::static_call_type ||
6500 adr.reloc() == relocInfo::static_stub_type ) {
6501 // This should be rip relative within the code cache and easily
6502 // reachable until we get huge code caches. (At which point
6503 // ic code is going to have issues).
6504 return true;
6505 }
6506 if (adr.reloc() != relocInfo::external_word_type &&
6507 adr.reloc() != relocInfo::poll_return_type && // these are really external_word but need special
6508 adr.reloc() != relocInfo::poll_type && // relocs to identify them
6509 adr.reloc() != relocInfo::runtime_call_type ) {
6510 return false;
6511 }
6512
6513 // Stress the correction code
6514 if (ForceUnreachable) {
6515 // Must be runtimecall reloc, see if it is in the codecache
6516 // Flipping stuff in the codecache to be unreachable causes issues
6517 // with things like inline caches where the additional instructions
6518 // are not handled.
6519 if (CodeCache::find_blob(adr._target) == NULL) {
6520 return false;
6521 }
6522 }
6523 // For external_word_type/runtime_call_type if it is reachable from where we
6524 // are now (possibly a temp buffer) and where we might end up
6525 // anywhere in the codeCache then we are always reachable.
6526 // This would have to change if we ever save/restore shared code
6527 // to be more pessimistic.
6528 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
6529 if (!is_simm32(disp)) return false;
6530 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
6531 if (!is_simm32(disp)) return false;
6532
6533 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
6534
6535 // Because rip relative is a disp + address_of_next_instruction and we
6536 // don't know the value of address_of_next_instruction we apply a fudge factor
6537 // to make sure we will be ok no matter the size of the instruction we get placed into.
6538 // We don't have to fudge the checks above here because they are already worst case.
6539
6540 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
6541 // + 4 because better safe than sorry.
6542 const int fudge = 12 + 4;
6543 if (disp < 0) {
6544 disp -= fudge;
6545 } else {
6546 disp += fudge;
6547 }
6548 return is_simm32(disp);
6549 }
6550
6551 // Check if the polling page is not reachable from the code cache using rip-relative
6552 // addressing.
6553 bool Assembler::is_polling_page_far() {
6554 intptr_t addr = (intptr_t)os::get_polling_page();
6555 return ForceUnreachable ||
6556 !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
6557 !is_simm32(addr - (intptr_t)CodeCache::high_bound());
6558 }
6559
6560 void Assembler::emit_data64(jlong data,
6561 relocInfo::relocType rtype,
6562 int format) {
6563 if (rtype == relocInfo::none) {
6564 emit_int64(data);
6565 } else {
6566 emit_data64(data, Relocation::spec_simple(rtype), format);
6567 }
6568 }
6569
6570 void Assembler::emit_data64(jlong data,
6571 RelocationHolder const& rspec,
6572 int format) {
6573 assert(imm_operand == 0, "default format must be immediate in this file");
6574 assert(imm_operand == format, "must be immediate");
6575 assert(inst_mark() != NULL, "must be inside InstructionMark");
6576 // Do not use AbstractAssembler::relocate, which is not intended for
6577 // embedded words. Instead, relocate to the enclosing instruction.
6578 code_section()->relocate(inst_mark(), rspec, format);
6579 #ifdef ASSERT
6580 check_relocation(rspec, format);
6581 #endif
6582 emit_int64(data);
6583 }
6584
6585 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
6586 if (reg_enc >= 8) {
6587 prefix(REX_B);
6588 reg_enc -= 8;
6589 } else if (byteinst && reg_enc >= 4) {
6590 prefix(REX);
6591 }
6592 return reg_enc;
6593 }
6594
6595 int Assembler::prefixq_and_encode(int reg_enc) {
6596 if (reg_enc < 8) {
6597 prefix(REX_W);
6598 } else {
6599 prefix(REX_WB);
6600 reg_enc -= 8;
6601 }
6602 return reg_enc;
6603 }
6604
6605 int Assembler::prefix_and_encode(int dst_enc, bool dst_is_byte, int src_enc, bool src_is_byte) {
6606 if (dst_enc < 8) {
6607 if (src_enc >= 8) {
6608 prefix(REX_B);
6609 src_enc -= 8;
6610 } else if ((src_is_byte && src_enc >= 4) || (dst_is_byte && dst_enc >= 4)) {
6611 prefix(REX);
6612 }
6613 } else {
6614 if (src_enc < 8) {
6615 prefix(REX_R);
6616 } else {
6617 prefix(REX_RB);
6618 src_enc -= 8;
6619 }
6620 dst_enc -= 8;
6621 }
6622 return dst_enc << 3 | src_enc;
6623 }
6624
6625 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
6626 if (dst_enc < 8) {
6627 if (src_enc < 8) {
6628 prefix(REX_W);
6629 } else {
6630 prefix(REX_WB);
6631 src_enc -= 8;
6632 }
6633 } else {
6634 if (src_enc < 8) {
6635 prefix(REX_WR);
6636 } else {
6637 prefix(REX_WRB);
6638 src_enc -= 8;
6639 }
6640 dst_enc -= 8;
6641 }
6642 return dst_enc << 3 | src_enc;
6643 }
6644
6645 void Assembler::prefix(Register reg) {
6646 if (reg->encoding() >= 8) {
6647 prefix(REX_B);
6648 }
6649 }
6650
6651 void Assembler::prefix(Register dst, Register src, Prefix p) {
6652 if (src->encoding() >= 8) {
6653 p = (Prefix)(p | REX_B);
6654 }
6655 if (dst->encoding() >= 8) {
6656 p = (Prefix)( p | REX_R);
6657 }
6658 if (p != Prefix_EMPTY) {
6659 // do not generate an empty prefix
6660 prefix(p);
6661 }
6662 }
6663
6664 void Assembler::prefix(Register dst, Address adr, Prefix p) {
6665 if (adr.base_needs_rex()) {
6666 if (adr.index_needs_rex()) {
6667 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6668 } else {
6669 prefix(REX_B);
6670 }
6671 } else {
6672 if (adr.index_needs_rex()) {
6673 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6674 }
6675 }
6676 if (dst->encoding() >= 8) {
6677 p = (Prefix)(p | REX_R);
6678 }
6679 if (p != Prefix_EMPTY) {
6680 // do not generate an empty prefix
6681 prefix(p);
6682 }
6683 }
6684
6685 void Assembler::prefix(Address adr) {
6686 if (adr.base_needs_rex()) {
6687 if (adr.index_needs_rex()) {
6688 prefix(REX_XB);
6689 } else {
6690 prefix(REX_B);
6691 }
6692 } else {
6693 if (adr.index_needs_rex()) {
6694 prefix(REX_X);
6695 }
6696 }
6697 }
6698
6699 void Assembler::prefixq(Address adr) {
6700 if (adr.base_needs_rex()) {
6701 if (adr.index_needs_rex()) {
6702 prefix(REX_WXB);
6703 } else {
6704 prefix(REX_WB);
6705 }
6706 } else {
6707 if (adr.index_needs_rex()) {
6708 prefix(REX_WX);
6709 } else {
6710 prefix(REX_W);
6711 }
6712 }
6713 }
6714
6715
6716 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
6717 if (reg->encoding() < 8) {
6718 if (adr.base_needs_rex()) {
6719 if (adr.index_needs_rex()) {
6720 prefix(REX_XB);
6721 } else {
6722 prefix(REX_B);
6723 }
6724 } else {
6725 if (adr.index_needs_rex()) {
6726 prefix(REX_X);
6727 } else if (byteinst && reg->encoding() >= 4 ) {
6728 prefix(REX);
6729 }
6730 }
6731 } else {
6732 if (adr.base_needs_rex()) {
6733 if (adr.index_needs_rex()) {
6734 prefix(REX_RXB);
6735 } else {
6736 prefix(REX_RB);
6737 }
6738 } else {
6739 if (adr.index_needs_rex()) {
6740 prefix(REX_RX);
6741 } else {
6742 prefix(REX_R);
6743 }
6744 }
6745 }
6746 }
6747
6748 void Assembler::prefixq(Address adr, Register src) {
6749 if (src->encoding() < 8) {
6750 if (adr.base_needs_rex()) {
6751 if (adr.index_needs_rex()) {
6752 prefix(REX_WXB);
6753 } else {
6754 prefix(REX_WB);
6755 }
6756 } else {
6757 if (adr.index_needs_rex()) {
6758 prefix(REX_WX);
6759 } else {
6760 prefix(REX_W);
6761 }
6762 }
6763 } else {
6764 if (adr.base_needs_rex()) {
6765 if (adr.index_needs_rex()) {
6766 prefix(REX_WRXB);
6767 } else {
6768 prefix(REX_WRB);
6769 }
6770 } else {
6771 if (adr.index_needs_rex()) {
6772 prefix(REX_WRX);
6773 } else {
6774 prefix(REX_WR);
6775 }
6776 }
6777 }
6778 }
6779
6780 void Assembler::prefix(Address adr, XMMRegister reg) {
6781 if (reg->encoding() < 8) {
6782 if (adr.base_needs_rex()) {
6783 if (adr.index_needs_rex()) {
6784 prefix(REX_XB);
6785 } else {
6786 prefix(REX_B);
6787 }
6788 } else {
6789 if (adr.index_needs_rex()) {
6790 prefix(REX_X);
6791 }
6792 }
6793 } else {
6794 if (adr.base_needs_rex()) {
6795 if (adr.index_needs_rex()) {
6796 prefix(REX_RXB);
6797 } else {
6798 prefix(REX_RB);
6799 }
6800 } else {
6801 if (adr.index_needs_rex()) {
6802 prefix(REX_RX);
6803 } else {
6804 prefix(REX_R);
6805 }
6806 }
6807 }
6808 }
6809
6810 void Assembler::prefixq(Address adr, XMMRegister src) {
6811 if (src->encoding() < 8) {
6812 if (adr.base_needs_rex()) {
6813 if (adr.index_needs_rex()) {
6814 prefix(REX_WXB);
6815 } else {
6816 prefix(REX_WB);
6817 }
6818 } else {
6819 if (adr.index_needs_rex()) {
6820 prefix(REX_WX);
6821 } else {
6822 prefix(REX_W);
6823 }
6824 }
6825 } else {
6826 if (adr.base_needs_rex()) {
6827 if (adr.index_needs_rex()) {
6828 prefix(REX_WRXB);
6829 } else {
6830 prefix(REX_WRB);
6831 }
6832 } else {
6833 if (adr.index_needs_rex()) {
6834 prefix(REX_WRX);
6835 } else {
6836 prefix(REX_WR);
6837 }
6838 }
6839 }
6840 }
6841
6842 void Assembler::adcq(Register dst, int32_t imm32) {
6843 (void) prefixq_and_encode(dst->encoding());
6844 emit_arith(0x81, 0xD0, dst, imm32);
6845 }
6846
6847 void Assembler::adcq(Register dst, Address src) {
6848 InstructionMark im(this);
6849 prefixq(src, dst);
6850 emit_int8(0x13);
6851 emit_operand(dst, src);
6852 }
6853
6854 void Assembler::adcq(Register dst, Register src) {
6855 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6856 emit_arith(0x13, 0xC0, dst, src);
6857 }
6858
6859 void Assembler::addq(Address dst, int32_t imm32) {
6860 InstructionMark im(this);
6861 prefixq(dst);
6862 emit_arith_operand(0x81, rax, dst,imm32);
6863 }
6864
6865 void Assembler::addq(Address dst, Register src) {
6866 InstructionMark im(this);
6867 prefixq(dst, src);
6868 emit_int8(0x01);
6869 emit_operand(src, dst);
6870 }
6871
6872 void Assembler::addq(Register dst, int32_t imm32) {
6873 (void) prefixq_and_encode(dst->encoding());
6874 emit_arith(0x81, 0xC0, dst, imm32);
6875 }
6876
6877 void Assembler::addq(Register dst, Address src) {
6878 InstructionMark im(this);
6879 prefixq(src, dst);
6880 emit_int8(0x03);
6881 emit_operand(dst, src);
6882 }
6883
6884 void Assembler::addq(Register dst, Register src) {
6885 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6886 emit_arith(0x03, 0xC0, dst, src);
6887 }
6888
6889 void Assembler::adcxq(Register dst, Register src) {
6890 //assert(VM_Version::supports_adx(), "adx instructions not supported");
6891 emit_int8((unsigned char)0x66);
6892 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6893 emit_int8(0x0F);
6894 emit_int8(0x38);
6895 emit_int8((unsigned char)0xF6);
6896 emit_int8((unsigned char)(0xC0 | encode));
6897 }
6898
6899 void Assembler::adoxq(Register dst, Register src) {
6900 //assert(VM_Version::supports_adx(), "adx instructions not supported");
6901 emit_int8((unsigned char)0xF3);
6902 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6903 emit_int8(0x0F);
6904 emit_int8(0x38);
6905 emit_int8((unsigned char)0xF6);
6906 emit_int8((unsigned char)(0xC0 | encode));
6907 }
6908
6909 void Assembler::andq(Address dst, int32_t imm32) {
6910 InstructionMark im(this);
6911 prefixq(dst);
6912 emit_int8((unsigned char)0x81);
6913 emit_operand(rsp, dst, 4);
6914 emit_int32(imm32);
6915 }
6916
6917 void Assembler::andq(Register dst, int32_t imm32) {
6918 (void) prefixq_and_encode(dst->encoding());
6919 emit_arith(0x81, 0xE0, dst, imm32);
6920 }
6921
6922 void Assembler::andq(Register dst, Address src) {
6923 InstructionMark im(this);
6924 prefixq(src, dst);
6925 emit_int8(0x23);
6926 emit_operand(dst, src);
6927 }
6928
6929 void Assembler::andq(Register dst, Register src) {
6930 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6931 emit_arith(0x23, 0xC0, dst, src);
6932 }
6933
6934 void Assembler::andnq(Register dst, Register src1, Register src2) {
6935 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6936 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6937 int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6938 emit_int8((unsigned char)0xF2);
6939 emit_int8((unsigned char)(0xC0 | encode));
6940 }
6941
6942 void Assembler::andnq(Register dst, Register src1, Address src2) {
6943 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6944 InstructionMark im(this);
6945 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6946 vex_prefix(src2, src1->encoding(), dst->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6947 emit_int8((unsigned char)0xF2);
6948 emit_operand(dst, src2);
6949 }
6950
6951 void Assembler::bsfq(Register dst, Register src) {
6952 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6953 emit_int8(0x0F);
6954 emit_int8((unsigned char)0xBC);
6955 emit_int8((unsigned char)(0xC0 | encode));
6956 }
6957
6958 void Assembler::bsrq(Register dst, Register src) {
6959 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6960 emit_int8(0x0F);
6961 emit_int8((unsigned char)0xBD);
6962 emit_int8((unsigned char)(0xC0 | encode));
6963 }
6964
6965 void Assembler::bswapq(Register reg) {
6966 int encode = prefixq_and_encode(reg->encoding());
6967 emit_int8(0x0F);
6968 emit_int8((unsigned char)(0xC8 | encode));
6969 }
6970
6971 void Assembler::blsiq(Register dst, Register src) {
6972 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6973 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6974 int encode = vex_prefix_and_encode(rbx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6975 emit_int8((unsigned char)0xF3);
6976 emit_int8((unsigned char)(0xC0 | encode));
6977 }
6978
6979 void Assembler::blsiq(Register dst, Address src) {
6980 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6981 InstructionMark im(this);
6982 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6983 vex_prefix(src, dst->encoding(), rbx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6984 emit_int8((unsigned char)0xF3);
6985 emit_operand(rbx, src);
6986 }
6987
6988 void Assembler::blsmskq(Register dst, Register src) {
6989 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6990 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
6991 int encode = vex_prefix_and_encode(rdx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
6992 emit_int8((unsigned char)0xF3);
6993 emit_int8((unsigned char)(0xC0 | encode));
6994 }
6995
6996 void Assembler::blsmskq(Register dst, Address src) {
6997 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6998 InstructionMark im(this);
6999 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7000 vex_prefix(src, dst->encoding(), rdx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7001 emit_int8((unsigned char)0xF3);
7002 emit_operand(rdx, src);
7003 }
7004
7005 void Assembler::blsrq(Register dst, Register src) {
7006 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7007 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7008 int encode = vex_prefix_and_encode(rcx->encoding(), dst->encoding(), src->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7009 emit_int8((unsigned char)0xF3);
7010 emit_int8((unsigned char)(0xC0 | encode));
7011 }
7012
7013 void Assembler::blsrq(Register dst, Address src) {
7014 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
7015 InstructionMark im(this);
7016 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7017 vex_prefix(src, dst->encoding(), rcx->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F_38, &attributes);
7018 emit_int8((unsigned char)0xF3);
7019 emit_operand(rcx, src);
7020 }
7021
7022 void Assembler::cdqq() {
7023 prefix(REX_W);
7024 emit_int8((unsigned char)0x99);
7025 }
7026
7027 void Assembler::clflush(Address adr) {
7028 prefix(adr);
7029 emit_int8(0x0F);
7030 emit_int8((unsigned char)0xAE);
7031 emit_operand(rdi, adr);
7032 }
7033
7034 void Assembler::cmovq(Condition cc, Register dst, Register src) {
7035 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7036 emit_int8(0x0F);
7037 emit_int8(0x40 | cc);
7038 emit_int8((unsigned char)(0xC0 | encode));
7039 }
7040
7041 void Assembler::cmovq(Condition cc, Register dst, Address src) {
7042 InstructionMark im(this);
7043 prefixq(src, dst);
7044 emit_int8(0x0F);
7045 emit_int8(0x40 | cc);
7046 emit_operand(dst, src);
7047 }
7048
7049 void Assembler::cmpq(Address dst, int32_t imm32) {
7050 InstructionMark im(this);
7051 prefixq(dst);
7052 emit_int8((unsigned char)0x81);
7053 emit_operand(rdi, dst, 4);
7054 emit_int32(imm32);
7055 }
7056
7057 void Assembler::cmpq(Register dst, int32_t imm32) {
7058 (void) prefixq_and_encode(dst->encoding());
7059 emit_arith(0x81, 0xF8, dst, imm32);
7060 }
7061
7062 void Assembler::cmpq(Address dst, Register src) {
7063 InstructionMark im(this);
7064 prefixq(dst, src);
7065 emit_int8(0x3B);
7066 emit_operand(src, dst);
7067 }
7068
7069 void Assembler::cmpq(Register dst, Register src) {
7070 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7071 emit_arith(0x3B, 0xC0, dst, src);
7072 }
7073
7074 void Assembler::cmpq(Register dst, Address src) {
7075 InstructionMark im(this);
7076 prefixq(src, dst);
7077 emit_int8(0x3B);
7078 emit_operand(dst, src);
7079 }
7080
7081 void Assembler::cmpxchgq(Register reg, Address adr) {
7082 InstructionMark im(this);
7083 prefixq(adr, reg);
7084 emit_int8(0x0F);
7085 emit_int8((unsigned char)0xB1);
7086 emit_operand(reg, adr);
7087 }
7088
7089 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7090 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7091 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7092 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7093 emit_int8(0x2A);
7094 emit_int8((unsigned char)(0xC0 | encode));
7095 }
7096
7097 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7098 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7099 InstructionMark im(this);
7100 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7101 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7102 simd_prefix(dst, dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7103 emit_int8(0x2A);
7104 emit_operand(dst, src);
7105 }
7106
7107 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7108 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7109 InstructionMark im(this);
7110 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7111 attributes.set_address_attributes(/* tuple_type */ EVEX_T1S, /* input_size_in_bits */ EVEX_64bit);
7112 simd_prefix(dst, dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7113 emit_int8(0x2A);
7114 emit_operand(dst, src);
7115 }
7116
7117 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7118 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7119 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7120 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F2, VEX_OPCODE_0F, &attributes);
7121 emit_int8(0x2C);
7122 emit_int8((unsigned char)(0xC0 | encode));
7123 }
7124
7125 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7126 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7127 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7128 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_F3, VEX_OPCODE_0F, &attributes);
7129 emit_int8(0x2C);
7130 emit_int8((unsigned char)(0xC0 | encode));
7131 }
7132
7133 void Assembler::decl(Register dst) {
7134 // Don't use it directly. Use MacroAssembler::decrementl() instead.
7135 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7136 int encode = prefix_and_encode(dst->encoding());
7137 emit_int8((unsigned char)0xFF);
7138 emit_int8((unsigned char)(0xC8 | encode));
7139 }
7140
7141 void Assembler::decq(Register dst) {
7142 // Don't use it directly. Use MacroAssembler::decrementq() instead.
7143 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7144 int encode = prefixq_and_encode(dst->encoding());
7145 emit_int8((unsigned char)0xFF);
7146 emit_int8(0xC8 | encode);
7147 }
7148
7149 void Assembler::decq(Address dst) {
7150 // Don't use it directly. Use MacroAssembler::decrementq() instead.
7151 InstructionMark im(this);
7152 prefixq(dst);
7153 emit_int8((unsigned char)0xFF);
7154 emit_operand(rcx, dst);
7155 }
7156
7157 void Assembler::fxrstor(Address src) {
7158 prefixq(src);
7159 emit_int8(0x0F);
7160 emit_int8((unsigned char)0xAE);
7161 emit_operand(as_Register(1), src);
7162 }
7163
7164 void Assembler::xrstor(Address src) {
7165 prefixq(src);
7166 emit_int8(0x0F);
7167 emit_int8((unsigned char)0xAE);
7168 emit_operand(as_Register(5), src);
7169 }
7170
7171 void Assembler::fxsave(Address dst) {
7172 prefixq(dst);
7173 emit_int8(0x0F);
7174 emit_int8((unsigned char)0xAE);
7175 emit_operand(as_Register(0), dst);
7176 }
7177
7178 void Assembler::xsave(Address dst) {
7179 prefixq(dst);
7180 emit_int8(0x0F);
7181 emit_int8((unsigned char)0xAE);
7182 emit_operand(as_Register(4), dst);
7183 }
7184
7185 void Assembler::idivq(Register src) {
7186 int encode = prefixq_and_encode(src->encoding());
7187 emit_int8((unsigned char)0xF7);
7188 emit_int8((unsigned char)(0xF8 | encode));
7189 }
7190
7191 void Assembler::imulq(Register dst, Register src) {
7192 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7193 emit_int8(0x0F);
7194 emit_int8((unsigned char)0xAF);
7195 emit_int8((unsigned char)(0xC0 | encode));
7196 }
7197
7198 void Assembler::imulq(Register dst, Register src, int value) {
7199 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7200 if (is8bit(value)) {
7201 emit_int8(0x6B);
7202 emit_int8((unsigned char)(0xC0 | encode));
7203 emit_int8(value & 0xFF);
7204 } else {
7205 emit_int8(0x69);
7206 emit_int8((unsigned char)(0xC0 | encode));
7207 emit_int32(value);
7208 }
7209 }
7210
7211 void Assembler::imulq(Register dst, Address src) {
7212 InstructionMark im(this);
7213 prefixq(src, dst);
7214 emit_int8(0x0F);
7215 emit_int8((unsigned char) 0xAF);
7216 emit_operand(dst, src);
7217 }
7218
7219 void Assembler::incl(Register dst) {
7220 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7221 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7222 int encode = prefix_and_encode(dst->encoding());
7223 emit_int8((unsigned char)0xFF);
7224 emit_int8((unsigned char)(0xC0 | encode));
7225 }
7226
7227 void Assembler::incq(Register dst) {
7228 // Don't use it directly. Use MacroAssembler::incrementq() instead.
7229 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7230 int encode = prefixq_and_encode(dst->encoding());
7231 emit_int8((unsigned char)0xFF);
7232 emit_int8((unsigned char)(0xC0 | encode));
7233 }
7234
7235 void Assembler::incq(Address dst) {
7236 // Don't use it directly. Use MacroAssembler::incrementq() instead.
7237 InstructionMark im(this);
7238 prefixq(dst);
7239 emit_int8((unsigned char)0xFF);
7240 emit_operand(rax, dst);
7241 }
7242
7243 void Assembler::lea(Register dst, Address src) {
7244 leaq(dst, src);
7245 }
7246
7247 void Assembler::leaq(Register dst, Address src) {
7248 InstructionMark im(this);
7249 prefixq(src, dst);
7250 emit_int8((unsigned char)0x8D);
7251 emit_operand(dst, src);
7252 }
7253
7254 void Assembler::mov64(Register dst, int64_t imm64) {
7255 InstructionMark im(this);
7256 int encode = prefixq_and_encode(dst->encoding());
7257 emit_int8((unsigned char)(0xB8 | encode));
7258 emit_int64(imm64);
7259 }
7260
7261 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7262 InstructionMark im(this);
7263 int encode = prefixq_and_encode(dst->encoding());
7264 emit_int8(0xB8 | encode);
7265 emit_data64(imm64, rspec);
7266 }
7267
7268 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7269 InstructionMark im(this);
7270 int encode = prefix_and_encode(dst->encoding());
7271 emit_int8((unsigned char)(0xB8 | encode));
7272 emit_data((int)imm32, rspec, narrow_oop_operand);
7273 }
7274
7275 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7276 InstructionMark im(this);
7277 prefix(dst);
7278 emit_int8((unsigned char)0xC7);
7279 emit_operand(rax, dst, 4);
7280 emit_data((int)imm32, rspec, narrow_oop_operand);
7281 }
7282
7283 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7284 InstructionMark im(this);
7285 int encode = prefix_and_encode(src1->encoding());
7286 emit_int8((unsigned char)0x81);
7287 emit_int8((unsigned char)(0xF8 | encode));
7288 emit_data((int)imm32, rspec, narrow_oop_operand);
7289 }
7290
7291 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7292 InstructionMark im(this);
7293 prefix(src1);
7294 emit_int8((unsigned char)0x81);
7295 emit_operand(rax, src1, 4);
7296 emit_data((int)imm32, rspec, narrow_oop_operand);
7297 }
7298
7299 void Assembler::lzcntq(Register dst, Register src) {
7300 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7301 emit_int8((unsigned char)0xF3);
7302 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7303 emit_int8(0x0F);
7304 emit_int8((unsigned char)0xBD);
7305 emit_int8((unsigned char)(0xC0 | encode));
7306 }
7307
7308 void Assembler::movdq(XMMRegister dst, Register src) {
7309 // table D-1 says MMX/SSE2
7310 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7311 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7312 int encode = simd_prefix_and_encode(dst, xnoreg, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7313 emit_int8(0x6E);
7314 emit_int8((unsigned char)(0xC0 | encode));
7315 }
7316
7317 void Assembler::movdq(Register dst, XMMRegister src) {
7318 // table D-1 says MMX/SSE2
7319 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7320 InstructionAttr attributes(AVX_128bit, /* rex_w */ true, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ false);
7321 // swap src/dst to get correct prefix
7322 int encode = simd_prefix_and_encode(src, xnoreg, as_XMMRegister(dst->encoding()), VEX_SIMD_66, VEX_OPCODE_0F, &attributes);
7323 emit_int8(0x7E);
7324 emit_int8((unsigned char)(0xC0 | encode));
7325 }
7326
7327 void Assembler::movq(Register dst, Register src) {
7328 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7329 emit_int8((unsigned char)0x8B);
7330 emit_int8((unsigned char)(0xC0 | encode));
7331 }
7332
7333 void Assembler::movq(Register dst, Address src) {
7334 InstructionMark im(this);
7335 prefixq(src, dst);
7336 emit_int8((unsigned char)0x8B);
7337 emit_operand(dst, src);
7338 }
7339
7340 void Assembler::movq(Address dst, Register src) {
7341 InstructionMark im(this);
7342 prefixq(dst, src);
7343 emit_int8((unsigned char)0x89);
7344 emit_operand(src, dst);
7345 }
7346
7347 void Assembler::movsbq(Register dst, Address src) {
7348 InstructionMark im(this);
7349 prefixq(src, dst);
7350 emit_int8(0x0F);
7351 emit_int8((unsigned char)0xBE);
7352 emit_operand(dst, src);
7353 }
7354
7355 void Assembler::movsbq(Register dst, Register src) {
7356 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7357 emit_int8(0x0F);
7358 emit_int8((unsigned char)0xBE);
7359 emit_int8((unsigned char)(0xC0 | encode));
7360 }
7361
7362 void Assembler::movslq(Register dst, int32_t imm32) {
7363 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
7364 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
7365 // as a result we shouldn't use until tested at runtime...
7366 ShouldNotReachHere();
7367 InstructionMark im(this);
7368 int encode = prefixq_and_encode(dst->encoding());
7369 emit_int8((unsigned char)(0xC7 | encode));
7370 emit_int32(imm32);
7371 }
7372
7373 void Assembler::movslq(Address dst, int32_t imm32) {
7374 assert(is_simm32(imm32), "lost bits");
7375 InstructionMark im(this);
7376 prefixq(dst);
7377 emit_int8((unsigned char)0xC7);
7378 emit_operand(rax, dst, 4);
7379 emit_int32(imm32);
7380 }
7381
7382 void Assembler::movslq(Register dst, Address src) {
7383 InstructionMark im(this);
7384 prefixq(src, dst);
7385 emit_int8(0x63);
7386 emit_operand(dst, src);
7387 }
7388
7389 void Assembler::movslq(Register dst, Register src) {
7390 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7391 emit_int8(0x63);
7392 emit_int8((unsigned char)(0xC0 | encode));
7393 }
7394
7395 void Assembler::movswq(Register dst, Address src) {
7396 InstructionMark im(this);
7397 prefixq(src, dst);
7398 emit_int8(0x0F);
7399 emit_int8((unsigned char)0xBF);
7400 emit_operand(dst, src);
7401 }
7402
7403 void Assembler::movswq(Register dst, Register src) {
7404 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7405 emit_int8((unsigned char)0x0F);
7406 emit_int8((unsigned char)0xBF);
7407 emit_int8((unsigned char)(0xC0 | encode));
7408 }
7409
7410 void Assembler::movzbq(Register dst, Address src) {
7411 InstructionMark im(this);
7412 prefixq(src, dst);
7413 emit_int8((unsigned char)0x0F);
7414 emit_int8((unsigned char)0xB6);
7415 emit_operand(dst, src);
7416 }
7417
7418 void Assembler::movzbq(Register dst, Register src) {
7419 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7420 emit_int8(0x0F);
7421 emit_int8((unsigned char)0xB6);
7422 emit_int8(0xC0 | encode);
7423 }
7424
7425 void Assembler::movzwq(Register dst, Address src) {
7426 InstructionMark im(this);
7427 prefixq(src, dst);
7428 emit_int8((unsigned char)0x0F);
7429 emit_int8((unsigned char)0xB7);
7430 emit_operand(dst, src);
7431 }
7432
7433 void Assembler::movzwq(Register dst, Register src) {
7434 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7435 emit_int8((unsigned char)0x0F);
7436 emit_int8((unsigned char)0xB7);
7437 emit_int8((unsigned char)(0xC0 | encode));
7438 }
7439
7440 void Assembler::mulq(Address src) {
7441 InstructionMark im(this);
7442 prefixq(src);
7443 emit_int8((unsigned char)0xF7);
7444 emit_operand(rsp, src);
7445 }
7446
7447 void Assembler::mulq(Register src) {
7448 int encode = prefixq_and_encode(src->encoding());
7449 emit_int8((unsigned char)0xF7);
7450 emit_int8((unsigned char)(0xE0 | encode));
7451 }
7452
7453 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
7454 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7455 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7456 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38, &attributes);
7457 emit_int8((unsigned char)0xF6);
7458 emit_int8((unsigned char)(0xC0 | encode));
7459 }
7460
7461 void Assembler::negq(Register dst) {
7462 int encode = prefixq_and_encode(dst->encoding());
7463 emit_int8((unsigned char)0xF7);
7464 emit_int8((unsigned char)(0xD8 | encode));
7465 }
7466
7467 void Assembler::notq(Register dst) {
7468 int encode = prefixq_and_encode(dst->encoding());
7469 emit_int8((unsigned char)0xF7);
7470 emit_int8((unsigned char)(0xD0 | encode));
7471 }
7472
7473 void Assembler::orq(Address dst, int32_t imm32) {
7474 InstructionMark im(this);
7475 prefixq(dst);
7476 emit_int8((unsigned char)0x81);
7477 emit_operand(rcx, dst, 4);
7478 emit_int32(imm32);
7479 }
7480
7481 void Assembler::orq(Register dst, int32_t imm32) {
7482 (void) prefixq_and_encode(dst->encoding());
7483 emit_arith(0x81, 0xC8, dst, imm32);
7484 }
7485
7486 void Assembler::orq(Register dst, Address src) {
7487 InstructionMark im(this);
7488 prefixq(src, dst);
7489 emit_int8(0x0B);
7490 emit_operand(dst, src);
7491 }
7492
7493 void Assembler::orq(Register dst, Register src) {
7494 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7495 emit_arith(0x0B, 0xC0, dst, src);
7496 }
7497
7498 void Assembler::popa() { // 64bit
7499 movq(r15, Address(rsp, 0));
7500 movq(r14, Address(rsp, wordSize));
7501 movq(r13, Address(rsp, 2 * wordSize));
7502 movq(r12, Address(rsp, 3 * wordSize));
7503 movq(r11, Address(rsp, 4 * wordSize));
7504 movq(r10, Address(rsp, 5 * wordSize));
7505 movq(r9, Address(rsp, 6 * wordSize));
7506 movq(r8, Address(rsp, 7 * wordSize));
7507 movq(rdi, Address(rsp, 8 * wordSize));
7508 movq(rsi, Address(rsp, 9 * wordSize));
7509 movq(rbp, Address(rsp, 10 * wordSize));
7510 // skip rsp
7511 movq(rbx, Address(rsp, 12 * wordSize));
7512 movq(rdx, Address(rsp, 13 * wordSize));
7513 movq(rcx, Address(rsp, 14 * wordSize));
7514 movq(rax, Address(rsp, 15 * wordSize));
7515
7516 addq(rsp, 16 * wordSize);
7517 }
7518
7519 void Assembler::popcntq(Register dst, Address src) {
7520 assert(VM_Version::supports_popcnt(), "must support");
7521 InstructionMark im(this);
7522 emit_int8((unsigned char)0xF3);
7523 prefixq(src, dst);
7524 emit_int8((unsigned char)0x0F);
7525 emit_int8((unsigned char)0xB8);
7526 emit_operand(dst, src);
7527 }
7528
7529 void Assembler::popcntq(Register dst, Register src) {
7530 assert(VM_Version::supports_popcnt(), "must support");
7531 emit_int8((unsigned char)0xF3);
7532 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7533 emit_int8((unsigned char)0x0F);
7534 emit_int8((unsigned char)0xB8);
7535 emit_int8((unsigned char)(0xC0 | encode));
7536 }
7537
7538 void Assembler::popq(Address dst) {
7539 InstructionMark im(this);
7540 prefixq(dst);
7541 emit_int8((unsigned char)0x8F);
7542 emit_operand(rax, dst);
7543 }
7544
7545 void Assembler::pusha() { // 64bit
7546 // we have to store original rsp. ABI says that 128 bytes
7547 // below rsp are local scratch.
7548 movq(Address(rsp, -5 * wordSize), rsp);
7549
7550 subq(rsp, 16 * wordSize);
7551
7552 movq(Address(rsp, 15 * wordSize), rax);
7553 movq(Address(rsp, 14 * wordSize), rcx);
7554 movq(Address(rsp, 13 * wordSize), rdx);
7555 movq(Address(rsp, 12 * wordSize), rbx);
7556 // skip rsp
7557 movq(Address(rsp, 10 * wordSize), rbp);
7558 movq(Address(rsp, 9 * wordSize), rsi);
7559 movq(Address(rsp, 8 * wordSize), rdi);
7560 movq(Address(rsp, 7 * wordSize), r8);
7561 movq(Address(rsp, 6 * wordSize), r9);
7562 movq(Address(rsp, 5 * wordSize), r10);
7563 movq(Address(rsp, 4 * wordSize), r11);
7564 movq(Address(rsp, 3 * wordSize), r12);
7565 movq(Address(rsp, 2 * wordSize), r13);
7566 movq(Address(rsp, wordSize), r14);
7567 movq(Address(rsp, 0), r15);
7568 }
7569
7570 void Assembler::pushq(Address src) {
7571 InstructionMark im(this);
7572 prefixq(src);
7573 emit_int8((unsigned char)0xFF);
7574 emit_operand(rsi, src);
7575 }
7576
7577 void Assembler::rclq(Register dst, int imm8) {
7578 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7579 int encode = prefixq_and_encode(dst->encoding());
7580 if (imm8 == 1) {
7581 emit_int8((unsigned char)0xD1);
7582 emit_int8((unsigned char)(0xD0 | encode));
7583 } else {
7584 emit_int8((unsigned char)0xC1);
7585 emit_int8((unsigned char)(0xD0 | encode));
7586 emit_int8(imm8);
7587 }
7588 }
7589
7590 void Assembler::rcrq(Register dst, int imm8) {
7591 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7592 int encode = prefixq_and_encode(dst->encoding());
7593 if (imm8 == 1) {
7594 emit_int8((unsigned char)0xD1);
7595 emit_int8((unsigned char)(0xD8 | encode));
7596 } else {
7597 emit_int8((unsigned char)0xC1);
7598 emit_int8((unsigned char)(0xD8 | encode));
7599 emit_int8(imm8);
7600 }
7601 }
7602
7603 void Assembler::rorq(Register dst, int imm8) {
7604 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7605 int encode = prefixq_and_encode(dst->encoding());
7606 if (imm8 == 1) {
7607 emit_int8((unsigned char)0xD1);
7608 emit_int8((unsigned char)(0xC8 | encode));
7609 } else {
7610 emit_int8((unsigned char)0xC1);
7611 emit_int8((unsigned char)(0xc8 | encode));
7612 emit_int8(imm8);
7613 }
7614 }
7615
7616 void Assembler::rorxq(Register dst, Register src, int imm8) {
7617 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7618 InstructionAttr attributes(AVX_128bit, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
7619 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A, &attributes);
7620 emit_int8((unsigned char)0xF0);
7621 emit_int8((unsigned char)(0xC0 | encode));
7622 emit_int8(imm8);
7623 }
7624
7625 void Assembler::sarq(Register dst, int imm8) {
7626 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7627 int encode = prefixq_and_encode(dst->encoding());
7628 if (imm8 == 1) {
7629 emit_int8((unsigned char)0xD1);
7630 emit_int8((unsigned char)(0xF8 | encode));
7631 } else {
7632 emit_int8((unsigned char)0xC1);
7633 emit_int8((unsigned char)(0xF8 | encode));
7634 emit_int8(imm8);
7635 }
7636 }
7637
7638 void Assembler::sarq(Register dst) {
7639 int encode = prefixq_and_encode(dst->encoding());
7640 emit_int8((unsigned char)0xD3);
7641 emit_int8((unsigned char)(0xF8 | encode));
7642 }
7643
7644 void Assembler::sbbq(Address dst, int32_t imm32) {
7645 InstructionMark im(this);
7646 prefixq(dst);
7647 emit_arith_operand(0x81, rbx, dst, imm32);
7648 }
7649
7650 void Assembler::sbbq(Register dst, int32_t imm32) {
7651 (void) prefixq_and_encode(dst->encoding());
7652 emit_arith(0x81, 0xD8, dst, imm32);
7653 }
7654
7655 void Assembler::sbbq(Register dst, Address src) {
7656 InstructionMark im(this);
7657 prefixq(src, dst);
7658 emit_int8(0x1B);
7659 emit_operand(dst, src);
7660 }
7661
7662 void Assembler::sbbq(Register dst, Register src) {
7663 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7664 emit_arith(0x1B, 0xC0, dst, src);
7665 }
7666
7667 void Assembler::shlq(Register dst, int imm8) {
7668 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7669 int encode = prefixq_and_encode(dst->encoding());
7670 if (imm8 == 1) {
7671 emit_int8((unsigned char)0xD1);
7672 emit_int8((unsigned char)(0xE0 | encode));
7673 } else {
7674 emit_int8((unsigned char)0xC1);
7675 emit_int8((unsigned char)(0xE0 | encode));
7676 emit_int8(imm8);
7677 }
7678 }
7679
7680 void Assembler::shlq(Register dst) {
7681 int encode = prefixq_and_encode(dst->encoding());
7682 emit_int8((unsigned char)0xD3);
7683 emit_int8((unsigned char)(0xE0 | encode));
7684 }
7685
7686 void Assembler::shrq(Register dst, int imm8) {
7687 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7688 int encode = prefixq_and_encode(dst->encoding());
7689 emit_int8((unsigned char)0xC1);
7690 emit_int8((unsigned char)(0xE8 | encode));
7691 emit_int8(imm8);
7692 }
7693
7694 void Assembler::shrq(Register dst) {
7695 int encode = prefixq_and_encode(dst->encoding());
7696 emit_int8((unsigned char)0xD3);
7697 emit_int8(0xE8 | encode);
7698 }
7699
7700 void Assembler::subq(Address dst, int32_t imm32) {
7701 InstructionMark im(this);
7702 prefixq(dst);
7703 emit_arith_operand(0x81, rbp, dst, imm32);
7704 }
7705
7706 void Assembler::subq(Address dst, Register src) {
7707 InstructionMark im(this);
7708 prefixq(dst, src);
7709 emit_int8(0x29);
7710 emit_operand(src, dst);
7711 }
7712
7713 void Assembler::subq(Register dst, int32_t imm32) {
7714 (void) prefixq_and_encode(dst->encoding());
7715 emit_arith(0x81, 0xE8, dst, imm32);
7716 }
7717
7718 // Force generation of a 4 byte immediate value even if it fits into 8bit
7719 void Assembler::subq_imm32(Register dst, int32_t imm32) {
7720 (void) prefixq_and_encode(dst->encoding());
7721 emit_arith_imm32(0x81, 0xE8, dst, imm32);
7722 }
7723
7724 void Assembler::subq(Register dst, Address src) {
7725 InstructionMark im(this);
7726 prefixq(src, dst);
7727 emit_int8(0x2B);
7728 emit_operand(dst, src);
7729 }
7730
7731 void Assembler::subq(Register dst, Register src) {
7732 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7733 emit_arith(0x2B, 0xC0, dst, src);
7734 }
7735
7736 void Assembler::testq(Register dst, int32_t imm32) {
7737 // not using emit_arith because test
7738 // doesn't support sign-extension of
7739 // 8bit operands
7740 int encode = dst->encoding();
7741 if (encode == 0) {
7742 prefix(REX_W);
7743 emit_int8((unsigned char)0xA9);
7744 } else {
7745 encode = prefixq_and_encode(encode);
7746 emit_int8((unsigned char)0xF7);
7747 emit_int8((unsigned char)(0xC0 | encode));
7748 }
7749 emit_int32(imm32);
7750 }
7751
7752 void Assembler::testq(Register dst, Register src) {
7753 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7754 emit_arith(0x85, 0xC0, dst, src);
7755 }
7756
7757 void Assembler::xaddq(Address dst, Register src) {
7758 InstructionMark im(this);
7759 prefixq(dst, src);
7760 emit_int8(0x0F);
7761 emit_int8((unsigned char)0xC1);
7762 emit_operand(src, dst);
7763 }
7764
7765 void Assembler::xchgq(Register dst, Address src) {
7766 InstructionMark im(this);
7767 prefixq(src, dst);
7768 emit_int8((unsigned char)0x87);
7769 emit_operand(dst, src);
7770 }
7771
7772 void Assembler::xchgq(Register dst, Register src) {
7773 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7774 emit_int8((unsigned char)0x87);
7775 emit_int8((unsigned char)(0xc0 | encode));
7776 }
7777
7778 void Assembler::xorq(Register dst, Register src) {
7779 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7780 emit_arith(0x33, 0xC0, dst, src);
7781 }
7782
7783 void Assembler::xorq(Register dst, Address src) {
7784 InstructionMark im(this);
7785 prefixq(src, dst);
7786 emit_int8(0x33);
7787 emit_operand(dst, src);
7788 }
7789
7790 #endif // !LP64