1 /*
2 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc/shared/cardTableModRefBS.hpp"
29 #include "gc/shared/collectedHeap.inline.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/biasedLocking.hpp"
34 #include "runtime/interfaceSupport.hpp"
35 #include "runtime/objectMonitor.hpp"
36 #include "runtime/os.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "runtime/stubRoutines.hpp"
39 #include "utilities/macros.hpp"
40 #if INCLUDE_ALL_GCS
41 #include "gc/g1/g1CollectedHeap.inline.hpp"
42 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
43 #include "gc/g1/heapRegion.hpp"
44 #endif // INCLUDE_ALL_GCS
45
46 #ifdef PRODUCT
47 #define BLOCK_COMMENT(str) /* nothing */
48 #define STOP(error) stop(error)
49 #else
50 #define BLOCK_COMMENT(str) block_comment(str)
51 #define STOP(error) block_comment(error); stop(error)
52 #endif
53
54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
55 // Implementation of AddressLiteral
56
57 // A 2-D table for managing compressed displacement(disp8) on EVEX enabled platforms.
58 unsigned char tuple_table[Assembler::EVEX_ETUP + 1][Assembler::AVX_512bit + 1] = {
59 // -----------------Table 4.5 -------------------- //
60 16, 32, 64, // EVEX_FV(0)
61 4, 4, 4, // EVEX_FV(1) - with Evex.b
62 16, 32, 64, // EVEX_FV(2) - with Evex.w
63 8, 8, 8, // EVEX_FV(3) - with Evex.w and Evex.b
64 8, 16, 32, // EVEX_HV(0)
65 4, 4, 4, // EVEX_HV(1) - with Evex.b
66 // -----------------Table 4.6 -------------------- //
67 16, 32, 64, // EVEX_FVM(0)
68 1, 1, 1, // EVEX_T1S(0)
69 2, 2, 2, // EVEX_T1S(1)
70 4, 4, 4, // EVEX_T1S(2)
71 8, 8, 8, // EVEX_T1S(3)
72 4, 4, 4, // EVEX_T1F(0)
73 8, 8, 8, // EVEX_T1F(1)
74 8, 8, 8, // EVEX_T2(0)
75 0, 16, 16, // EVEX_T2(1)
76 0, 16, 16, // EVEX_T4(0)
77 0, 0, 32, // EVEX_T4(1)
78 0, 0, 32, // EVEX_T8(0)
79 8, 16, 32, // EVEX_HVM(0)
80 4, 8, 16, // EVEX_QVM(0)
81 2, 4, 8, // EVEX_OVM(0)
82 16, 16, 16, // EVEX_M128(0)
83 8, 32, 64, // EVEX_DUP(0)
84 0, 0, 0 // EVEX_NTUP
85 };
86
87 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
88 _is_lval = false;
89 _target = target;
90 switch (rtype) {
91 case relocInfo::oop_type:
92 case relocInfo::metadata_type:
93 // Oops are a special case. Normally they would be their own section
94 // but in cases like icBuffer they are literals in the code stream that
95 // we don't have a section for. We use none so that we get a literal address
96 // which is always patchable.
97 break;
98 case relocInfo::external_word_type:
99 _rspec = external_word_Relocation::spec(target);
100 break;
101 case relocInfo::internal_word_type:
102 _rspec = internal_word_Relocation::spec(target);
103 break;
104 case relocInfo::opt_virtual_call_type:
105 _rspec = opt_virtual_call_Relocation::spec();
106 break;
107 case relocInfo::static_call_type:
108 _rspec = static_call_Relocation::spec();
109 break;
110 case relocInfo::runtime_call_type:
111 _rspec = runtime_call_Relocation::spec();
112 break;
113 case relocInfo::poll_type:
114 case relocInfo::poll_return_type:
115 _rspec = Relocation::spec_simple(rtype);
116 break;
117 case relocInfo::none:
118 break;
119 default:
120 ShouldNotReachHere();
121 break;
122 }
123 }
124
125 // Implementation of Address
126
127 #ifdef _LP64
128
129 Address Address::make_array(ArrayAddress adr) {
130 // Not implementable on 64bit machines
131 // Should have been handled higher up the call chain.
132 ShouldNotReachHere();
133 return Address();
134 }
135
136 // exceedingly dangerous constructor
137 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
138 _base = noreg;
139 _index = noreg;
140 _scale = no_scale;
141 _disp = disp;
142 switch (rtype) {
143 case relocInfo::external_word_type:
144 _rspec = external_word_Relocation::spec(loc);
145 break;
146 case relocInfo::internal_word_type:
147 _rspec = internal_word_Relocation::spec(loc);
148 break;
149 case relocInfo::runtime_call_type:
150 // HMM
151 _rspec = runtime_call_Relocation::spec();
152 break;
153 case relocInfo::poll_type:
154 case relocInfo::poll_return_type:
155 _rspec = Relocation::spec_simple(rtype);
156 break;
157 case relocInfo::none:
158 break;
159 default:
160 ShouldNotReachHere();
161 }
162 }
163 #else // LP64
164
165 Address Address::make_array(ArrayAddress adr) {
166 AddressLiteral base = adr.base();
167 Address index = adr.index();
168 assert(index._disp == 0, "must not have disp"); // maybe it can?
169 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
170 array._rspec = base._rspec;
171 return array;
172 }
173
174 // exceedingly dangerous constructor
175 Address::Address(address loc, RelocationHolder spec) {
176 _base = noreg;
177 _index = noreg;
178 _scale = no_scale;
179 _disp = (intptr_t) loc;
180 _rspec = spec;
181 }
182
183 #endif // _LP64
184
185
186
187 // Convert the raw encoding form into the form expected by the constructor for
188 // Address. An index of 4 (rsp) corresponds to having no index, so convert
189 // that to noreg for the Address constructor.
190 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
191 RelocationHolder rspec;
192 if (disp_reloc != relocInfo::none) {
193 rspec = Relocation::spec_simple(disp_reloc);
194 }
195 bool valid_index = index != rsp->encoding();
196 if (valid_index) {
197 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
198 madr._rspec = rspec;
199 return madr;
200 } else {
201 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
202 madr._rspec = rspec;
203 return madr;
204 }
205 }
206
207 // Implementation of Assembler
208
209 int AbstractAssembler::code_fill_byte() {
210 return (u_char)'\xF4'; // hlt
211 }
212
213 // make this go away someday
214 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
215 if (rtype == relocInfo::none)
216 emit_int32(data);
217 else
218 emit_data(data, Relocation::spec_simple(rtype), format);
219 }
220
221 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
222 assert(imm_operand == 0, "default format must be immediate in this file");
223 assert(inst_mark() != NULL, "must be inside InstructionMark");
224 if (rspec.type() != relocInfo::none) {
225 #ifdef ASSERT
226 check_relocation(rspec, format);
227 #endif
228 // Do not use AbstractAssembler::relocate, which is not intended for
229 // embedded words. Instead, relocate to the enclosing instruction.
230
231 // hack. call32 is too wide for mask so use disp32
232 if (format == call32_operand)
233 code_section()->relocate(inst_mark(), rspec, disp32_operand);
234 else
235 code_section()->relocate(inst_mark(), rspec, format);
236 }
237 emit_int32(data);
238 }
239
240 static int encode(Register r) {
241 int enc = r->encoding();
242 if (enc >= 8) {
243 enc -= 8;
244 }
245 return enc;
246 }
247
248 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
249 assert(dst->has_byte_register(), "must have byte register");
250 assert(isByte(op1) && isByte(op2), "wrong opcode");
251 assert(isByte(imm8), "not a byte");
252 assert((op1 & 0x01) == 0, "should be 8bit operation");
253 emit_int8(op1);
254 emit_int8(op2 | encode(dst));
255 emit_int8(imm8);
256 }
257
258
259 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
260 assert(isByte(op1) && isByte(op2), "wrong opcode");
261 assert((op1 & 0x01) == 1, "should be 32bit operation");
262 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
263 if (is8bit(imm32)) {
264 emit_int8(op1 | 0x02); // set sign bit
265 emit_int8(op2 | encode(dst));
266 emit_int8(imm32 & 0xFF);
267 } else {
268 emit_int8(op1);
269 emit_int8(op2 | encode(dst));
270 emit_int32(imm32);
271 }
272 }
273
274 // Force generation of a 4 byte immediate value even if it fits into 8bit
275 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
276 assert(isByte(op1) && isByte(op2), "wrong opcode");
277 assert((op1 & 0x01) == 1, "should be 32bit operation");
278 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
279 emit_int8(op1);
280 emit_int8(op2 | encode(dst));
281 emit_int32(imm32);
282 }
283
284 // immediate-to-memory forms
285 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
286 assert((op1 & 0x01) == 1, "should be 32bit operation");
287 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
288 if (is8bit(imm32)) {
289 emit_int8(op1 | 0x02); // set sign bit
290 emit_operand(rm, adr, 1);
291 emit_int8(imm32 & 0xFF);
292 } else {
293 emit_int8(op1);
294 emit_operand(rm, adr, 4);
295 emit_int32(imm32);
296 }
297 }
298
299
300 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
301 assert(isByte(op1) && isByte(op2), "wrong opcode");
302 emit_int8(op1);
303 emit_int8(op2 | encode(dst) << 3 | encode(src));
304 }
305
306
307 bool Assembler::query_compressed_disp_byte(int disp, bool is_evex_inst, int vector_len,
308 int cur_tuple_type, int in_size_in_bits, int cur_encoding) {
309 int mod_idx = 0;
310 // We will test if the displacement fits the compressed format and if so
311 // apply the compression to the displacment iff the result is8bit.
312 if (VM_Version::supports_evex() && is_evex_inst) {
313 switch (cur_tuple_type) {
314 case EVEX_FV:
315 if ((cur_encoding & VEX_W) == VEX_W) {
316 mod_idx += 2 + ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
317 } else {
318 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
319 }
320 break;
321
322 case EVEX_HV:
323 mod_idx = ((cur_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
324 break;
325
326 case EVEX_FVM:
327 break;
328
329 case EVEX_T1S:
330 switch (in_size_in_bits) {
331 case EVEX_8bit:
332 break;
333
334 case EVEX_16bit:
335 mod_idx = 1;
336 break;
337
338 case EVEX_32bit:
339 mod_idx = 2;
340 break;
341
342 case EVEX_64bit:
343 mod_idx = 3;
344 break;
345 }
346 break;
347
348 case EVEX_T1F:
349 case EVEX_T2:
350 case EVEX_T4:
351 mod_idx = (in_size_in_bits == EVEX_64bit) ? 1 : 0;
352 break;
353
354 case EVEX_T8:
355 break;
356
357 case EVEX_HVM:
358 break;
359
360 case EVEX_QVM:
361 break;
362
363 case EVEX_OVM:
364 break;
365
366 case EVEX_M128:
367 break;
368
369 case EVEX_DUP:
370 break;
371
372 default:
373 assert(0, "no valid evex tuple_table entry");
374 break;
375 }
376
377 if (vector_len >= AVX_128bit && vector_len <= AVX_512bit) {
378 int disp_factor = tuple_table[cur_tuple_type + mod_idx][vector_len];
379 if ((disp % disp_factor) == 0) {
380 int new_disp = disp / disp_factor;
381 if ((-0x80 <= new_disp && new_disp < 0x80)) {
382 disp = new_disp;
383 }
384 } else {
385 return false;
386 }
387 }
388 }
389 return (-0x80 <= disp && disp < 0x80);
390 }
391
392
393 bool Assembler::emit_compressed_disp_byte(int &disp) {
394 int mod_idx = 0;
395 // We will test if the displacement fits the compressed format and if so
396 // apply the compression to the displacment iff the result is8bit.
397 if (VM_Version::supports_evex() && _is_evex_instruction) {
398 switch (_tuple_type) {
399 case EVEX_FV:
400 if ((_evex_encoding & VEX_W) == VEX_W) {
401 mod_idx += 2 + ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
402 } else {
403 mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
404 }
405 break;
406
407 case EVEX_HV:
408 mod_idx = ((_evex_encoding & EVEX_Rb) == EVEX_Rb) ? 1 : 0;
409 break;
410
411 case EVEX_FVM:
412 break;
413
414 case EVEX_T1S:
415 switch (_input_size_in_bits) {
416 case EVEX_8bit:
417 break;
418
419 case EVEX_16bit:
420 mod_idx = 1;
421 break;
422
423 case EVEX_32bit:
424 mod_idx = 2;
425 break;
426
427 case EVEX_64bit:
428 mod_idx = 3;
429 break;
430 }
431 break;
432
433 case EVEX_T1F:
434 case EVEX_T2:
435 case EVEX_T4:
436 mod_idx = (_input_size_in_bits == EVEX_64bit) ? 1 : 0;
437 break;
438
439 case EVEX_T8:
440 break;
441
442 case EVEX_HVM:
443 break;
444
445 case EVEX_QVM:
446 break;
447
448 case EVEX_OVM:
449 break;
450
451 case EVEX_M128:
452 break;
453
454 case EVEX_DUP:
455 break;
456
457 default:
458 assert(0, "no valid evex tuple_table entry");
459 break;
460 }
461
462 if (_avx_vector_len >= AVX_128bit && _avx_vector_len <= AVX_512bit) {
463 int disp_factor = tuple_table[_tuple_type + mod_idx][_avx_vector_len];
464 if ((disp % disp_factor) == 0) {
465 int new_disp = disp / disp_factor;
466 if (is8bit(new_disp)) {
467 disp = new_disp;
468 }
469 } else {
470 return false;
471 }
472 }
473 }
474 return is8bit(disp);
475 }
476
477
478 void Assembler::emit_operand(Register reg, Register base, Register index,
479 Address::ScaleFactor scale, int disp,
480 RelocationHolder const& rspec,
481 int rip_relative_correction) {
482 relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
483
484 // Encode the registers as needed in the fields they are used in
485
486 int regenc = encode(reg) << 3;
487 int indexenc = index->is_valid() ? encode(index) << 3 : 0;
488 int baseenc = base->is_valid() ? encode(base) : 0;
489
490 if (base->is_valid()) {
491 if (index->is_valid()) {
492 assert(scale != Address::no_scale, "inconsistent address");
493 // [base + index*scale + disp]
494 if (disp == 0 && rtype == relocInfo::none &&
495 base != rbp LP64_ONLY(&& base != r13)) {
496 // [base + index*scale]
497 // [00 reg 100][ss index base]
498 assert(index != rsp, "illegal addressing mode");
499 emit_int8(0x04 | regenc);
500 emit_int8(scale << 6 | indexenc | baseenc);
501 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
502 // [base + index*scale + imm8]
503 // [01 reg 100][ss index base] imm8
504 assert(index != rsp, "illegal addressing mode");
505 emit_int8(0x44 | regenc);
506 emit_int8(scale << 6 | indexenc | baseenc);
507 emit_int8(disp & 0xFF);
508 } else {
509 // [base + index*scale + disp32]
510 // [10 reg 100][ss index base] disp32
511 assert(index != rsp, "illegal addressing mode");
512 emit_int8(0x84 | regenc);
513 emit_int8(scale << 6 | indexenc | baseenc);
514 emit_data(disp, rspec, disp32_operand);
515 }
516 } else if (base == rsp LP64_ONLY(|| base == r12)) {
517 // [rsp + disp]
518 if (disp == 0 && rtype == relocInfo::none) {
519 // [rsp]
520 // [00 reg 100][00 100 100]
521 emit_int8(0x04 | regenc);
522 emit_int8(0x24);
523 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
524 // [rsp + imm8]
525 // [01 reg 100][00 100 100] disp8
526 emit_int8(0x44 | regenc);
527 emit_int8(0x24);
528 emit_int8(disp & 0xFF);
529 } else {
530 // [rsp + imm32]
531 // [10 reg 100][00 100 100] disp32
532 emit_int8(0x84 | regenc);
533 emit_int8(0x24);
534 emit_data(disp, rspec, disp32_operand);
535 }
536 } else {
537 // [base + disp]
538 assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
539 if (disp == 0 && rtype == relocInfo::none &&
540 base != rbp LP64_ONLY(&& base != r13)) {
541 // [base]
542 // [00 reg base]
543 emit_int8(0x00 | regenc | baseenc);
544 } else if (emit_compressed_disp_byte(disp) && rtype == relocInfo::none) {
545 // [base + disp8]
546 // [01 reg base] disp8
547 emit_int8(0x40 | regenc | baseenc);
548 emit_int8(disp & 0xFF);
549 } else {
550 // [base + disp32]
551 // [10 reg base] disp32
552 emit_int8(0x80 | regenc | baseenc);
553 emit_data(disp, rspec, disp32_operand);
554 }
555 }
556 } else {
557 if (index->is_valid()) {
558 assert(scale != Address::no_scale, "inconsistent address");
559 // [index*scale + disp]
560 // [00 reg 100][ss index 101] disp32
561 assert(index != rsp, "illegal addressing mode");
562 emit_int8(0x04 | regenc);
563 emit_int8(scale << 6 | indexenc | 0x05);
564 emit_data(disp, rspec, disp32_operand);
565 } else if (rtype != relocInfo::none ) {
566 // [disp] (64bit) RIP-RELATIVE (32bit) abs
567 // [00 000 101] disp32
568
569 emit_int8(0x05 | regenc);
570 // Note that the RIP-rel. correction applies to the generated
571 // disp field, but _not_ to the target address in the rspec.
572
573 // disp was created by converting the target address minus the pc
574 // at the start of the instruction. That needs more correction here.
575 // intptr_t disp = target - next_ip;
576 assert(inst_mark() != NULL, "must be inside InstructionMark");
577 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
578 int64_t adjusted = disp;
579 // Do rip-rel adjustment for 64bit
580 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
581 assert(is_simm32(adjusted),
582 "must be 32bit offset (RIP relative address)");
583 emit_data((int32_t) adjusted, rspec, disp32_operand);
584
585 } else {
586 // 32bit never did this, did everything as the rip-rel/disp code above
587 // [disp] ABSOLUTE
588 // [00 reg 100][00 100 101] disp32
589 emit_int8(0x04 | regenc);
590 emit_int8(0x25);
591 emit_data(disp, rspec, disp32_operand);
592 }
593 }
594 _is_evex_instruction = false;
595 }
596
597 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
598 Address::ScaleFactor scale, int disp,
599 RelocationHolder const& rspec) {
600 if (UseAVX > 2) {
601 int xreg_enc = reg->encoding();
602 if (xreg_enc > 15) {
603 XMMRegister new_reg = as_XMMRegister(xreg_enc & 0xf);
604 emit_operand((Register)new_reg, base, index, scale, disp, rspec);
605 return;
606 }
607 }
608 emit_operand((Register)reg, base, index, scale, disp, rspec);
609 }
610
611 // Secret local extension to Assembler::WhichOperand:
612 #define end_pc_operand (_WhichOperand_limit)
613
614 address Assembler::locate_operand(address inst, WhichOperand which) {
615 // Decode the given instruction, and return the address of
616 // an embedded 32-bit operand word.
617
618 // If "which" is disp32_operand, selects the displacement portion
619 // of an effective address specifier.
620 // If "which" is imm64_operand, selects the trailing immediate constant.
621 // If "which" is call32_operand, selects the displacement of a call or jump.
622 // Caller is responsible for ensuring that there is such an operand,
623 // and that it is 32/64 bits wide.
624
625 // If "which" is end_pc_operand, find the end of the instruction.
626
627 address ip = inst;
628 bool is_64bit = false;
629
630 debug_only(bool has_disp32 = false);
631 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
632
633 again_after_prefix:
634 switch (0xFF & *ip++) {
635
636 // These convenience macros generate groups of "case" labels for the switch.
637 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
638 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
639 case (x)+4: case (x)+5: case (x)+6: case (x)+7
640 #define REP16(x) REP8((x)+0): \
641 case REP8((x)+8)
642
643 case CS_segment:
644 case SS_segment:
645 case DS_segment:
646 case ES_segment:
647 case FS_segment:
648 case GS_segment:
649 // Seems dubious
650 LP64_ONLY(assert(false, "shouldn't have that prefix"));
651 assert(ip == inst+1, "only one prefix allowed");
652 goto again_after_prefix;
653
654 case 0x67:
655 case REX:
656 case REX_B:
657 case REX_X:
658 case REX_XB:
659 case REX_R:
660 case REX_RB:
661 case REX_RX:
662 case REX_RXB:
663 NOT_LP64(assert(false, "64bit prefixes"));
664 goto again_after_prefix;
665
666 case REX_W:
667 case REX_WB:
668 case REX_WX:
669 case REX_WXB:
670 case REX_WR:
671 case REX_WRB:
672 case REX_WRX:
673 case REX_WRXB:
674 NOT_LP64(assert(false, "64bit prefixes"));
675 is_64bit = true;
676 goto again_after_prefix;
677
678 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
679 case 0x88: // movb a, r
680 case 0x89: // movl a, r
681 case 0x8A: // movb r, a
682 case 0x8B: // movl r, a
683 case 0x8F: // popl a
684 debug_only(has_disp32 = true);
685 break;
686
687 case 0x68: // pushq #32
688 if (which == end_pc_operand) {
689 return ip + 4;
690 }
691 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
692 return ip; // not produced by emit_operand
693
694 case 0x66: // movw ... (size prefix)
695 again_after_size_prefix2:
696 switch (0xFF & *ip++) {
697 case REX:
698 case REX_B:
699 case REX_X:
700 case REX_XB:
701 case REX_R:
702 case REX_RB:
703 case REX_RX:
704 case REX_RXB:
705 case REX_W:
706 case REX_WB:
707 case REX_WX:
708 case REX_WXB:
709 case REX_WR:
710 case REX_WRB:
711 case REX_WRX:
712 case REX_WRXB:
713 NOT_LP64(assert(false, "64bit prefix found"));
714 goto again_after_size_prefix2;
715 case 0x8B: // movw r, a
716 case 0x89: // movw a, r
717 debug_only(has_disp32 = true);
718 break;
719 case 0xC7: // movw a, #16
720 debug_only(has_disp32 = true);
721 tail_size = 2; // the imm16
722 break;
723 case 0x0F: // several SSE/SSE2 variants
724 ip--; // reparse the 0x0F
725 goto again_after_prefix;
726 default:
727 ShouldNotReachHere();
728 }
729 break;
730
731 case REP8(0xB8): // movl/q r, #32/#64(oop?)
732 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
733 // these asserts are somewhat nonsensical
734 #ifndef _LP64
735 assert(which == imm_operand || which == disp32_operand,
736 err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)));
737 #else
738 assert((which == call32_operand || which == imm_operand) && is_64bit ||
739 which == narrow_oop_operand && !is_64bit,
740 err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip)));
741 #endif // _LP64
742 return ip;
743
744 case 0x69: // imul r, a, #32
745 case 0xC7: // movl a, #32(oop?)
746 tail_size = 4;
747 debug_only(has_disp32 = true); // has both kinds of operands!
748 break;
749
750 case 0x0F: // movx..., etc.
751 switch (0xFF & *ip++) {
752 case 0x3A: // pcmpestri
753 tail_size = 1;
754 case 0x38: // ptest, pmovzxbw
755 ip++; // skip opcode
756 debug_only(has_disp32 = true); // has both kinds of operands!
757 break;
758
759 case 0x70: // pshufd r, r/a, #8
760 debug_only(has_disp32 = true); // has both kinds of operands!
761 case 0x73: // psrldq r, #8
762 tail_size = 1;
763 break;
764
765 case 0x12: // movlps
766 case 0x28: // movaps
767 case 0x2E: // ucomiss
768 case 0x2F: // comiss
769 case 0x54: // andps
770 case 0x55: // andnps
771 case 0x56: // orps
772 case 0x57: // xorps
773 case 0x59: //mulpd
774 case 0x6E: // movd
775 case 0x7E: // movd
776 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
777 debug_only(has_disp32 = true);
778 break;
779
780 case 0xAD: // shrd r, a, %cl
781 case 0xAF: // imul r, a
782 case 0xBE: // movsbl r, a (movsxb)
783 case 0xBF: // movswl r, a (movsxw)
784 case 0xB6: // movzbl r, a (movzxb)
785 case 0xB7: // movzwl r, a (movzxw)
786 case REP16(0x40): // cmovl cc, r, a
787 case 0xB0: // cmpxchgb
788 case 0xB1: // cmpxchg
789 case 0xC1: // xaddl
790 case 0xC7: // cmpxchg8
791 case REP16(0x90): // setcc a
792 debug_only(has_disp32 = true);
793 // fall out of the switch to decode the address
794 break;
795
796 case 0xC4: // pinsrw r, a, #8
797 debug_only(has_disp32 = true);
798 case 0xC5: // pextrw r, r, #8
799 tail_size = 1; // the imm8
800 break;
801
802 case 0xAC: // shrd r, a, #8
803 debug_only(has_disp32 = true);
804 tail_size = 1; // the imm8
805 break;
806
807 case REP16(0x80): // jcc rdisp32
808 if (which == end_pc_operand) return ip + 4;
809 assert(which == call32_operand, "jcc has no disp32 or imm");
810 return ip;
811 default:
812 ShouldNotReachHere();
813 }
814 break;
815
816 case 0x81: // addl a, #32; addl r, #32
817 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
818 // on 32bit in the case of cmpl, the imm might be an oop
819 tail_size = 4;
820 debug_only(has_disp32 = true); // has both kinds of operands!
821 break;
822
823 case 0x83: // addl a, #8; addl r, #8
824 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
825 debug_only(has_disp32 = true); // has both kinds of operands!
826 tail_size = 1;
827 break;
828
829 case 0x9B:
830 switch (0xFF & *ip++) {
831 case 0xD9: // fnstcw a
832 debug_only(has_disp32 = true);
833 break;
834 default:
835 ShouldNotReachHere();
836 }
837 break;
838
839 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
840 case REP4(0x10): // adc...
841 case REP4(0x20): // and...
842 case REP4(0x30): // xor...
843 case REP4(0x08): // or...
844 case REP4(0x18): // sbb...
845 case REP4(0x28): // sub...
846 case 0xF7: // mull a
847 case 0x8D: // lea r, a
848 case 0x87: // xchg r, a
849 case REP4(0x38): // cmp...
850 case 0x85: // test r, a
851 debug_only(has_disp32 = true); // has both kinds of operands!
852 break;
853
854 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
855 case 0xC6: // movb a, #8
856 case 0x80: // cmpb a, #8
857 case 0x6B: // imul r, a, #8
858 debug_only(has_disp32 = true); // has both kinds of operands!
859 tail_size = 1; // the imm8
860 break;
861
862 case 0xC4: // VEX_3bytes
863 case 0xC5: // VEX_2bytes
864 assert((UseAVX > 0), "shouldn't have VEX prefix");
865 assert(ip == inst+1, "no prefixes allowed");
866 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
867 // but they have prefix 0x0F and processed when 0x0F processed above.
868 //
869 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
870 // instructions (these instructions are not supported in 64-bit mode).
871 // To distinguish them bits [7:6] are set in the VEX second byte since
872 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
873 // those VEX bits REX and vvvv bits are inverted.
874 //
875 // Fortunately C2 doesn't generate these instructions so we don't need
876 // to check for them in product version.
877
878 // Check second byte
879 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
880
881 // First byte
882 if ((0xFF & *inst) == VEX_3bytes) {
883 ip++; // third byte
884 is_64bit = ((VEX_W & *ip) == VEX_W);
885 }
886 ip++; // opcode
887 // To find the end of instruction (which == end_pc_operand).
888 switch (0xFF & *ip) {
889 case 0x61: // pcmpestri r, r/a, #8
890 case 0x70: // pshufd r, r/a, #8
891 case 0x73: // psrldq r, #8
892 tail_size = 1; // the imm8
893 break;
894 default:
895 break;
896 }
897 ip++; // skip opcode
898 debug_only(has_disp32 = true); // has both kinds of operands!
899 break;
900
901 case 0x62: // EVEX_4bytes
902 assert((UseAVX > 0), "shouldn't have EVEX prefix");
903 assert(ip == inst+1, "no prefixes allowed");
904 // no EVEX collisions, all instructions that have 0x62 opcodes
905 // have EVEX versions and are subopcodes of 0x66
906 ip++; // skip P0 and exmaine W in P1
907 is_64bit = ((VEX_W & *ip) == VEX_W);
908 ip++; // move to P2
909 ip++; // skip P2, move to opcode
910 // To find the end of instruction (which == end_pc_operand).
911 switch (0xFF & *ip) {
912 case 0x61: // pcmpestri r, r/a, #8
913 case 0x70: // pshufd r, r/a, #8
914 case 0x73: // psrldq r, #8
915 tail_size = 1; // the imm8
916 break;
917 default:
918 break;
919 }
920 ip++; // skip opcode
921 debug_only(has_disp32 = true); // has both kinds of operands!
922 break;
923
924 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
925 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
926 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
927 case 0xDD: // fld_d a; fst_d a; fstp_d a
928 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
929 case 0xDF: // fild_d a; fistp_d a
930 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
931 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
932 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
933 debug_only(has_disp32 = true);
934 break;
935
936 case 0xE8: // call rdisp32
937 case 0xE9: // jmp rdisp32
938 if (which == end_pc_operand) return ip + 4;
939 assert(which == call32_operand, "call has no disp32 or imm");
940 return ip;
941
942 case 0xF0: // Lock
943 assert(os::is_MP(), "only on MP");
944 goto again_after_prefix;
945
946 case 0xF3: // For SSE
947 case 0xF2: // For SSE2
948 switch (0xFF & *ip++) {
949 case REX:
950 case REX_B:
951 case REX_X:
952 case REX_XB:
953 case REX_R:
954 case REX_RB:
955 case REX_RX:
956 case REX_RXB:
957 case REX_W:
958 case REX_WB:
959 case REX_WX:
960 case REX_WXB:
961 case REX_WR:
962 case REX_WRB:
963 case REX_WRX:
964 case REX_WRXB:
965 NOT_LP64(assert(false, "found 64bit prefix"));
966 ip++;
967 default:
968 ip++;
969 }
970 debug_only(has_disp32 = true); // has both kinds of operands!
971 break;
972
973 default:
974 ShouldNotReachHere();
975
976 #undef REP8
977 #undef REP16
978 }
979
980 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
981 #ifdef _LP64
982 assert(which != imm_operand, "instruction is not a movq reg, imm64");
983 #else
984 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
985 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
986 #endif // LP64
987 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
988
989 // parse the output of emit_operand
990 int op2 = 0xFF & *ip++;
991 int base = op2 & 0x07;
992 int op3 = -1;
993 const int b100 = 4;
994 const int b101 = 5;
995 if (base == b100 && (op2 >> 6) != 3) {
996 op3 = 0xFF & *ip++;
997 base = op3 & 0x07; // refetch the base
998 }
999 // now ip points at the disp (if any)
1000
1001 switch (op2 >> 6) {
1002 case 0:
1003 // [00 reg 100][ss index base]
1004 // [00 reg 100][00 100 esp]
1005 // [00 reg base]
1006 // [00 reg 100][ss index 101][disp32]
1007 // [00 reg 101] [disp32]
1008
1009 if (base == b101) {
1010 if (which == disp32_operand)
1011 return ip; // caller wants the disp32
1012 ip += 4; // skip the disp32
1013 }
1014 break;
1015
1016 case 1:
1017 // [01 reg 100][ss index base][disp8]
1018 // [01 reg 100][00 100 esp][disp8]
1019 // [01 reg base] [disp8]
1020 ip += 1; // skip the disp8
1021 break;
1022
1023 case 2:
1024 // [10 reg 100][ss index base][disp32]
1025 // [10 reg 100][00 100 esp][disp32]
1026 // [10 reg base] [disp32]
1027 if (which == disp32_operand)
1028 return ip; // caller wants the disp32
1029 ip += 4; // skip the disp32
1030 break;
1031
1032 case 3:
1033 // [11 reg base] (not a memory addressing mode)
1034 break;
1035 }
1036
1037 if (which == end_pc_operand) {
1038 return ip + tail_size;
1039 }
1040
1041 #ifdef _LP64
1042 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
1043 #else
1044 assert(which == imm_operand, "instruction has only an imm field");
1045 #endif // LP64
1046 return ip;
1047 }
1048
1049 address Assembler::locate_next_instruction(address inst) {
1050 // Secretly share code with locate_operand:
1051 return locate_operand(inst, end_pc_operand);
1052 }
1053
1054
1055 #ifdef ASSERT
1056 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
1057 address inst = inst_mark();
1058 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
1059 address opnd;
1060
1061 Relocation* r = rspec.reloc();
1062 if (r->type() == relocInfo::none) {
1063 return;
1064 } else if (r->is_call() || format == call32_operand) {
1065 // assert(format == imm32_operand, "cannot specify a nonzero format");
1066 opnd = locate_operand(inst, call32_operand);
1067 } else if (r->is_data()) {
1068 assert(format == imm_operand || format == disp32_operand
1069 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
1070 opnd = locate_operand(inst, (WhichOperand)format);
1071 } else {
1072 assert(format == imm_operand, "cannot specify a format");
1073 return;
1074 }
1075 assert(opnd == pc(), "must put operand where relocs can find it");
1076 }
1077 #endif // ASSERT
1078
1079 void Assembler::emit_operand32(Register reg, Address adr) {
1080 assert(reg->encoding() < 8, "no extended registers");
1081 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1082 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1083 adr._rspec);
1084 }
1085
1086 void Assembler::emit_operand(Register reg, Address adr,
1087 int rip_relative_correction) {
1088 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1089 adr._rspec,
1090 rip_relative_correction);
1091 }
1092
1093 void Assembler::emit_operand(XMMRegister reg, Address adr) {
1094 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
1095 adr._rspec);
1096 }
1097
1098 // MMX operations
1099 void Assembler::emit_operand(MMXRegister reg, Address adr) {
1100 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1101 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1102 }
1103
1104 // work around gcc (3.2.1-7a) bug
1105 void Assembler::emit_operand(Address adr, MMXRegister reg) {
1106 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
1107 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
1108 }
1109
1110
1111 void Assembler::emit_farith(int b1, int b2, int i) {
1112 assert(isByte(b1) && isByte(b2), "wrong opcode");
1113 assert(0 <= i && i < 8, "illegal stack offset");
1114 emit_int8(b1);
1115 emit_int8(b2 + i);
1116 }
1117
1118
1119 // Now the Assembler instructions (identical for 32/64 bits)
1120
1121 void Assembler::adcl(Address dst, int32_t imm32) {
1122 InstructionMark im(this);
1123 prefix(dst);
1124 emit_arith_operand(0x81, rdx, dst, imm32);
1125 }
1126
1127 void Assembler::adcl(Address dst, Register src) {
1128 InstructionMark im(this);
1129 prefix(dst, src);
1130 emit_int8(0x11);
1131 emit_operand(src, dst);
1132 }
1133
1134 void Assembler::adcl(Register dst, int32_t imm32) {
1135 prefix(dst);
1136 emit_arith(0x81, 0xD0, dst, imm32);
1137 }
1138
1139 void Assembler::adcl(Register dst, Address src) {
1140 InstructionMark im(this);
1141 prefix(src, dst);
1142 emit_int8(0x13);
1143 emit_operand(dst, src);
1144 }
1145
1146 void Assembler::adcl(Register dst, Register src) {
1147 (void) prefix_and_encode(dst->encoding(), src->encoding());
1148 emit_arith(0x13, 0xC0, dst, src);
1149 }
1150
1151 void Assembler::addl(Address dst, int32_t imm32) {
1152 InstructionMark im(this);
1153 prefix(dst);
1154 emit_arith_operand(0x81, rax, dst, imm32);
1155 }
1156
1157 void Assembler::addl(Address dst, Register src) {
1158 InstructionMark im(this);
1159 prefix(dst, src);
1160 emit_int8(0x01);
1161 emit_operand(src, dst);
1162 }
1163
1164 void Assembler::addl(Register dst, int32_t imm32) {
1165 prefix(dst);
1166 emit_arith(0x81, 0xC0, dst, imm32);
1167 }
1168
1169 void Assembler::addl(Register dst, Address src) {
1170 InstructionMark im(this);
1171 prefix(src, dst);
1172 emit_int8(0x03);
1173 emit_operand(dst, src);
1174 }
1175
1176 void Assembler::addl(Register dst, Register src) {
1177 (void) prefix_and_encode(dst->encoding(), src->encoding());
1178 emit_arith(0x03, 0xC0, dst, src);
1179 }
1180
1181 void Assembler::addr_nop_4() {
1182 assert(UseAddressNop, "no CPU support");
1183 // 4 bytes: NOP DWORD PTR [EAX+0]
1184 emit_int8(0x0F);
1185 emit_int8(0x1F);
1186 emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
1187 emit_int8(0); // 8-bits offset (1 byte)
1188 }
1189
1190 void Assembler::addr_nop_5() {
1191 assert(UseAddressNop, "no CPU support");
1192 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
1193 emit_int8(0x0F);
1194 emit_int8(0x1F);
1195 emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
1196 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1197 emit_int8(0); // 8-bits offset (1 byte)
1198 }
1199
1200 void Assembler::addr_nop_7() {
1201 assert(UseAddressNop, "no CPU support");
1202 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
1203 emit_int8(0x0F);
1204 emit_int8(0x1F);
1205 emit_int8((unsigned char)0x80);
1206 // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
1207 emit_int32(0); // 32-bits offset (4 bytes)
1208 }
1209
1210 void Assembler::addr_nop_8() {
1211 assert(UseAddressNop, "no CPU support");
1212 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
1213 emit_int8(0x0F);
1214 emit_int8(0x1F);
1215 emit_int8((unsigned char)0x84);
1216 // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
1217 emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
1218 emit_int32(0); // 32-bits offset (4 bytes)
1219 }
1220
1221 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
1222 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1223 if (VM_Version::supports_evex()) {
1224 emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
1225 } else {
1226 emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
1227 }
1228 }
1229
1230 void Assembler::addsd(XMMRegister dst, Address src) {
1231 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1232 if (VM_Version::supports_evex()) {
1233 _tuple_type = EVEX_T1S;
1234 _input_size_in_bits = EVEX_64bit;
1235 emit_simd_arith_q(0x58, dst, src, VEX_SIMD_F2);
1236 } else {
1237 emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
1238 }
1239 }
1240
1241 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1242 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1243 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1244 }
1245
1246 void Assembler::addss(XMMRegister dst, Address src) {
1247 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1248 if (VM_Version::supports_evex()) {
1249 _tuple_type = EVEX_T1S;
1250 _input_size_in_bits = EVEX_32bit;
1251 }
1252 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1253 }
1254
1255 void Assembler::aesdec(XMMRegister dst, Address src) {
1256 assert(VM_Version::supports_aes(), "");
1257 InstructionMark im(this);
1258 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1259 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1260 emit_int8((unsigned char)0xDE);
1261 emit_operand(dst, src);
1262 }
1263
1264 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1265 assert(VM_Version::supports_aes(), "");
1266 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1267 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1268 emit_int8((unsigned char)0xDE);
1269 emit_int8(0xC0 | encode);
1270 }
1271
1272 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1273 assert(VM_Version::supports_aes(), "");
1274 InstructionMark im(this);
1275 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1276 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1277 emit_int8((unsigned char)0xDF);
1278 emit_operand(dst, src);
1279 }
1280
1281 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1282 assert(VM_Version::supports_aes(), "");
1283 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1284 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1285 emit_int8((unsigned char)0xDF);
1286 emit_int8((unsigned char)(0xC0 | encode));
1287 }
1288
1289 void Assembler::aesenc(XMMRegister dst, Address src) {
1290 assert(VM_Version::supports_aes(), "");
1291 InstructionMark im(this);
1292 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1293 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1294 emit_int8((unsigned char)0xDC);
1295 emit_operand(dst, src);
1296 }
1297
1298 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1299 assert(VM_Version::supports_aes(), "");
1300 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1301 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1302 emit_int8((unsigned char)0xDC);
1303 emit_int8(0xC0 | encode);
1304 }
1305
1306 void Assembler::aesenclast(XMMRegister dst, Address src) {
1307 assert(VM_Version::supports_aes(), "");
1308 InstructionMark im(this);
1309 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1310 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1311 emit_int8((unsigned char)0xDD);
1312 emit_operand(dst, src);
1313 }
1314
1315 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1316 assert(VM_Version::supports_aes(), "");
1317 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
1318 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
1319 emit_int8((unsigned char)0xDD);
1320 emit_int8((unsigned char)(0xC0 | encode));
1321 }
1322
1323 void Assembler::andl(Address dst, int32_t imm32) {
1324 InstructionMark im(this);
1325 prefix(dst);
1326 emit_int8((unsigned char)0x81);
1327 emit_operand(rsp, dst, 4);
1328 emit_int32(imm32);
1329 }
1330
1331 void Assembler::andl(Register dst, int32_t imm32) {
1332 prefix(dst);
1333 emit_arith(0x81, 0xE0, dst, imm32);
1334 }
1335
1336 void Assembler::andl(Register dst, Address src) {
1337 InstructionMark im(this);
1338 prefix(src, dst);
1339 emit_int8(0x23);
1340 emit_operand(dst, src);
1341 }
1342
1343 void Assembler::andl(Register dst, Register src) {
1344 (void) prefix_and_encode(dst->encoding(), src->encoding());
1345 emit_arith(0x23, 0xC0, dst, src);
1346 }
1347
1348 void Assembler::andnl(Register dst, Register src1, Register src2) {
1349 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1350 int encode = vex_prefix_0F38_and_encode_legacy(dst, src1, src2);
1351 emit_int8((unsigned char)0xF2);
1352 emit_int8((unsigned char)(0xC0 | encode));
1353 }
1354
1355 void Assembler::andnl(Register dst, Register src1, Address src2) {
1356 InstructionMark im(this);
1357 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1358 vex_prefix_0F38_legacy(dst, src1, src2);
1359 emit_int8((unsigned char)0xF2);
1360 emit_operand(dst, src2);
1361 }
1362
1363 void Assembler::bsfl(Register dst, Register src) {
1364 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1365 emit_int8(0x0F);
1366 emit_int8((unsigned char)0xBC);
1367 emit_int8((unsigned char)(0xC0 | encode));
1368 }
1369
1370 void Assembler::bsrl(Register dst, Register src) {
1371 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1372 emit_int8(0x0F);
1373 emit_int8((unsigned char)0xBD);
1374 emit_int8((unsigned char)(0xC0 | encode));
1375 }
1376
1377 void Assembler::bswapl(Register reg) { // bswap
1378 int encode = prefix_and_encode(reg->encoding());
1379 emit_int8(0x0F);
1380 emit_int8((unsigned char)(0xC8 | encode));
1381 }
1382
1383 void Assembler::blsil(Register dst, Register src) {
1384 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1385 int encode = vex_prefix_0F38_and_encode_legacy(rbx, dst, src);
1386 emit_int8((unsigned char)0xF3);
1387 emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389
1390 void Assembler::blsil(Register dst, Address src) {
1391 InstructionMark im(this);
1392 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1393 vex_prefix_0F38_legacy(rbx, dst, src);
1394 emit_int8((unsigned char)0xF3);
1395 emit_operand(rbx, src);
1396 }
1397
1398 void Assembler::blsmskl(Register dst, Register src) {
1399 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1400 int encode = vex_prefix_0F38_and_encode_legacy(rdx, dst, src);
1401 emit_int8((unsigned char)0xF3);
1402 emit_int8((unsigned char)(0xC0 | encode));
1403 }
1404
1405 void Assembler::blsmskl(Register dst, Address src) {
1406 InstructionMark im(this);
1407 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1408 vex_prefix_0F38_legacy(rdx, dst, src);
1409 emit_int8((unsigned char)0xF3);
1410 emit_operand(rdx, src);
1411 }
1412
1413 void Assembler::blsrl(Register dst, Register src) {
1414 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1415 int encode = vex_prefix_0F38_and_encode_legacy(rcx, dst, src);
1416 emit_int8((unsigned char)0xF3);
1417 emit_int8((unsigned char)(0xC0 | encode));
1418 }
1419
1420 void Assembler::blsrl(Register dst, Address src) {
1421 InstructionMark im(this);
1422 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1423 vex_prefix_0F38_legacy(rcx, dst, src);
1424 emit_int8((unsigned char)0xF3);
1425 emit_operand(rcx, src);
1426 }
1427
1428 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1429 // suspect disp32 is always good
1430 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1431
1432 if (L.is_bound()) {
1433 const int long_size = 5;
1434 int offs = (int)( target(L) - pc() );
1435 assert(offs <= 0, "assembler error");
1436 InstructionMark im(this);
1437 // 1110 1000 #32-bit disp
1438 emit_int8((unsigned char)0xE8);
1439 emit_data(offs - long_size, rtype, operand);
1440 } else {
1441 InstructionMark im(this);
1442 // 1110 1000 #32-bit disp
1443 L.add_patch_at(code(), locator());
1444
1445 emit_int8((unsigned char)0xE8);
1446 emit_data(int(0), rtype, operand);
1447 }
1448 }
1449
1450 void Assembler::call(Register dst) {
1451 int encode = prefix_and_encode(dst->encoding());
1452 emit_int8((unsigned char)0xFF);
1453 emit_int8((unsigned char)(0xD0 | encode));
1454 }
1455
1456
1457 void Assembler::call(Address adr) {
1458 InstructionMark im(this);
1459 prefix(adr);
1460 emit_int8((unsigned char)0xFF);
1461 emit_operand(rdx, adr);
1462 }
1463
1464 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1465 assert(entry != NULL, "call most probably wrong");
1466 InstructionMark im(this);
1467 emit_int8((unsigned char)0xE8);
1468 intptr_t disp = entry - (pc() + sizeof(int32_t));
1469 assert(is_simm32(disp), "must be 32bit offset (call2)");
1470 // Technically, should use call32_operand, but this format is
1471 // implied by the fact that we're emitting a call instruction.
1472
1473 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1474 emit_data((int) disp, rspec, operand);
1475 }
1476
1477 void Assembler::cdql() {
1478 emit_int8((unsigned char)0x99);
1479 }
1480
1481 void Assembler::cld() {
1482 emit_int8((unsigned char)0xFC);
1483 }
1484
1485 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1486 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1487 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1488 emit_int8(0x0F);
1489 emit_int8(0x40 | cc);
1490 emit_int8((unsigned char)(0xC0 | encode));
1491 }
1492
1493
1494 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1495 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1496 prefix(src, dst);
1497 emit_int8(0x0F);
1498 emit_int8(0x40 | cc);
1499 emit_operand(dst, src);
1500 }
1501
1502 void Assembler::cmpb(Address dst, int imm8) {
1503 InstructionMark im(this);
1504 prefix(dst);
1505 emit_int8((unsigned char)0x80);
1506 emit_operand(rdi, dst, 1);
1507 emit_int8(imm8);
1508 }
1509
1510 void Assembler::cmpl(Address dst, int32_t imm32) {
1511 InstructionMark im(this);
1512 prefix(dst);
1513 emit_int8((unsigned char)0x81);
1514 emit_operand(rdi, dst, 4);
1515 emit_int32(imm32);
1516 }
1517
1518 void Assembler::cmpl(Register dst, int32_t imm32) {
1519 prefix(dst);
1520 emit_arith(0x81, 0xF8, dst, imm32);
1521 }
1522
1523 void Assembler::cmpl(Register dst, Register src) {
1524 (void) prefix_and_encode(dst->encoding(), src->encoding());
1525 emit_arith(0x3B, 0xC0, dst, src);
1526 }
1527
1528
1529 void Assembler::cmpl(Register dst, Address src) {
1530 InstructionMark im(this);
1531 prefix(src, dst);
1532 emit_int8((unsigned char)0x3B);
1533 emit_operand(dst, src);
1534 }
1535
1536 void Assembler::cmpw(Address dst, int imm16) {
1537 InstructionMark im(this);
1538 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1539 emit_int8(0x66);
1540 emit_int8((unsigned char)0x81);
1541 emit_operand(rdi, dst, 2);
1542 emit_int16(imm16);
1543 }
1544
1545 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1546 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1547 // The ZF is set if the compared values were equal, and cleared otherwise.
1548 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1549 InstructionMark im(this);
1550 prefix(adr, reg);
1551 emit_int8(0x0F);
1552 emit_int8((unsigned char)0xB1);
1553 emit_operand(reg, adr);
1554 }
1555
1556 // The 8-bit cmpxchg compares the value at adr with the contents of rax,
1557 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1558 // The ZF is set if the compared values were equal, and cleared otherwise.
1559 void Assembler::cmpxchgb(Register reg, Address adr) { // cmpxchg
1560 InstructionMark im(this);
1561 prefix(adr, reg, true);
1562 emit_int8(0x0F);
1563 emit_int8((unsigned char)0xB0);
1564 emit_operand(reg, adr);
1565 }
1566
1567 void Assembler::comisd(XMMRegister dst, Address src) {
1568 // NOTE: dbx seems to decode this as comiss even though the
1569 // 0x66 is there. Strangly ucomisd comes out correct
1570 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1571 if (VM_Version::supports_evex()) {
1572 _tuple_type = EVEX_T1S;
1573 _input_size_in_bits = EVEX_64bit;
1574 emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
1575 } else {
1576 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1577 }
1578 }
1579
1580 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1581 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1582 if (VM_Version::supports_evex()) {
1583 emit_simd_arith_nonds_q(0x2F, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
1584 } else {
1585 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1586 }
1587 }
1588
1589 void Assembler::comiss(XMMRegister dst, Address src) {
1590 if (VM_Version::supports_evex()) {
1591 _tuple_type = EVEX_T1S;
1592 _input_size_in_bits = EVEX_32bit;
1593 }
1594 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1595 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
1596 }
1597
1598 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1599 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1600 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
1601 }
1602
1603 void Assembler::cpuid() {
1604 emit_int8(0x0F);
1605 emit_int8((unsigned char)0xA2);
1606 }
1607
1608 // Opcode / Instruction Op / En 64 - Bit Mode Compat / Leg Mode Description Implemented
1609 // F2 0F 38 F0 / r CRC32 r32, r / m8 RM Valid Valid Accumulate CRC32 on r / m8. v
1610 // F2 REX 0F 38 F0 / r CRC32 r32, r / m8* RM Valid N.E. Accumulate CRC32 on r / m8. -
1611 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E. Accumulate CRC32 on r / m8. -
1612 //
1613 // F2 0F 38 F1 / r CRC32 r32, r / m16 RM Valid Valid Accumulate CRC32 on r / m16. v
1614 //
1615 // F2 0F 38 F1 / r CRC32 r32, r / m32 RM Valid Valid Accumulate CRC32 on r / m32. v
1616 //
1617 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E. Accumulate CRC32 on r / m64. v
1618 void Assembler::crc32(Register crc, Register v, int8_t sizeInBytes) {
1619 assert(VM_Version::supports_sse4_2(), "");
1620 int8_t w = 0x01;
1621 Prefix p = Prefix_EMPTY;
1622
1623 emit_int8((int8_t)0xF2);
1624 switch (sizeInBytes) {
1625 case 1:
1626 w = 0;
1627 break;
1628 case 2:
1629 case 4:
1630 break;
1631 LP64_ONLY(case 8:)
1632 // This instruction is not valid in 32 bits
1633 // Note:
1634 // http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-instruction-set-reference-manual-325383.pdf
1635 //
1636 // Page B - 72 Vol. 2C says
1637 // qwreg2 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : 11 qwreg1 qwreg2
1638 // mem64 to qwreg 1111 0010 : 0100 1R0B : 0000 1111 : 0011 1000 : 1111 0000 : mod qwreg r / m
1639 // F0!!!
1640 // while 3 - 208 Vol. 2A
1641 // F2 REX.W 0F 38 F1 / r CRC32 r64, r / m64 RM Valid N.E.Accumulate CRC32 on r / m64.
1642 //
1643 // the 0 on a last bit is reserved for a different flavor of this instruction :
1644 // F2 REX.W 0F 38 F0 / r CRC32 r64, r / m8 RM Valid N.E.Accumulate CRC32 on r / m8.
1645 p = REX_W;
1646 break;
1647 default:
1648 assert(0, "Unsupported value for a sizeInBytes argument");
1649 break;
1650 }
1651 LP64_ONLY(prefix(crc, v, p);)
1652 emit_int8((int8_t)0x0F);
1653 emit_int8(0x38);
1654 emit_int8((int8_t)(0xF0 | w));
1655 emit_int8(0xC0 | ((crc->encoding() & 0x7) << 3) | (v->encoding() & 7));
1656 }
1657
1658 void Assembler::crc32(Register crc, Address adr, int8_t sizeInBytes) {
1659 assert(VM_Version::supports_sse4_2(), "");
1660 InstructionMark im(this);
1661 int8_t w = 0x01;
1662 Prefix p = Prefix_EMPTY;
1663
1664 emit_int8((int8_t)0xF2);
1665 switch (sizeInBytes) {
1666 case 1:
1667 w = 0;
1668 break;
1669 case 2:
1670 case 4:
1671 break;
1672 LP64_ONLY(case 8:)
1673 // This instruction is not valid in 32 bits
1674 p = REX_W;
1675 break;
1676 default:
1677 assert(0, "Unsupported value for a sizeInBytes argument");
1678 break;
1679 }
1680 LP64_ONLY(prefix(crc, adr, p);)
1681 emit_int8((int8_t)0x0F);
1682 emit_int8(0x38);
1683 emit_int8((int8_t)(0xF0 | w));
1684 emit_operand(crc, adr);
1685 }
1686
1687 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1688 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1689 emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3, /* no_mask_reg */ false, /* legacy_mode */ true);
1690 }
1691
1692 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1693 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1694 emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ true);
1695 }
1696
1697 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1698 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1699 if (VM_Version::supports_evex()) {
1700 emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
1701 } else {
1702 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1703 }
1704 }
1705
1706 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1707 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1708 if (VM_Version::supports_evex()) {
1709 _tuple_type = EVEX_T1F;
1710 _input_size_in_bits = EVEX_64bit;
1711 emit_simd_arith_q(0x5A, dst, src, VEX_SIMD_F2);
1712 } else {
1713 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1714 }
1715 }
1716
1717 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1718 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1719 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2, VM_Version::supports_evex());
1720 emit_int8(0x2A);
1721 emit_int8((unsigned char)(0xC0 | encode));
1722 }
1723
1724 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1725 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1726 if (VM_Version::supports_evex()) {
1727 _tuple_type = EVEX_T1S;
1728 _input_size_in_bits = EVEX_32bit;
1729 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
1730 } else {
1731 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
1732 }
1733 }
1734
1735 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1736 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1737 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
1738 emit_int8(0x2A);
1739 emit_int8((unsigned char)(0xC0 | encode));
1740 }
1741
1742 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1743 if (VM_Version::supports_evex()) {
1744 _tuple_type = EVEX_T1S;
1745 _input_size_in_bits = EVEX_32bit;
1746 }
1747 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1748 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
1749 }
1750
1751 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
1752 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1753 int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
1754 emit_int8(0x2A);
1755 emit_int8((unsigned char)(0xC0 | encode));
1756 }
1757
1758 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1759 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1760 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1761 }
1762
1763 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1764 if (VM_Version::supports_evex()) {
1765 _tuple_type = EVEX_T1S;
1766 _input_size_in_bits = EVEX_32bit;
1767 }
1768 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1769 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1770 }
1771
1772
1773 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1774 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1775 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);
1776 emit_int8(0x2C);
1777 emit_int8((unsigned char)(0xC0 | encode));
1778 }
1779
1780 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1781 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1782 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);
1783 emit_int8(0x2C);
1784 emit_int8((unsigned char)(0xC0 | encode));
1785 }
1786
1787 void Assembler::decl(Address dst) {
1788 // Don't use it directly. Use MacroAssembler::decrement() instead.
1789 InstructionMark im(this);
1790 prefix(dst);
1791 emit_int8((unsigned char)0xFF);
1792 emit_operand(rcx, dst);
1793 }
1794
1795 void Assembler::divsd(XMMRegister dst, Address src) {
1796 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1797 if (VM_Version::supports_evex()) {
1798 _tuple_type = EVEX_T1S;
1799 _input_size_in_bits = EVEX_64bit;
1800 emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
1801 } else {
1802 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1803 }
1804 }
1805
1806 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1807 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1808 if (VM_Version::supports_evex()) {
1809 emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_F2);
1810 } else {
1811 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1812 }
1813 }
1814
1815 void Assembler::divss(XMMRegister dst, Address src) {
1816 if (VM_Version::supports_evex()) {
1817 _tuple_type = EVEX_T1S;
1818 _input_size_in_bits = EVEX_32bit;
1819 }
1820 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1821 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1822 }
1823
1824 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1825 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1826 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1827 }
1828
1829 void Assembler::emms() {
1830 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1831 emit_int8(0x0F);
1832 emit_int8(0x77);
1833 }
1834
1835 void Assembler::hlt() {
1836 emit_int8((unsigned char)0xF4);
1837 }
1838
1839 void Assembler::idivl(Register src) {
1840 int encode = prefix_and_encode(src->encoding());
1841 emit_int8((unsigned char)0xF7);
1842 emit_int8((unsigned char)(0xF8 | encode));
1843 }
1844
1845 void Assembler::divl(Register src) { // Unsigned
1846 int encode = prefix_and_encode(src->encoding());
1847 emit_int8((unsigned char)0xF7);
1848 emit_int8((unsigned char)(0xF0 | encode));
1849 }
1850
1851 void Assembler::imull(Register dst, Register src) {
1852 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1853 emit_int8(0x0F);
1854 emit_int8((unsigned char)0xAF);
1855 emit_int8((unsigned char)(0xC0 | encode));
1856 }
1857
1858
1859 void Assembler::imull(Register dst, Register src, int value) {
1860 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1861 if (is8bit(value)) {
1862 emit_int8(0x6B);
1863 emit_int8((unsigned char)(0xC0 | encode));
1864 emit_int8(value & 0xFF);
1865 } else {
1866 emit_int8(0x69);
1867 emit_int8((unsigned char)(0xC0 | encode));
1868 emit_int32(value);
1869 }
1870 }
1871
1872 void Assembler::imull(Register dst, Address src) {
1873 InstructionMark im(this);
1874 prefix(src, dst);
1875 emit_int8(0x0F);
1876 emit_int8((unsigned char) 0xAF);
1877 emit_operand(dst, src);
1878 }
1879
1880
1881 void Assembler::incl(Address dst) {
1882 // Don't use it directly. Use MacroAssembler::increment() instead.
1883 InstructionMark im(this);
1884 prefix(dst);
1885 emit_int8((unsigned char)0xFF);
1886 emit_operand(rax, dst);
1887 }
1888
1889 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1890 InstructionMark im(this);
1891 assert((0 <= cc) && (cc < 16), "illegal cc");
1892 if (L.is_bound()) {
1893 address dst = target(L);
1894 assert(dst != NULL, "jcc most probably wrong");
1895
1896 const int short_size = 2;
1897 const int long_size = 6;
1898 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1899 if (maybe_short && is8bit(offs - short_size)) {
1900 // 0111 tttn #8-bit disp
1901 emit_int8(0x70 | cc);
1902 emit_int8((offs - short_size) & 0xFF);
1903 } else {
1904 // 0000 1111 1000 tttn #32-bit disp
1905 assert(is_simm32(offs - long_size),
1906 "must be 32bit offset (call4)");
1907 emit_int8(0x0F);
1908 emit_int8((unsigned char)(0x80 | cc));
1909 emit_int32(offs - long_size);
1910 }
1911 } else {
1912 // Note: could eliminate cond. jumps to this jump if condition
1913 // is the same however, seems to be rather unlikely case.
1914 // Note: use jccb() if label to be bound is very close to get
1915 // an 8-bit displacement
1916 L.add_patch_at(code(), locator());
1917 emit_int8(0x0F);
1918 emit_int8((unsigned char)(0x80 | cc));
1919 emit_int32(0);
1920 }
1921 }
1922
1923 void Assembler::jccb(Condition cc, Label& L) {
1924 if (L.is_bound()) {
1925 const int short_size = 2;
1926 address entry = target(L);
1927 #ifdef ASSERT
1928 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1929 intptr_t delta = short_branch_delta();
1930 if (delta != 0) {
1931 dist += (dist < 0 ? (-delta) :delta);
1932 }
1933 assert(is8bit(dist), "Dispacement too large for a short jmp");
1934 #endif
1935 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1936 // 0111 tttn #8-bit disp
1937 emit_int8(0x70 | cc);
1938 emit_int8((offs - short_size) & 0xFF);
1939 } else {
1940 InstructionMark im(this);
1941 L.add_patch_at(code(), locator());
1942 emit_int8(0x70 | cc);
1943 emit_int8(0);
1944 }
1945 }
1946
1947 void Assembler::jmp(Address adr) {
1948 InstructionMark im(this);
1949 prefix(adr);
1950 emit_int8((unsigned char)0xFF);
1951 emit_operand(rsp, adr);
1952 }
1953
1954 void Assembler::jmp(Label& L, bool maybe_short) {
1955 if (L.is_bound()) {
1956 address entry = target(L);
1957 assert(entry != NULL, "jmp most probably wrong");
1958 InstructionMark im(this);
1959 const int short_size = 2;
1960 const int long_size = 5;
1961 intptr_t offs = entry - pc();
1962 if (maybe_short && is8bit(offs - short_size)) {
1963 emit_int8((unsigned char)0xEB);
1964 emit_int8((offs - short_size) & 0xFF);
1965 } else {
1966 emit_int8((unsigned char)0xE9);
1967 emit_int32(offs - long_size);
1968 }
1969 } else {
1970 // By default, forward jumps are always 32-bit displacements, since
1971 // we can't yet know where the label will be bound. If you're sure that
1972 // the forward jump will not run beyond 256 bytes, use jmpb to
1973 // force an 8-bit displacement.
1974 InstructionMark im(this);
1975 L.add_patch_at(code(), locator());
1976 emit_int8((unsigned char)0xE9);
1977 emit_int32(0);
1978 }
1979 }
1980
1981 void Assembler::jmp(Register entry) {
1982 int encode = prefix_and_encode(entry->encoding());
1983 emit_int8((unsigned char)0xFF);
1984 emit_int8((unsigned char)(0xE0 | encode));
1985 }
1986
1987 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
1988 InstructionMark im(this);
1989 emit_int8((unsigned char)0xE9);
1990 assert(dest != NULL, "must have a target");
1991 intptr_t disp = dest - (pc() + sizeof(int32_t));
1992 assert(is_simm32(disp), "must be 32bit offset (jmp)");
1993 emit_data(disp, rspec.reloc(), call32_operand);
1994 }
1995
1996 void Assembler::jmpb(Label& L) {
1997 if (L.is_bound()) {
1998 const int short_size = 2;
1999 address entry = target(L);
2000 assert(entry != NULL, "jmp most probably wrong");
2001 #ifdef ASSERT
2002 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
2003 intptr_t delta = short_branch_delta();
2004 if (delta != 0) {
2005 dist += (dist < 0 ? (-delta) :delta);
2006 }
2007 assert(is8bit(dist), "Dispacement too large for a short jmp");
2008 #endif
2009 intptr_t offs = entry - pc();
2010 emit_int8((unsigned char)0xEB);
2011 emit_int8((offs - short_size) & 0xFF);
2012 } else {
2013 InstructionMark im(this);
2014 L.add_patch_at(code(), locator());
2015 emit_int8((unsigned char)0xEB);
2016 emit_int8(0);
2017 }
2018 }
2019
2020 void Assembler::ldmxcsr( Address src) {
2021 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2022 InstructionMark im(this);
2023 prefix(src);
2024 emit_int8(0x0F);
2025 emit_int8((unsigned char)0xAE);
2026 emit_operand(as_Register(2), src);
2027 }
2028
2029 void Assembler::leal(Register dst, Address src) {
2030 InstructionMark im(this);
2031 #ifdef _LP64
2032 emit_int8(0x67); // addr32
2033 prefix(src, dst);
2034 #endif // LP64
2035 emit_int8((unsigned char)0x8D);
2036 emit_operand(dst, src);
2037 }
2038
2039 void Assembler::lfence() {
2040 emit_int8(0x0F);
2041 emit_int8((unsigned char)0xAE);
2042 emit_int8((unsigned char)0xE8);
2043 }
2044
2045 void Assembler::lock() {
2046 emit_int8((unsigned char)0xF0);
2047 }
2048
2049 void Assembler::lzcntl(Register dst, Register src) {
2050 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
2051 emit_int8((unsigned char)0xF3);
2052 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2053 emit_int8(0x0F);
2054 emit_int8((unsigned char)0xBD);
2055 emit_int8((unsigned char)(0xC0 | encode));
2056 }
2057
2058 // Emit mfence instruction
2059 void Assembler::mfence() {
2060 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
2061 emit_int8(0x0F);
2062 emit_int8((unsigned char)0xAE);
2063 emit_int8((unsigned char)0xF0);
2064 }
2065
2066 void Assembler::mov(Register dst, Register src) {
2067 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
2068 }
2069
2070 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
2071 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2072 if (VM_Version::supports_avx512novl()) {
2073 int vector_len = AVX_512bit;
2074 int dst_enc = dst->encoding();
2075 int src_enc = src->encoding();
2076 int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F,
2077 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2078 emit_int8(0x28);
2079 emit_int8((unsigned char)(0xC0 | encode));
2080 } else if (VM_Version::supports_evex()) {
2081 emit_simd_arith_nonds_q(0x28, dst, src, VEX_SIMD_66);
2082 } else {
2083 emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
2084 }
2085 }
2086
2087 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
2088 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2089 if (VM_Version::supports_avx512novl()) {
2090 int vector_len = AVX_512bit;
2091 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_NONE, vector_len);
2092 emit_int8(0x28);
2093 emit_int8((unsigned char)(0xC0 | encode));
2094 } else {
2095 emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
2096 }
2097 }
2098
2099 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
2100 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2101 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
2102 emit_int8(0x16);
2103 emit_int8((unsigned char)(0xC0 | encode));
2104 }
2105
2106 void Assembler::movb(Register dst, Address src) {
2107 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2108 InstructionMark im(this);
2109 prefix(src, dst, true);
2110 emit_int8((unsigned char)0x8A);
2111 emit_operand(dst, src);
2112 }
2113
2114 void Assembler::kmovql(KRegister dst, KRegister src) {
2115 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2116 int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE,
2117 /* no_mask_reg */ true, VEX_OPCODE_0F, /* rex_w */ true);
2118 emit_int8((unsigned char)0x90);
2119 emit_int8((unsigned char)(0xC0 | encode));
2120 }
2121
2122 void Assembler::kmovql(KRegister dst, Address src) {
2123 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2124 int dst_enc = dst->encoding();
2125 int nds_enc = 0;
2126 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_NONE,
2127 VEX_OPCODE_0F, /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
2128 emit_int8((unsigned char)0x90);
2129 emit_operand((Register)dst, src);
2130 }
2131
2132 void Assembler::kmovql(Address dst, KRegister src) {
2133 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2134 int src_enc = src->encoding();
2135 int nds_enc = 0;
2136 vex_prefix(dst, nds_enc, src_enc, VEX_SIMD_NONE,
2137 VEX_OPCODE_0F, /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_reg_mask */ true);
2138 emit_int8((unsigned char)0x90);
2139 emit_operand((Register)src, dst);
2140 }
2141
2142 void Assembler::kmovql(KRegister dst, Register src) {
2143 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2144 VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2145 int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true,
2146 VEX_OPCODE_0F, /* legacy_mode */ !_legacy_mode_bw);
2147 emit_int8((unsigned char)0x92);
2148 emit_int8((unsigned char)(0xC0 | encode));
2149 }
2150
2151 void Assembler::kmovdl(KRegister dst, Register src) {
2152 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2153 VexSimdPrefix pre = !_legacy_mode_bw ? VEX_SIMD_F2 : VEX_SIMD_NONE;
2154 int encode = kreg_prefix_and_encode(dst, knoreg, src, pre, /* no_mask_reg */ true);
2155 emit_int8((unsigned char)0x92);
2156 emit_int8((unsigned char)(0xC0 | encode));
2157 }
2158
2159 void Assembler::kmovwl(KRegister dst, Register src) {
2160 NOT_LP64(assert(VM_Version::supports_evex(), ""));
2161 int encode = kreg_prefix_and_encode(dst, knoreg, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
2162 emit_int8((unsigned char)0x92);
2163 emit_int8((unsigned char)(0xC0 | encode));
2164 }
2165
2166 void Assembler::movb(Address dst, int imm8) {
2167 InstructionMark im(this);
2168 prefix(dst);
2169 emit_int8((unsigned char)0xC6);
2170 emit_operand(rax, dst, 1);
2171 emit_int8(imm8);
2172 }
2173
2174
2175 void Assembler::movb(Address dst, Register src) {
2176 assert(src->has_byte_register(), "must have byte register");
2177 InstructionMark im(this);
2178 prefix(dst, src, true);
2179 emit_int8((unsigned char)0x88);
2180 emit_operand(src, dst);
2181 }
2182
2183 void Assembler::movdl(XMMRegister dst, Register src) {
2184 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2185 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2186 emit_int8(0x6E);
2187 emit_int8((unsigned char)(0xC0 | encode));
2188 }
2189
2190 void Assembler::movdl(Register dst, XMMRegister src) {
2191 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2192 // swap src/dst to get correct prefix
2193 int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
2194 emit_int8(0x7E);
2195 emit_int8((unsigned char)(0xC0 | encode));
2196 }
2197
2198 void Assembler::movdl(XMMRegister dst, Address src) {
2199 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2200 if (VM_Version::supports_evex()) {
2201 _tuple_type = EVEX_T1S;
2202 _input_size_in_bits = EVEX_32bit;
2203 }
2204 InstructionMark im(this);
2205 simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);
2206 emit_int8(0x6E);
2207 emit_operand(dst, src);
2208 }
2209
2210 void Assembler::movdl(Address dst, XMMRegister src) {
2211 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2212 if (VM_Version::supports_evex()) {
2213 _tuple_type = EVEX_T1S;
2214 _input_size_in_bits = EVEX_32bit;
2215 }
2216 InstructionMark im(this);
2217 simd_prefix(dst, src, VEX_SIMD_66, /* no_reg_mask */ true);
2218 emit_int8(0x7E);
2219 emit_operand(src, dst);
2220 }
2221
2222 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
2223 _instruction_uses_vl = true;
2224 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2225 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
2226 }
2227
2228 void Assembler::movdqa(XMMRegister dst, Address src) {
2229 _instruction_uses_vl = true;
2230 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2231 if (VM_Version::supports_evex()) {
2232 _tuple_type = EVEX_FVM;
2233 }
2234 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
2235 }
2236
2237 void Assembler::movdqu(XMMRegister dst, Address src) {
2238 _instruction_uses_vl = true;
2239 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2240 if (VM_Version::supports_evex()) {
2241 _tuple_type = EVEX_FVM;
2242 }
2243 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
2244 }
2245
2246 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
2247 _instruction_uses_vl = true;
2248 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2249 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
2250 }
2251
2252 void Assembler::movdqu(Address dst, XMMRegister src) {
2253 _instruction_uses_vl = true;
2254 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2255 if (VM_Version::supports_evex()) {
2256 _tuple_type = EVEX_FVM;
2257 }
2258 InstructionMark im(this);
2259 simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);
2260 emit_int8(0x7F);
2261 emit_operand(src, dst);
2262 }
2263
2264 // Move Unaligned 256bit Vector
2265 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
2266 _instruction_uses_vl = true;
2267 assert(UseAVX > 0, "");
2268 int vector_len = AVX_256bit;
2269 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2270 emit_int8(0x6F);
2271 emit_int8((unsigned char)(0xC0 | encode));
2272 }
2273
2274 void Assembler::vmovdqu(XMMRegister dst, Address src) {
2275 _instruction_uses_vl = true;
2276 assert(UseAVX > 0, "");
2277 if (VM_Version::supports_evex()) {
2278 _tuple_type = EVEX_FVM;
2279 }
2280 InstructionMark im(this);
2281 int vector_len = AVX_256bit;
2282 vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2283 emit_int8(0x6F);
2284 emit_operand(dst, src);
2285 }
2286
2287 void Assembler::vmovdqu(Address dst, XMMRegister src) {
2288 _instruction_uses_vl = true;
2289 assert(UseAVX > 0, "");
2290 if (VM_Version::supports_evex()) {
2291 _tuple_type = EVEX_FVM;
2292 }
2293 InstructionMark im(this);
2294 int vector_len = AVX_256bit;
2295 // swap src<->dst for encoding
2296 assert(src != xnoreg, "sanity");
2297 vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2298 emit_int8(0x7F);
2299 emit_operand(src, dst);
2300 }
2301
2302 // Move Unaligned EVEX enabled Vector (programmable : 8,16,32,64)
2303 void Assembler::evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
2304 _instruction_uses_vl = true;
2305 assert(UseAVX > 0, "");
2306 int src_enc = src->encoding();
2307 int dst_enc = dst->encoding();
2308 int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
2309 /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2310 emit_int8(0x6F);
2311 emit_int8((unsigned char)(0xC0 | encode));
2312 }
2313
2314 void Assembler::evmovdqul(XMMRegister dst, Address src, int vector_len) {
2315 _instruction_uses_vl = true;
2316 assert(UseAVX > 0, "");
2317 InstructionMark im(this);
2318 if (VM_Version::supports_evex()) {
2319 _tuple_type = EVEX_FVM;
2320 }
2321 vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2322 emit_int8(0x6F);
2323 emit_operand(dst, src);
2324 }
2325
2326 void Assembler::evmovdqul(Address dst, XMMRegister src, int vector_len) {
2327 _instruction_uses_vl = true;
2328 assert(UseAVX > 0, "");
2329 InstructionMark im(this);
2330 assert(src != xnoreg, "sanity");
2331 if (VM_Version::supports_evex()) {
2332 _tuple_type = EVEX_FVM;
2333 }
2334 // swap src<->dst for encoding
2335 vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2336 emit_int8(0x7F);
2337 emit_operand(src, dst);
2338 }
2339
2340 void Assembler::evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
2341 _instruction_uses_vl = true;
2342 assert(UseAVX > 0, "");
2343 int src_enc = src->encoding();
2344 int dst_enc = dst->encoding();
2345 int encode = vex_prefix_and_encode(dst_enc, 0, src_enc, VEX_SIMD_F3, VEX_OPCODE_0F,
2346 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
2347 emit_int8(0x6F);
2348 emit_int8((unsigned char)(0xC0 | encode));
2349 }
2350
2351 void Assembler::evmovdquq(XMMRegister dst, Address src, int vector_len) {
2352 _instruction_uses_vl = true;
2353 assert(UseAVX > 2, "");
2354 InstructionMark im(this);
2355 _tuple_type = EVEX_FVM;
2356 vex_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, vector_len);
2357 emit_int8(0x6F);
2358 emit_operand(dst, src);
2359 }
2360
2361 void Assembler::evmovdquq(Address dst, XMMRegister src, int vector_len) {
2362 _instruction_uses_vl = true;
2363 assert(UseAVX > 2, "");
2364 InstructionMark im(this);
2365 assert(src != xnoreg, "sanity");
2366 _tuple_type = EVEX_FVM;
2367 // swap src<->dst for encoding
2368 vex_prefix_q(src, xnoreg, dst, VEX_SIMD_F3, vector_len);
2369 emit_int8(0x7F);
2370 emit_operand(src, dst);
2371 }
2372
2373 // Uses zero extension on 64bit
2374
2375 void Assembler::movl(Register dst, int32_t imm32) {
2376 int encode = prefix_and_encode(dst->encoding());
2377 emit_int8((unsigned char)(0xB8 | encode));
2378 emit_int32(imm32);
2379 }
2380
2381 void Assembler::movl(Register dst, Register src) {
2382 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2383 emit_int8((unsigned char)0x8B);
2384 emit_int8((unsigned char)(0xC0 | encode));
2385 }
2386
2387 void Assembler::movl(Register dst, Address src) {
2388 InstructionMark im(this);
2389 prefix(src, dst);
2390 emit_int8((unsigned char)0x8B);
2391 emit_operand(dst, src);
2392 }
2393
2394 void Assembler::movl(Address dst, int32_t imm32) {
2395 InstructionMark im(this);
2396 prefix(dst);
2397 emit_int8((unsigned char)0xC7);
2398 emit_operand(rax, dst, 4);
2399 emit_int32(imm32);
2400 }
2401
2402 void Assembler::movl(Address dst, Register src) {
2403 InstructionMark im(this);
2404 prefix(dst, src);
2405 emit_int8((unsigned char)0x89);
2406 emit_operand(src, dst);
2407 }
2408
2409 // New cpus require to use movsd and movss to avoid partial register stall
2410 // when loading from memory. But for old Opteron use movlpd instead of movsd.
2411 // The selection is done in MacroAssembler::movdbl() and movflt().
2412 void Assembler::movlpd(XMMRegister dst, Address src) {
2413 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2414 if (VM_Version::supports_evex()) {
2415 _tuple_type = EVEX_T1S;
2416 _input_size_in_bits = EVEX_32bit;
2417 emit_simd_arith_q(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2418 } else {
2419 emit_simd_arith(0x12, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2420 }
2421 }
2422
2423 void Assembler::movq( MMXRegister dst, Address src ) {
2424 assert( VM_Version::supports_mmx(), "" );
2425 emit_int8(0x0F);
2426 emit_int8(0x6F);
2427 emit_operand(dst, src);
2428 }
2429
2430 void Assembler::movq( Address dst, MMXRegister src ) {
2431 assert( VM_Version::supports_mmx(), "" );
2432 emit_int8(0x0F);
2433 emit_int8(0x7F);
2434 // workaround gcc (3.2.1-7a) bug
2435 // In that version of gcc with only an emit_operand(MMX, Address)
2436 // gcc will tail jump and try and reverse the parameters completely
2437 // obliterating dst in the process. By having a version available
2438 // that doesn't need to swap the args at the tail jump the bug is
2439 // avoided.
2440 emit_operand(dst, src);
2441 }
2442
2443 void Assembler::movq(XMMRegister dst, Address src) {
2444 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2445 InstructionMark im(this);
2446 if (VM_Version::supports_evex()) {
2447 _tuple_type = EVEX_T1S;
2448 _input_size_in_bits = EVEX_64bit;
2449 simd_prefix_q(dst, xnoreg, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2450 } else {
2451 simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2452 }
2453 emit_int8(0x7E);
2454 emit_operand(dst, src);
2455 }
2456
2457 void Assembler::movq(Address dst, XMMRegister src) {
2458 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2459 InstructionMark im(this);
2460 if (VM_Version::supports_evex()) {
2461 _tuple_type = EVEX_T1S;
2462 _input_size_in_bits = EVEX_64bit;
2463 simd_prefix(src, xnoreg, dst, VEX_SIMD_66, /* no_mask_reg */ true,
2464 VEX_OPCODE_0F, /* rex_w */ true);
2465 } else {
2466 simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
2467 }
2468 emit_int8((unsigned char)0xD6);
2469 emit_operand(src, dst);
2470 }
2471
2472 void Assembler::movsbl(Register dst, Address src) { // movsxb
2473 InstructionMark im(this);
2474 prefix(src, dst);
2475 emit_int8(0x0F);
2476 emit_int8((unsigned char)0xBE);
2477 emit_operand(dst, src);
2478 }
2479
2480 void Assembler::movsbl(Register dst, Register src) { // movsxb
2481 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2482 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
2483 emit_int8(0x0F);
2484 emit_int8((unsigned char)0xBE);
2485 emit_int8((unsigned char)(0xC0 | encode));
2486 }
2487
2488 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2489 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2490 if (VM_Version::supports_evex()) {
2491 emit_simd_arith_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
2492 } else {
2493 emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
2494 }
2495 }
2496
2497 void Assembler::movsd(XMMRegister dst, Address src) {
2498 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2499 if (VM_Version::supports_evex()) {
2500 _tuple_type = EVEX_T1S;
2501 _input_size_in_bits = EVEX_64bit;
2502 emit_simd_arith_nonds_q(0x10, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
2503 } else {
2504 emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
2505 }
2506 }
2507
2508 void Assembler::movsd(Address dst, XMMRegister src) {
2509 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2510 InstructionMark im(this);
2511 if (VM_Version::supports_evex()) {
2512 _tuple_type = EVEX_T1S;
2513 _input_size_in_bits = EVEX_64bit;
2514 simd_prefix_q(src, xnoreg, dst, VEX_SIMD_F2);
2515 } else {
2516 simd_prefix(src, xnoreg, dst, VEX_SIMD_F2, /* no_mask_reg */ false);
2517 }
2518 emit_int8(0x11);
2519 emit_operand(src, dst);
2520 }
2521
2522 void Assembler::movss(XMMRegister dst, XMMRegister src) {
2523 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2524 emit_simd_arith(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2525 }
2526
2527 void Assembler::movss(XMMRegister dst, Address src) {
2528 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2529 if (VM_Version::supports_evex()) {
2530 _tuple_type = EVEX_T1S;
2531 _input_size_in_bits = EVEX_32bit;
2532 }
2533 emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
2534 }
2535
2536 void Assembler::movss(Address dst, XMMRegister src) {
2537 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2538 if (VM_Version::supports_evex()) {
2539 _tuple_type = EVEX_T1S;
2540 _input_size_in_bits = EVEX_32bit;
2541 }
2542 InstructionMark im(this);
2543 simd_prefix(dst, src, VEX_SIMD_F3, /* no_mask_reg */ false);
2544 emit_int8(0x11);
2545 emit_operand(src, dst);
2546 }
2547
2548 void Assembler::movswl(Register dst, Address src) { // movsxw
2549 InstructionMark im(this);
2550 prefix(src, dst);
2551 emit_int8(0x0F);
2552 emit_int8((unsigned char)0xBF);
2553 emit_operand(dst, src);
2554 }
2555
2556 void Assembler::movswl(Register dst, Register src) { // movsxw
2557 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2558 emit_int8(0x0F);
2559 emit_int8((unsigned char)0xBF);
2560 emit_int8((unsigned char)(0xC0 | encode));
2561 }
2562
2563 void Assembler::movw(Address dst, int imm16) {
2564 InstructionMark im(this);
2565
2566 emit_int8(0x66); // switch to 16-bit mode
2567 prefix(dst);
2568 emit_int8((unsigned char)0xC7);
2569 emit_operand(rax, dst, 2);
2570 emit_int16(imm16);
2571 }
2572
2573 void Assembler::movw(Register dst, Address src) {
2574 InstructionMark im(this);
2575 emit_int8(0x66);
2576 prefix(src, dst);
2577 emit_int8((unsigned char)0x8B);
2578 emit_operand(dst, src);
2579 }
2580
2581 void Assembler::movw(Address dst, Register src) {
2582 InstructionMark im(this);
2583 emit_int8(0x66);
2584 prefix(dst, src);
2585 emit_int8((unsigned char)0x89);
2586 emit_operand(src, dst);
2587 }
2588
2589 void Assembler::movzbl(Register dst, Address src) { // movzxb
2590 InstructionMark im(this);
2591 prefix(src, dst);
2592 emit_int8(0x0F);
2593 emit_int8((unsigned char)0xB6);
2594 emit_operand(dst, src);
2595 }
2596
2597 void Assembler::movzbl(Register dst, Register src) { // movzxb
2598 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
2599 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
2600 emit_int8(0x0F);
2601 emit_int8((unsigned char)0xB6);
2602 emit_int8(0xC0 | encode);
2603 }
2604
2605 void Assembler::movzwl(Register dst, Address src) { // movzxw
2606 InstructionMark im(this);
2607 prefix(src, dst);
2608 emit_int8(0x0F);
2609 emit_int8((unsigned char)0xB7);
2610 emit_operand(dst, src);
2611 }
2612
2613 void Assembler::movzwl(Register dst, Register src) { // movzxw
2614 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2615 emit_int8(0x0F);
2616 emit_int8((unsigned char)0xB7);
2617 emit_int8(0xC0 | encode);
2618 }
2619
2620 void Assembler::mull(Address src) {
2621 InstructionMark im(this);
2622 prefix(src);
2623 emit_int8((unsigned char)0xF7);
2624 emit_operand(rsp, src);
2625 }
2626
2627 void Assembler::mull(Register src) {
2628 int encode = prefix_and_encode(src->encoding());
2629 emit_int8((unsigned char)0xF7);
2630 emit_int8((unsigned char)(0xE0 | encode));
2631 }
2632
2633 void Assembler::mulsd(XMMRegister dst, Address src) {
2634 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2635 if (VM_Version::supports_evex()) {
2636 _tuple_type = EVEX_T1S;
2637 _input_size_in_bits = EVEX_64bit;
2638 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
2639 } else {
2640 emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2641 }
2642 }
2643
2644 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2645 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2646 if (VM_Version::supports_evex()) {
2647 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_F2);
2648 } else {
2649 emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2650 }
2651 }
2652
2653 void Assembler::mulss(XMMRegister dst, Address src) {
2654 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2655 if (VM_Version::supports_evex()) {
2656 _tuple_type = EVEX_T1S;
2657 _input_size_in_bits = EVEX_32bit;
2658 }
2659 emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
2660 }
2661
2662 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2663 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2664 emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
2665 }
2666
2667 void Assembler::negl(Register dst) {
2668 int encode = prefix_and_encode(dst->encoding());
2669 emit_int8((unsigned char)0xF7);
2670 emit_int8((unsigned char)(0xD8 | encode));
2671 }
2672
2673 void Assembler::nop(int i) {
2674 #ifdef ASSERT
2675 assert(i > 0, " ");
2676 // The fancy nops aren't currently recognized by debuggers making it a
2677 // pain to disassemble code while debugging. If asserts are on clearly
2678 // speed is not an issue so simply use the single byte traditional nop
2679 // to do alignment.
2680
2681 for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2682 return;
2683
2684 #endif // ASSERT
2685
2686 if (UseAddressNop && VM_Version::is_intel()) {
2687 //
2688 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2689 // 1: 0x90
2690 // 2: 0x66 0x90
2691 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2692 // 4: 0x0F 0x1F 0x40 0x00
2693 // 5: 0x0F 0x1F 0x44 0x00 0x00
2694 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2695 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2696 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2697 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2698 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2699 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2700
2701 // The rest coding is Intel specific - don't use consecutive address nops
2702
2703 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2704 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2705 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2706 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2707
2708 while(i >= 15) {
2709 // For Intel don't generate consecutive addess nops (mix with regular nops)
2710 i -= 15;
2711 emit_int8(0x66); // size prefix
2712 emit_int8(0x66); // size prefix
2713 emit_int8(0x66); // size prefix
2714 addr_nop_8();
2715 emit_int8(0x66); // size prefix
2716 emit_int8(0x66); // size prefix
2717 emit_int8(0x66); // size prefix
2718 emit_int8((unsigned char)0x90);
2719 // nop
2720 }
2721 switch (i) {
2722 case 14:
2723 emit_int8(0x66); // size prefix
2724 case 13:
2725 emit_int8(0x66); // size prefix
2726 case 12:
2727 addr_nop_8();
2728 emit_int8(0x66); // size prefix
2729 emit_int8(0x66); // size prefix
2730 emit_int8(0x66); // size prefix
2731 emit_int8((unsigned char)0x90);
2732 // nop
2733 break;
2734 case 11:
2735 emit_int8(0x66); // size prefix
2736 case 10:
2737 emit_int8(0x66); // size prefix
2738 case 9:
2739 emit_int8(0x66); // size prefix
2740 case 8:
2741 addr_nop_8();
2742 break;
2743 case 7:
2744 addr_nop_7();
2745 break;
2746 case 6:
2747 emit_int8(0x66); // size prefix
2748 case 5:
2749 addr_nop_5();
2750 break;
2751 case 4:
2752 addr_nop_4();
2753 break;
2754 case 3:
2755 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2756 emit_int8(0x66); // size prefix
2757 case 2:
2758 emit_int8(0x66); // size prefix
2759 case 1:
2760 emit_int8((unsigned char)0x90);
2761 // nop
2762 break;
2763 default:
2764 assert(i == 0, " ");
2765 }
2766 return;
2767 }
2768 if (UseAddressNop && VM_Version::is_amd()) {
2769 //
2770 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2771 // 1: 0x90
2772 // 2: 0x66 0x90
2773 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2774 // 4: 0x0F 0x1F 0x40 0x00
2775 // 5: 0x0F 0x1F 0x44 0x00 0x00
2776 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2777 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2778 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2779 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2780 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2781 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2782
2783 // The rest coding is AMD specific - use consecutive address nops
2784
2785 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2786 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2787 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2788 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2789 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2790 // Size prefixes (0x66) are added for larger sizes
2791
2792 while(i >= 22) {
2793 i -= 11;
2794 emit_int8(0x66); // size prefix
2795 emit_int8(0x66); // size prefix
2796 emit_int8(0x66); // size prefix
2797 addr_nop_8();
2798 }
2799 // Generate first nop for size between 21-12
2800 switch (i) {
2801 case 21:
2802 i -= 1;
2803 emit_int8(0x66); // size prefix
2804 case 20:
2805 case 19:
2806 i -= 1;
2807 emit_int8(0x66); // size prefix
2808 case 18:
2809 case 17:
2810 i -= 1;
2811 emit_int8(0x66); // size prefix
2812 case 16:
2813 case 15:
2814 i -= 8;
2815 addr_nop_8();
2816 break;
2817 case 14:
2818 case 13:
2819 i -= 7;
2820 addr_nop_7();
2821 break;
2822 case 12:
2823 i -= 6;
2824 emit_int8(0x66); // size prefix
2825 addr_nop_5();
2826 break;
2827 default:
2828 assert(i < 12, " ");
2829 }
2830
2831 // Generate second nop for size between 11-1
2832 switch (i) {
2833 case 11:
2834 emit_int8(0x66); // size prefix
2835 case 10:
2836 emit_int8(0x66); // size prefix
2837 case 9:
2838 emit_int8(0x66); // size prefix
2839 case 8:
2840 addr_nop_8();
2841 break;
2842 case 7:
2843 addr_nop_7();
2844 break;
2845 case 6:
2846 emit_int8(0x66); // size prefix
2847 case 5:
2848 addr_nop_5();
2849 break;
2850 case 4:
2851 addr_nop_4();
2852 break;
2853 case 3:
2854 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2855 emit_int8(0x66); // size prefix
2856 case 2:
2857 emit_int8(0x66); // size prefix
2858 case 1:
2859 emit_int8((unsigned char)0x90);
2860 // nop
2861 break;
2862 default:
2863 assert(i == 0, " ");
2864 }
2865 return;
2866 }
2867
2868 // Using nops with size prefixes "0x66 0x90".
2869 // From AMD Optimization Guide:
2870 // 1: 0x90
2871 // 2: 0x66 0x90
2872 // 3: 0x66 0x66 0x90
2873 // 4: 0x66 0x66 0x66 0x90
2874 // 5: 0x66 0x66 0x90 0x66 0x90
2875 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
2876 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2877 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2878 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2879 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2880 //
2881 while(i > 12) {
2882 i -= 4;
2883 emit_int8(0x66); // size prefix
2884 emit_int8(0x66);
2885 emit_int8(0x66);
2886 emit_int8((unsigned char)0x90);
2887 // nop
2888 }
2889 // 1 - 12 nops
2890 if(i > 8) {
2891 if(i > 9) {
2892 i -= 1;
2893 emit_int8(0x66);
2894 }
2895 i -= 3;
2896 emit_int8(0x66);
2897 emit_int8(0x66);
2898 emit_int8((unsigned char)0x90);
2899 }
2900 // 1 - 8 nops
2901 if(i > 4) {
2902 if(i > 6) {
2903 i -= 1;
2904 emit_int8(0x66);
2905 }
2906 i -= 3;
2907 emit_int8(0x66);
2908 emit_int8(0x66);
2909 emit_int8((unsigned char)0x90);
2910 }
2911 switch (i) {
2912 case 4:
2913 emit_int8(0x66);
2914 case 3:
2915 emit_int8(0x66);
2916 case 2:
2917 emit_int8(0x66);
2918 case 1:
2919 emit_int8((unsigned char)0x90);
2920 break;
2921 default:
2922 assert(i == 0, " ");
2923 }
2924 }
2925
2926 void Assembler::notl(Register dst) {
2927 int encode = prefix_and_encode(dst->encoding());
2928 emit_int8((unsigned char)0xF7);
2929 emit_int8((unsigned char)(0xD0 | encode));
2930 }
2931
2932 void Assembler::orl(Address dst, int32_t imm32) {
2933 InstructionMark im(this);
2934 prefix(dst);
2935 emit_arith_operand(0x81, rcx, dst, imm32);
2936 }
2937
2938 void Assembler::orl(Register dst, int32_t imm32) {
2939 prefix(dst);
2940 emit_arith(0x81, 0xC8, dst, imm32);
2941 }
2942
2943 void Assembler::orl(Register dst, Address src) {
2944 InstructionMark im(this);
2945 prefix(src, dst);
2946 emit_int8(0x0B);
2947 emit_operand(dst, src);
2948 }
2949
2950 void Assembler::orl(Register dst, Register src) {
2951 (void) prefix_and_encode(dst->encoding(), src->encoding());
2952 emit_arith(0x0B, 0xC0, dst, src);
2953 }
2954
2955 void Assembler::orl(Address dst, Register src) {
2956 InstructionMark im(this);
2957 prefix(dst, src);
2958 emit_int8(0x09);
2959 emit_operand(src, dst);
2960 }
2961
2962 void Assembler::packuswb(XMMRegister dst, Address src) {
2963 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2964 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2965 if (VM_Version::supports_evex()) {
2966 _tuple_type = EVEX_FV;
2967 _input_size_in_bits = EVEX_32bit;
2968 }
2969 emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
2970 }
2971
2972 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2973 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2974 emit_simd_arith(0x67, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
2975 }
2976
2977 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
2978 assert(UseAVX > 0, "some form of AVX must be enabled");
2979 emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
2980 }
2981
2982 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, int vector_len) {
2983 _instruction_uses_vl = true;
2984 assert(VM_Version::supports_avx2(), "");
2985 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
2986 VEX_OPCODE_0F_3A, /* rex_w */ true, vector_len);
2987 emit_int8(0x00);
2988 emit_int8(0xC0 | encode);
2989 emit_int8(imm8);
2990 }
2991
2992 void Assembler::pause() {
2993 emit_int8((unsigned char)0xF3);
2994 emit_int8((unsigned char)0x90);
2995 }
2996
2997 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2998 assert(VM_Version::supports_sse4_2(), "");
2999 InstructionMark im(this);
3000 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_3A,
3001 /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3002 emit_int8(0x61);
3003 emit_operand(dst, src);
3004 emit_int8(imm8);
3005 }
3006
3007 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3008 assert(VM_Version::supports_sse4_2(), "");
3009 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3010 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3011 emit_int8(0x61);
3012 emit_int8((unsigned char)(0xC0 | encode));
3013 emit_int8(imm8);
3014 }
3015
3016 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
3017 assert(VM_Version::supports_sse4_1(), "");
3018 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3019 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3020 emit_int8(0x16);
3021 emit_int8((unsigned char)(0xC0 | encode));
3022 emit_int8(imm8);
3023 }
3024
3025 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
3026 assert(VM_Version::supports_sse4_1(), "");
3027 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3028 VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3029 emit_int8(0x16);
3030 emit_int8((unsigned char)(0xC0 | encode));
3031 emit_int8(imm8);
3032 }
3033
3034 void Assembler::pextrw(Register dst, XMMRegister src, int imm8) {
3035 assert(VM_Version::supports_sse2(), "");
3036 int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ true,
3037 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3038 emit_int8(0x15);
3039 emit_int8((unsigned char)(0xC0 | encode));
3040 emit_int8(imm8);
3041 }
3042
3043 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
3044 assert(VM_Version::supports_sse4_1(), "");
3045 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3046 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3047 emit_int8(0x22);
3048 emit_int8((unsigned char)(0xC0 | encode));
3049 emit_int8(imm8);
3050 }
3051
3052 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
3053 assert(VM_Version::supports_sse4_1(), "");
3054 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3055 VEX_OPCODE_0F_3A, /* rex_w */ true, AVX_128bit, /* legacy_mode */ _legacy_mode_dq);
3056 emit_int8(0x22);
3057 emit_int8((unsigned char)(0xC0 | encode));
3058 emit_int8(imm8);
3059 }
3060
3061 void Assembler::pinsrw(XMMRegister dst, Register src, int imm8) {
3062 assert(VM_Version::supports_sse2(), "");
3063 int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, /* no_mask_reg */ true,
3064 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3065 emit_int8((unsigned char)0xC4);
3066 emit_int8((unsigned char)(0xC0 | encode));
3067 emit_int8(imm8);
3068 }
3069
3070 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
3071 assert(VM_Version::supports_sse4_1(), "");
3072 if (VM_Version::supports_evex()) {
3073 _tuple_type = EVEX_HVM;
3074 }
3075 InstructionMark im(this);
3076 simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);
3077 emit_int8(0x30);
3078 emit_operand(dst, src);
3079 }
3080
3081 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3082 assert(VM_Version::supports_sse4_1(), "");
3083 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F_38);
3084 emit_int8(0x30);
3085 emit_int8((unsigned char)(0xC0 | encode));
3086 }
3087
3088 // generic
3089 void Assembler::pop(Register dst) {
3090 int encode = prefix_and_encode(dst->encoding());
3091 emit_int8(0x58 | encode);
3092 }
3093
3094 void Assembler::popcntl(Register dst, Address src) {
3095 assert(VM_Version::supports_popcnt(), "must support");
3096 InstructionMark im(this);
3097 emit_int8((unsigned char)0xF3);
3098 prefix(src, dst);
3099 emit_int8(0x0F);
3100 emit_int8((unsigned char)0xB8);
3101 emit_operand(dst, src);
3102 }
3103
3104 void Assembler::popcntl(Register dst, Register src) {
3105 assert(VM_Version::supports_popcnt(), "must support");
3106 emit_int8((unsigned char)0xF3);
3107 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3108 emit_int8(0x0F);
3109 emit_int8((unsigned char)0xB8);
3110 emit_int8((unsigned char)(0xC0 | encode));
3111 }
3112
3113 void Assembler::popf() {
3114 emit_int8((unsigned char)0x9D);
3115 }
3116
3117 #ifndef _LP64 // no 32bit push/pop on amd64
3118 void Assembler::popl(Address dst) {
3119 // NOTE: this will adjust stack by 8byte on 64bits
3120 InstructionMark im(this);
3121 prefix(dst);
3122 emit_int8((unsigned char)0x8F);
3123 emit_operand(rax, dst);
3124 }
3125 #endif
3126
3127 void Assembler::prefetch_prefix(Address src) {
3128 prefix(src);
3129 emit_int8(0x0F);
3130 }
3131
3132 void Assembler::prefetchnta(Address src) {
3133 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3134 InstructionMark im(this);
3135 prefetch_prefix(src);
3136 emit_int8(0x18);
3137 emit_operand(rax, src); // 0, src
3138 }
3139
3140 void Assembler::prefetchr(Address src) {
3141 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3142 InstructionMark im(this);
3143 prefetch_prefix(src);
3144 emit_int8(0x0D);
3145 emit_operand(rax, src); // 0, src
3146 }
3147
3148 void Assembler::prefetcht0(Address src) {
3149 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3150 InstructionMark im(this);
3151 prefetch_prefix(src);
3152 emit_int8(0x18);
3153 emit_operand(rcx, src); // 1, src
3154 }
3155
3156 void Assembler::prefetcht1(Address src) {
3157 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3158 InstructionMark im(this);
3159 prefetch_prefix(src);
3160 emit_int8(0x18);
3161 emit_operand(rdx, src); // 2, src
3162 }
3163
3164 void Assembler::prefetcht2(Address src) {
3165 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
3166 InstructionMark im(this);
3167 prefetch_prefix(src);
3168 emit_int8(0x18);
3169 emit_operand(rbx, src); // 3, src
3170 }
3171
3172 void Assembler::prefetchw(Address src) {
3173 assert(VM_Version::supports_3dnow_prefetch(), "must support");
3174 InstructionMark im(this);
3175 prefetch_prefix(src);
3176 emit_int8(0x0D);
3177 emit_operand(rcx, src); // 1, src
3178 }
3179
3180 void Assembler::prefix(Prefix p) {
3181 emit_int8(p);
3182 }
3183
3184 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
3185 assert(VM_Version::supports_ssse3(), "");
3186 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
3187 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3188 emit_int8(0x00);
3189 emit_int8((unsigned char)(0xC0 | encode));
3190 }
3191
3192 void Assembler::pshufb(XMMRegister dst, Address src) {
3193 assert(VM_Version::supports_ssse3(), "");
3194 if (VM_Version::supports_evex()) {
3195 _tuple_type = EVEX_FVM;
3196 }
3197 InstructionMark im(this);
3198 simd_prefix(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
3199 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3200 emit_int8(0x00);
3201 emit_operand(dst, src);
3202 }
3203
3204 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
3205 _instruction_uses_vl = true;
3206 assert(isByte(mode), "invalid value");
3207 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3208 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
3209 emit_int8(mode & 0xFF);
3210 }
3211
3212 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
3213 _instruction_uses_vl = true;
3214 assert(isByte(mode), "invalid value");
3215 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3216 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3217 if (VM_Version::supports_evex()) {
3218 _tuple_type = EVEX_FV;
3219 _input_size_in_bits = EVEX_32bit;
3220 }
3221 InstructionMark im(this);
3222 simd_prefix(dst, src, VEX_SIMD_66, /* no_mask_reg */ false);
3223 emit_int8(0x70);
3224 emit_operand(dst, src);
3225 emit_int8(mode & 0xFF);
3226 }
3227
3228 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
3229 assert(isByte(mode), "invalid value");
3230 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3231 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
3232 emit_int8(mode & 0xFF);
3233 }
3234
3235 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
3236 assert(isByte(mode), "invalid value");
3237 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3238 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3239 if (VM_Version::supports_evex()) {
3240 _tuple_type = EVEX_FVM;
3241 }
3242 InstructionMark im(this);
3243 simd_prefix(dst, xnoreg, src, VEX_SIMD_F2, /* no_mask_reg */ false,
3244 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3245 emit_int8(0x70);
3246 emit_operand(dst, src);
3247 emit_int8(mode & 0xFF);
3248 }
3249
3250 void Assembler::psrldq(XMMRegister dst, int shift) {
3251 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3252 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3253 // XMM3 is for /3 encoding: 66 0F 73 /3 ib
3254 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
3255 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3256 emit_int8(0x73);
3257 emit_int8((unsigned char)(0xC0 | encode));
3258 emit_int8(shift);
3259 }
3260
3261 void Assembler::pslldq(XMMRegister dst, int shift) {
3262 // Shift left 128 bit value in dst XMMRegister by shift number of bytes.
3263 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3264 // XMM7 is for /7 encoding: 66 0F 73 /7 ib
3265 int encode = simd_prefix_and_encode(xmm7, dst, dst, VEX_SIMD_66, /* no_mask_reg */ true,
3266 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
3267 emit_int8(0x73);
3268 emit_int8((unsigned char)(0xC0 | encode));
3269 emit_int8(shift);
3270 }
3271
3272 void Assembler::ptest(XMMRegister dst, Address src) {
3273 assert(VM_Version::supports_sse4_1(), "");
3274 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3275 InstructionMark im(this);
3276 simd_prefix(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3277 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3278 emit_int8(0x17);
3279 emit_operand(dst, src);
3280 }
3281
3282 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
3283 assert(VM_Version::supports_sse4_1(), "");
3284 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, /* no_mask_reg */ false,
3285 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
3286 emit_int8(0x17);
3287 emit_int8((unsigned char)(0xC0 | encode));
3288 }
3289
3290 void Assembler::vptest(XMMRegister dst, Address src) {
3291 assert(VM_Version::supports_avx(), "");
3292 InstructionMark im(this);
3293 int vector_len = AVX_256bit;
3294 assert(dst != xnoreg, "sanity");
3295 int dst_enc = dst->encoding();
3296 // swap src<->dst for encoding
3297 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* rex_w */ false,
3298 vector_len, /* legacy_mode */ true, /* no_mask_reg */ false);
3299 emit_int8(0x17);
3300 emit_operand(dst, src);
3301 }
3302
3303 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
3304 assert(VM_Version::supports_avx(), "");
3305 int vector_len = AVX_256bit;
3306 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
3307 emit_int8(0x17);
3308 emit_int8((unsigned char)(0xC0 | encode));
3309 }
3310
3311 void Assembler::punpcklbw(XMMRegister dst, Address src) {
3312 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3313 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3314 if (VM_Version::supports_evex()) {
3315 _tuple_type = EVEX_FVM;
3316 }
3317 emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);
3318 }
3319
3320 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
3321 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3322 emit_simd_arith(0x60, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_vlbw);
3323 }
3324
3325 void Assembler::punpckldq(XMMRegister dst, Address src) {
3326 _instruction_uses_vl = true;
3327 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3328 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
3329 if (VM_Version::supports_evex()) {
3330 _tuple_type = EVEX_FV;
3331 _input_size_in_bits = EVEX_32bit;
3332 }
3333 emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
3334 }
3335
3336 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
3337 _instruction_uses_vl = true;
3338 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3339 emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
3340 }
3341
3342 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
3343 _instruction_uses_vl = true;
3344 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3345 if (VM_Version::supports_evex()) {
3346 emit_simd_arith_q(0x6C, dst, src, VEX_SIMD_66);
3347 } else {
3348 emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
3349 }
3350 }
3351
3352 void Assembler::push(int32_t imm32) {
3353 // in 64bits we push 64bits onto the stack but only
3354 // take a 32bit immediate
3355 emit_int8(0x68);
3356 emit_int32(imm32);
3357 }
3358
3359 void Assembler::push(Register src) {
3360 int encode = prefix_and_encode(src->encoding());
3361
3362 emit_int8(0x50 | encode);
3363 }
3364
3365 void Assembler::pushf() {
3366 emit_int8((unsigned char)0x9C);
3367 }
3368
3369 #ifndef _LP64 // no 32bit push/pop on amd64
3370 void Assembler::pushl(Address src) {
3371 // Note this will push 64bit on 64bit
3372 InstructionMark im(this);
3373 prefix(src);
3374 emit_int8((unsigned char)0xFF);
3375 emit_operand(rsi, src);
3376 }
3377 #endif
3378
3379 void Assembler::rcll(Register dst, int imm8) {
3380 assert(isShiftCount(imm8), "illegal shift count");
3381 int encode = prefix_and_encode(dst->encoding());
3382 if (imm8 == 1) {
3383 emit_int8((unsigned char)0xD1);
3384 emit_int8((unsigned char)(0xD0 | encode));
3385 } else {
3386 emit_int8((unsigned char)0xC1);
3387 emit_int8((unsigned char)0xD0 | encode);
3388 emit_int8(imm8);
3389 }
3390 }
3391
3392 void Assembler::rdtsc() {
3393 emit_int8((unsigned char)0x0F);
3394 emit_int8((unsigned char)0x31);
3395 }
3396
3397 // copies data from [esi] to [edi] using rcx pointer sized words
3398 // generic
3399 void Assembler::rep_mov() {
3400 emit_int8((unsigned char)0xF3);
3401 // MOVSQ
3402 LP64_ONLY(prefix(REX_W));
3403 emit_int8((unsigned char)0xA5);
3404 }
3405
3406 // sets rcx bytes with rax, value at [edi]
3407 void Assembler::rep_stosb() {
3408 emit_int8((unsigned char)0xF3); // REP
3409 LP64_ONLY(prefix(REX_W));
3410 emit_int8((unsigned char)0xAA); // STOSB
3411 }
3412
3413 // sets rcx pointer sized words with rax, value at [edi]
3414 // generic
3415 void Assembler::rep_stos() {
3416 emit_int8((unsigned char)0xF3); // REP
3417 LP64_ONLY(prefix(REX_W)); // LP64:STOSQ, LP32:STOSD
3418 emit_int8((unsigned char)0xAB);
3419 }
3420
3421 // scans rcx pointer sized words at [edi] for occurance of rax,
3422 // generic
3423 void Assembler::repne_scan() { // repne_scan
3424 emit_int8((unsigned char)0xF2);
3425 // SCASQ
3426 LP64_ONLY(prefix(REX_W));
3427 emit_int8((unsigned char)0xAF);
3428 }
3429
3430 #ifdef _LP64
3431 // scans rcx 4 byte words at [edi] for occurance of rax,
3432 // generic
3433 void Assembler::repne_scanl() { // repne_scan
3434 emit_int8((unsigned char)0xF2);
3435 // SCASL
3436 emit_int8((unsigned char)0xAF);
3437 }
3438 #endif
3439
3440 void Assembler::ret(int imm16) {
3441 if (imm16 == 0) {
3442 emit_int8((unsigned char)0xC3);
3443 } else {
3444 emit_int8((unsigned char)0xC2);
3445 emit_int16(imm16);
3446 }
3447 }
3448
3449 void Assembler::sahf() {
3450 #ifdef _LP64
3451 // Not supported in 64bit mode
3452 ShouldNotReachHere();
3453 #endif
3454 emit_int8((unsigned char)0x9E);
3455 }
3456
3457 void Assembler::sarl(Register dst, int imm8) {
3458 int encode = prefix_and_encode(dst->encoding());
3459 assert(isShiftCount(imm8), "illegal shift count");
3460 if (imm8 == 1) {
3461 emit_int8((unsigned char)0xD1);
3462 emit_int8((unsigned char)(0xF8 | encode));
3463 } else {
3464 emit_int8((unsigned char)0xC1);
3465 emit_int8((unsigned char)(0xF8 | encode));
3466 emit_int8(imm8);
3467 }
3468 }
3469
3470 void Assembler::sarl(Register dst) {
3471 int encode = prefix_and_encode(dst->encoding());
3472 emit_int8((unsigned char)0xD3);
3473 emit_int8((unsigned char)(0xF8 | encode));
3474 }
3475
3476 void Assembler::sbbl(Address dst, int32_t imm32) {
3477 InstructionMark im(this);
3478 prefix(dst);
3479 emit_arith_operand(0x81, rbx, dst, imm32);
3480 }
3481
3482 void Assembler::sbbl(Register dst, int32_t imm32) {
3483 prefix(dst);
3484 emit_arith(0x81, 0xD8, dst, imm32);
3485 }
3486
3487
3488 void Assembler::sbbl(Register dst, Address src) {
3489 InstructionMark im(this);
3490 prefix(src, dst);
3491 emit_int8(0x1B);
3492 emit_operand(dst, src);
3493 }
3494
3495 void Assembler::sbbl(Register dst, Register src) {
3496 (void) prefix_and_encode(dst->encoding(), src->encoding());
3497 emit_arith(0x1B, 0xC0, dst, src);
3498 }
3499
3500 void Assembler::setb(Condition cc, Register dst) {
3501 assert(0 <= cc && cc < 16, "illegal cc");
3502 int encode = prefix_and_encode(dst->encoding(), true);
3503 emit_int8(0x0F);
3504 emit_int8((unsigned char)0x90 | cc);
3505 emit_int8((unsigned char)(0xC0 | encode));
3506 }
3507
3508 void Assembler::shll(Register dst, int imm8) {
3509 assert(isShiftCount(imm8), "illegal shift count");
3510 int encode = prefix_and_encode(dst->encoding());
3511 if (imm8 == 1 ) {
3512 emit_int8((unsigned char)0xD1);
3513 emit_int8((unsigned char)(0xE0 | encode));
3514 } else {
3515 emit_int8((unsigned char)0xC1);
3516 emit_int8((unsigned char)(0xE0 | encode));
3517 emit_int8(imm8);
3518 }
3519 }
3520
3521 void Assembler::shll(Register dst) {
3522 int encode = prefix_and_encode(dst->encoding());
3523 emit_int8((unsigned char)0xD3);
3524 emit_int8((unsigned char)(0xE0 | encode));
3525 }
3526
3527 void Assembler::shrl(Register dst, int imm8) {
3528 assert(isShiftCount(imm8), "illegal shift count");
3529 int encode = prefix_and_encode(dst->encoding());
3530 emit_int8((unsigned char)0xC1);
3531 emit_int8((unsigned char)(0xE8 | encode));
3532 emit_int8(imm8);
3533 }
3534
3535 void Assembler::shrl(Register dst) {
3536 int encode = prefix_and_encode(dst->encoding());
3537 emit_int8((unsigned char)0xD3);
3538 emit_int8((unsigned char)(0xE8 | encode));
3539 }
3540
3541 // copies a single word from [esi] to [edi]
3542 void Assembler::smovl() {
3543 emit_int8((unsigned char)0xA5);
3544 }
3545
3546 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
3547 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3548 if (VM_Version::supports_evex()) {
3549 emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
3550 } else {
3551 emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
3552 }
3553 }
3554
3555 void Assembler::sqrtsd(XMMRegister dst, Address src) {
3556 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3557 if (VM_Version::supports_evex()) {
3558 _tuple_type = EVEX_T1S;
3559 _input_size_in_bits = EVEX_64bit;
3560 emit_simd_arith_q(0x51, dst, src, VEX_SIMD_F2);
3561 } else {
3562 emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
3563 }
3564 }
3565
3566 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
3567 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3568 emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
3569 }
3570
3571 void Assembler::std() {
3572 emit_int8((unsigned char)0xFD);
3573 }
3574
3575 void Assembler::sqrtss(XMMRegister dst, Address src) {
3576 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3577 if (VM_Version::supports_evex()) {
3578 _tuple_type = EVEX_T1S;
3579 _input_size_in_bits = EVEX_32bit;
3580 }
3581 emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
3582 }
3583
3584 void Assembler::stmxcsr( Address dst) {
3585 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3586 InstructionMark im(this);
3587 prefix(dst);
3588 emit_int8(0x0F);
3589 emit_int8((unsigned char)0xAE);
3590 emit_operand(as_Register(3), dst);
3591 }
3592
3593 void Assembler::subl(Address dst, int32_t imm32) {
3594 InstructionMark im(this);
3595 prefix(dst);
3596 emit_arith_operand(0x81, rbp, dst, imm32);
3597 }
3598
3599 void Assembler::subl(Address dst, Register src) {
3600 InstructionMark im(this);
3601 prefix(dst, src);
3602 emit_int8(0x29);
3603 emit_operand(src, dst);
3604 }
3605
3606 void Assembler::subl(Register dst, int32_t imm32) {
3607 prefix(dst);
3608 emit_arith(0x81, 0xE8, dst, imm32);
3609 }
3610
3611 // Force generation of a 4 byte immediate value even if it fits into 8bit
3612 void Assembler::subl_imm32(Register dst, int32_t imm32) {
3613 prefix(dst);
3614 emit_arith_imm32(0x81, 0xE8, dst, imm32);
3615 }
3616
3617 void Assembler::subl(Register dst, Address src) {
3618 InstructionMark im(this);
3619 prefix(src, dst);
3620 emit_int8(0x2B);
3621 emit_operand(dst, src);
3622 }
3623
3624 void Assembler::subl(Register dst, Register src) {
3625 (void) prefix_and_encode(dst->encoding(), src->encoding());
3626 emit_arith(0x2B, 0xC0, dst, src);
3627 }
3628
3629 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
3630 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3631 if (VM_Version::supports_evex()) {
3632 emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
3633 } else {
3634 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
3635 }
3636 }
3637
3638 void Assembler::subsd(XMMRegister dst, Address src) {
3639 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3640 if (VM_Version::supports_evex()) {
3641 _tuple_type = EVEX_T1S;
3642 _input_size_in_bits = EVEX_64bit;
3643 }
3644 if (VM_Version::supports_evex()) {
3645 emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_F2);
3646 } else {
3647 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
3648 }
3649 }
3650
3651 void Assembler::subss(XMMRegister dst, XMMRegister src) {
3652 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3653 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
3654 }
3655
3656 void Assembler::subss(XMMRegister dst, Address src) {
3657 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3658 if (VM_Version::supports_evex()) {
3659 _tuple_type = EVEX_T1S;
3660 _input_size_in_bits = EVEX_32bit;
3661 }
3662 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
3663 }
3664
3665 void Assembler::testb(Register dst, int imm8) {
3666 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
3667 (void) prefix_and_encode(dst->encoding(), true);
3668 emit_arith_b(0xF6, 0xC0, dst, imm8);
3669 }
3670
3671 void Assembler::testl(Register dst, int32_t imm32) {
3672 // not using emit_arith because test
3673 // doesn't support sign-extension of
3674 // 8bit operands
3675 int encode = dst->encoding();
3676 if (encode == 0) {
3677 emit_int8((unsigned char)0xA9);
3678 } else {
3679 encode = prefix_and_encode(encode);
3680 emit_int8((unsigned char)0xF7);
3681 emit_int8((unsigned char)(0xC0 | encode));
3682 }
3683 emit_int32(imm32);
3684 }
3685
3686 void Assembler::testl(Register dst, Register src) {
3687 (void) prefix_and_encode(dst->encoding(), src->encoding());
3688 emit_arith(0x85, 0xC0, dst, src);
3689 }
3690
3691 void Assembler::testl(Register dst, Address src) {
3692 InstructionMark im(this);
3693 prefix(src, dst);
3694 emit_int8((unsigned char)0x85);
3695 emit_operand(dst, src);
3696 }
3697
3698 void Assembler::tzcntl(Register dst, Register src) {
3699 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3700 emit_int8((unsigned char)0xF3);
3701 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3702 emit_int8(0x0F);
3703 emit_int8((unsigned char)0xBC);
3704 emit_int8((unsigned char)0xC0 | encode);
3705 }
3706
3707 void Assembler::tzcntq(Register dst, Register src) {
3708 assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
3709 emit_int8((unsigned char)0xF3);
3710 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3711 emit_int8(0x0F);
3712 emit_int8((unsigned char)0xBC);
3713 emit_int8((unsigned char)(0xC0 | encode));
3714 }
3715
3716 void Assembler::ucomisd(XMMRegister dst, Address src) {
3717 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3718 if (VM_Version::supports_evex()) {
3719 _tuple_type = EVEX_T1S;
3720 _input_size_in_bits = EVEX_64bit;
3721 emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
3722 } else {
3723 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
3724 }
3725 }
3726
3727 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
3728 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3729 if (VM_Version::supports_evex()) {
3730 emit_simd_arith_nonds_q(0x2E, dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
3731 } else {
3732 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
3733 }
3734 }
3735
3736 void Assembler::ucomiss(XMMRegister dst, Address src) {
3737 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3738 if (VM_Version::supports_evex()) {
3739 _tuple_type = EVEX_T1S;
3740 _input_size_in_bits = EVEX_32bit;
3741 }
3742 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
3743 }
3744
3745 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
3746 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3747 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ true);
3748 }
3749
3750 void Assembler::xabort(int8_t imm8) {
3751 emit_int8((unsigned char)0xC6);
3752 emit_int8((unsigned char)0xF8);
3753 emit_int8((unsigned char)(imm8 & 0xFF));
3754 }
3755
3756 void Assembler::xaddl(Address dst, Register src) {
3757 InstructionMark im(this);
3758 prefix(dst, src);
3759 emit_int8(0x0F);
3760 emit_int8((unsigned char)0xC1);
3761 emit_operand(src, dst);
3762 }
3763
3764 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3765 InstructionMark im(this);
3766 relocate(rtype);
3767 if (abort.is_bound()) {
3768 address entry = target(abort);
3769 assert(entry != NULL, "abort entry NULL");
3770 intptr_t offset = entry - pc();
3771 emit_int8((unsigned char)0xC7);
3772 emit_int8((unsigned char)0xF8);
3773 emit_int32(offset - 6); // 2 opcode + 4 address
3774 } else {
3775 abort.add_patch_at(code(), locator());
3776 emit_int8((unsigned char)0xC7);
3777 emit_int8((unsigned char)0xF8);
3778 emit_int32(0);
3779 }
3780 }
3781
3782 void Assembler::xchgl(Register dst, Address src) { // xchg
3783 InstructionMark im(this);
3784 prefix(src, dst);
3785 emit_int8((unsigned char)0x87);
3786 emit_operand(dst, src);
3787 }
3788
3789 void Assembler::xchgl(Register dst, Register src) {
3790 int encode = prefix_and_encode(dst->encoding(), src->encoding());
3791 emit_int8((unsigned char)0x87);
3792 emit_int8((unsigned char)(0xC0 | encode));
3793 }
3794
3795 void Assembler::xend() {
3796 emit_int8((unsigned char)0x0F);
3797 emit_int8((unsigned char)0x01);
3798 emit_int8((unsigned char)0xD5);
3799 }
3800
3801 void Assembler::xgetbv() {
3802 emit_int8(0x0F);
3803 emit_int8(0x01);
3804 emit_int8((unsigned char)0xD0);
3805 }
3806
3807 void Assembler::xorl(Register dst, int32_t imm32) {
3808 prefix(dst);
3809 emit_arith(0x81, 0xF0, dst, imm32);
3810 }
3811
3812 void Assembler::xorl(Register dst, Address src) {
3813 InstructionMark im(this);
3814 prefix(src, dst);
3815 emit_int8(0x33);
3816 emit_operand(dst, src);
3817 }
3818
3819 void Assembler::xorl(Register dst, Register src) {
3820 (void) prefix_and_encode(dst->encoding(), src->encoding());
3821 emit_arith(0x33, 0xC0, dst, src);
3822 }
3823
3824
3825 // AVX 3-operands scalar float-point arithmetic instructions
3826
3827 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3828 assert(VM_Version::supports_avx(), "");
3829 if (VM_Version::supports_evex()) {
3830 _tuple_type = EVEX_T1S;
3831 _input_size_in_bits = EVEX_64bit;
3832 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3833 } else {
3834 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3835 }
3836 }
3837
3838 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3839 assert(VM_Version::supports_avx(), "");
3840 if (VM_Version::supports_evex()) {
3841 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3842 } else {
3843 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3844 }
3845 }
3846
3847 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3848 assert(VM_Version::supports_avx(), "");
3849 if (VM_Version::supports_evex()) {
3850 _tuple_type = EVEX_T1S;
3851 _input_size_in_bits = EVEX_32bit;
3852 }
3853 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3854 }
3855
3856 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3857 assert(VM_Version::supports_avx(), "");
3858 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3859 }
3860
3861 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3862 assert(VM_Version::supports_avx(), "");
3863 if (VM_Version::supports_evex()) {
3864 _tuple_type = EVEX_T1S;
3865 _input_size_in_bits = EVEX_64bit;
3866 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3867 } else {
3868 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3869 }
3870 }
3871
3872 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3873 assert(VM_Version::supports_avx(), "");
3874 if (VM_Version::supports_evex()) {
3875 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3876 } else {
3877 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3878 }
3879 }
3880
3881 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3882 assert(VM_Version::supports_avx(), "");
3883 if (VM_Version::supports_evex()) {
3884 _tuple_type = EVEX_T1S;
3885 _input_size_in_bits = EVEX_32bit;
3886 }
3887 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3888 }
3889
3890 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3891 assert(VM_Version::supports_avx(), "");
3892 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3893 }
3894
3895 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3896 assert(VM_Version::supports_avx(), "");
3897 if (VM_Version::supports_evex()) {
3898 _tuple_type = EVEX_T1S;
3899 _input_size_in_bits = EVEX_64bit;
3900 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3901 } else {
3902 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3903 }
3904 }
3905
3906 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3907 assert(VM_Version::supports_avx(), "");
3908 if (VM_Version::supports_evex()) {
3909 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3910 } else {
3911 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3912 }
3913 }
3914
3915 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
3916 assert(VM_Version::supports_avx(), "");
3917 if (VM_Version::supports_evex()) {
3918 _tuple_type = EVEX_T1S;
3919 _input_size_in_bits = EVEX_32bit;
3920 }
3921 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3922 }
3923
3924 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3925 assert(VM_Version::supports_avx(), "");
3926 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3927 }
3928
3929 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
3930 assert(VM_Version::supports_avx(), "");
3931 if (VM_Version::supports_evex()) {
3932 _tuple_type = EVEX_T1S;
3933 _input_size_in_bits = EVEX_64bit;
3934 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3935 } else {
3936 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3937 }
3938 }
3939
3940 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3941 assert(VM_Version::supports_avx(), "");
3942 if (VM_Version::supports_evex()) {
3943 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3944 } else {
3945 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, AVX_128bit);
3946 }
3947 }
3948
3949 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
3950 assert(VM_Version::supports_avx(), "");
3951 if (VM_Version::supports_evex()) {
3952 _tuple_type = EVEX_T1S;
3953 _input_size_in_bits = EVEX_32bit;
3954 }
3955 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3956 }
3957
3958 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3959 assert(VM_Version::supports_avx(), "");
3960 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, AVX_128bit);
3961 }
3962
3963 //====================VECTOR ARITHMETIC=====================================
3964
3965 // Float-point vector arithmetic
3966
3967 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
3968 _instruction_uses_vl = true;
3969 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3970 if (VM_Version::supports_evex()) {
3971 emit_simd_arith_q(0x58, dst, src, VEX_SIMD_66);
3972 } else {
3973 emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
3974 }
3975 }
3976
3977 void Assembler::addps(XMMRegister dst, XMMRegister src) {
3978 _instruction_uses_vl = true;
3979 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3980 emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
3981 }
3982
3983 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3984 _instruction_uses_vl = true;
3985 assert(VM_Version::supports_avx(), "");
3986 if (VM_Version::supports_evex()) {
3987 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
3988 } else {
3989 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
3990 }
3991 }
3992
3993 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
3994 _instruction_uses_vl = true;
3995 assert(VM_Version::supports_avx(), "");
3996 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);
3997 }
3998
3999 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4000 _instruction_uses_vl = true;
4001 assert(VM_Version::supports_avx(), "");
4002 if (VM_Version::supports_evex()) {
4003 _tuple_type = EVEX_FV;
4004 _input_size_in_bits = EVEX_64bit;
4005 emit_vex_arith_q(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4006 } else {
4007 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector_len);
4008 }
4009 }
4010
4011 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4012 _instruction_uses_vl = true;
4013 assert(VM_Version::supports_avx(), "");
4014 if (VM_Version::supports_evex()) {
4015 _tuple_type = EVEX_FV;
4016 _input_size_in_bits = EVEX_32bit;
4017 }
4018 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector_len);
4019 }
4020
4021 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
4022 _instruction_uses_vl = true;
4023 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4024 if (VM_Version::supports_evex()) {
4025 emit_simd_arith_q(0x5C, dst, src, VEX_SIMD_66);
4026 } else {
4027 emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
4028 }
4029 }
4030
4031 void Assembler::subps(XMMRegister dst, XMMRegister src) {
4032 _instruction_uses_vl = true;
4033 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4034 emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
4035 }
4036
4037 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4038 _instruction_uses_vl = true;
4039 assert(VM_Version::supports_avx(), "");
4040 if (VM_Version::supports_evex()) {
4041 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4042 } else {
4043 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4044 }
4045 }
4046
4047 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4048 _instruction_uses_vl = true;
4049 assert(VM_Version::supports_avx(), "");
4050 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);
4051 }
4052
4053 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4054 _instruction_uses_vl = true;
4055 assert(VM_Version::supports_avx(), "");
4056 if (VM_Version::supports_evex()) {
4057 _tuple_type = EVEX_FV;
4058 _input_size_in_bits = EVEX_64bit;
4059 emit_vex_arith_q(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4060 } else {
4061 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector_len);
4062 }
4063 }
4064
4065 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4066 _instruction_uses_vl = true;
4067 assert(VM_Version::supports_avx(), "");
4068 if (VM_Version::supports_evex()) {
4069 _tuple_type = EVEX_FV;
4070 _input_size_in_bits = EVEX_32bit;
4071 }
4072 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector_len);
4073 }
4074
4075 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
4076 _instruction_uses_vl = true;
4077 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4078 if (VM_Version::supports_evex()) {
4079 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
4080 } else {
4081 emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
4082 }
4083 }
4084
4085 void Assembler::mulpd(XMMRegister dst, Address src) {
4086 _instruction_uses_vl = true;
4087 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4088 if (VM_Version::supports_evex()) {
4089 emit_simd_arith_q(0x59, dst, src, VEX_SIMD_66);
4090 } else {
4091 emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
4092 }
4093 }
4094
4095 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
4096 _instruction_uses_vl = true;
4097 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4098 emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
4099 }
4100
4101 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4102 _instruction_uses_vl = true;
4103 assert(VM_Version::supports_avx(), "");
4104 if (VM_Version::supports_evex()) {
4105 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4106 } else {
4107 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4108 }
4109 }
4110
4111 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4112 _instruction_uses_vl = true;
4113 assert(VM_Version::supports_avx(), "");
4114 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);
4115 }
4116
4117 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4118 _instruction_uses_vl = true;
4119 assert(VM_Version::supports_avx(), "");
4120 if (VM_Version::supports_evex()) {
4121 _tuple_type = EVEX_FV;
4122 _input_size_in_bits = EVEX_64bit;
4123 emit_vex_arith_q(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4124 } else {
4125 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector_len);
4126 }
4127 }
4128
4129 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4130 _instruction_uses_vl = true;
4131 assert(VM_Version::supports_avx(), "");
4132 if (VM_Version::supports_evex()) {
4133 _tuple_type = EVEX_FV;
4134 _input_size_in_bits = EVEX_32bit;
4135 }
4136 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector_len);
4137 }
4138
4139 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
4140 _instruction_uses_vl = true;
4141 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4142 if (VM_Version::supports_evex()) {
4143 emit_simd_arith_q(0x5E, dst, src, VEX_SIMD_66);
4144 } else {
4145 emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
4146 }
4147 }
4148
4149 void Assembler::divps(XMMRegister dst, XMMRegister src) {
4150 _instruction_uses_vl = true;
4151 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4152 emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
4153 }
4154
4155 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4156 _instruction_uses_vl = true;
4157 assert(VM_Version::supports_avx(), "");
4158 if (VM_Version::supports_evex()) {
4159 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4160 } else {
4161 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4162 }
4163 }
4164
4165 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4166 _instruction_uses_vl = true;
4167 assert(VM_Version::supports_avx(), "");
4168 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);
4169 }
4170
4171 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4172 _instruction_uses_vl = true;
4173 assert(VM_Version::supports_avx(), "");
4174 if (VM_Version::supports_evex()) {
4175 _tuple_type = EVEX_FV;
4176 _input_size_in_bits = EVEX_64bit;
4177 emit_vex_arith_q(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4178 } else {
4179 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector_len);
4180 }
4181 }
4182
4183 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4184 _instruction_uses_vl = true;
4185 assert(VM_Version::supports_avx(), "");
4186 if (VM_Version::supports_evex()) {
4187 _tuple_type = EVEX_FV;
4188 _input_size_in_bits = EVEX_32bit;
4189 }
4190 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector_len);
4191 }
4192
4193 void Assembler::vsqrtpd(XMMRegister dst, XMMRegister src, int vector_len) {
4194 _instruction_uses_vl = true;
4195 assert(VM_Version::supports_avx(), "");
4196 if (VM_Version::supports_evex()) {
4197 emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4198 } else {
4199 emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4200 }
4201 }
4202
4203 void Assembler::vsqrtpd(XMMRegister dst, Address src, int vector_len) {
4204 _instruction_uses_vl = true;
4205 assert(VM_Version::supports_avx(), "");
4206 if (VM_Version::supports_evex()) {
4207 _tuple_type = EVEX_FV;
4208 _input_size_in_bits = EVEX_64bit;
4209 emit_vex_arith_q(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4210 } else {
4211 emit_vex_arith(0x51, dst, xnoreg, src, VEX_SIMD_66, vector_len);
4212 }
4213 }
4214
4215 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
4216 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4217 if (VM_Version::supports_avx512dq()) {
4218 emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
4219 } else {
4220 emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4221 }
4222 }
4223
4224 void Assembler::andps(XMMRegister dst, XMMRegister src) {
4225 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4226 emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4227 }
4228
4229 void Assembler::andps(XMMRegister dst, Address src) {
4230 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4231 if (VM_Version::supports_evex()) {
4232 _tuple_type = EVEX_FV;
4233 _input_size_in_bits = EVEX_32bit;
4234 }
4235 emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4236 }
4237
4238 void Assembler::andpd(XMMRegister dst, Address src) {
4239 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4240 if (VM_Version::supports_avx512dq()) {
4241 _tuple_type = EVEX_FV;
4242 _input_size_in_bits = EVEX_64bit;
4243 emit_simd_arith_q(0x54, dst, src, VEX_SIMD_66);
4244 } else {
4245 emit_simd_arith(0x54, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4246 }
4247 }
4248
4249 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4250 assert(VM_Version::supports_avx(), "");
4251 if (VM_Version::supports_avx512dq()) {
4252 emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
4253 } else {
4254 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4255 }
4256 }
4257
4258 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4259 assert(VM_Version::supports_avx(), "");
4260 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4261 }
4262
4263 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4264 assert(VM_Version::supports_avx(), "");
4265 if (VM_Version::supports_avx512dq()) {
4266 _tuple_type = EVEX_FV;
4267 _input_size_in_bits = EVEX_64bit;
4268 emit_vex_arith_q(0x54, dst, nds, src, VEX_SIMD_66, vector_len);
4269 } else {
4270 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4271 }
4272 }
4273
4274 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4275 assert(VM_Version::supports_avx(), "");
4276 if (VM_Version::supports_evex()) {
4277 _tuple_type = EVEX_FV;
4278 _input_size_in_bits = EVEX_32bit;
4279 }
4280 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4281 }
4282
4283 void Assembler::unpckhpd(XMMRegister dst, XMMRegister src) {
4284 _instruction_uses_vl = true;
4285 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4286 if (VM_Version::supports_evex()) {
4287 emit_simd_arith_q(0x15, dst, src, VEX_SIMD_66);
4288 } else {
4289 emit_simd_arith(0x15, dst, src, VEX_SIMD_66);
4290 }
4291 }
4292
4293 void Assembler::unpcklpd(XMMRegister dst, XMMRegister src) {
4294 _instruction_uses_vl = true;
4295 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4296 if (VM_Version::supports_evex()) {
4297 emit_simd_arith_q(0x14, dst, src, VEX_SIMD_66);
4298 } else {
4299 emit_simd_arith(0x14, dst, src, VEX_SIMD_66);
4300 }
4301 }
4302
4303 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
4304 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4305 if (VM_Version::supports_avx512dq()) {
4306 emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
4307 } else {
4308 emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4309 }
4310 }
4311
4312 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
4313 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4314 emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4315 }
4316
4317 void Assembler::xorpd(XMMRegister dst, Address src) {
4318 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4319 if (VM_Version::supports_avx512dq()) {
4320 _tuple_type = EVEX_FV;
4321 _input_size_in_bits = EVEX_64bit;
4322 emit_simd_arith_q(0x57, dst, src, VEX_SIMD_66);
4323 } else {
4324 emit_simd_arith(0x57, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ true);
4325 }
4326 }
4327
4328 void Assembler::xorps(XMMRegister dst, Address src) {
4329 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4330 if (VM_Version::supports_evex()) {
4331 _tuple_type = EVEX_FV;
4332 _input_size_in_bits = EVEX_32bit;
4333 }
4334 emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4335 }
4336
4337 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4338 assert(VM_Version::supports_avx(), "");
4339 if (VM_Version::supports_avx512dq()) {
4340 emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
4341 } else {
4342 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4343 }
4344 }
4345
4346 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4347 assert(VM_Version::supports_avx(), "");
4348 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4349 }
4350
4351 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4352 assert(VM_Version::supports_avx(), "");
4353 if (VM_Version::supports_avx512dq()) {
4354 _tuple_type = EVEX_FV;
4355 _input_size_in_bits = EVEX_64bit;
4356 emit_vex_arith_q(0x57, dst, nds, src, VEX_SIMD_66, vector_len);
4357 } else {
4358 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ true);
4359 }
4360 }
4361
4362 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4363 assert(VM_Version::supports_avx(), "");
4364 if (VM_Version::supports_evex()) {
4365 _tuple_type = EVEX_FV;
4366 _input_size_in_bits = EVEX_32bit;
4367 }
4368 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_dq);
4369 }
4370
4371 // Integer vector arithmetic
4372 void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4373 assert(VM_Version::supports_avx() && (vector_len == 0) ||
4374 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4375 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
4376 emit_int8(0x01);
4377 emit_int8((unsigned char)(0xC0 | encode));
4378 }
4379
4380 void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4381 assert(VM_Version::supports_avx() && (vector_len == 0) ||
4382 VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
4383 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38, /* legacy_mode */ true);
4384 emit_int8(0x02);
4385 emit_int8((unsigned char)(0xC0 | encode));
4386 }
4387
4388 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
4389 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4390 emit_simd_arith(0xFC, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4391 }
4392
4393 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
4394 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4395 emit_simd_arith(0xFD, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4396 }
4397
4398 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
4399 _instruction_uses_vl = true;
4400 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4401 emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
4402 }
4403
4404 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
4405 _instruction_uses_vl = true;
4406 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4407 if (VM_Version::supports_evex()) {
4408 emit_simd_arith_q(0xD4, dst, src, VEX_SIMD_66);
4409 } else {
4410 emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
4411 }
4412 }
4413
4414 void Assembler::phaddw(XMMRegister dst, XMMRegister src) {
4415 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4416 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
4417 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
4418 emit_int8(0x01);
4419 emit_int8((unsigned char)(0xC0 | encode));
4420 }
4421
4422 void Assembler::phaddd(XMMRegister dst, XMMRegister src) {
4423 NOT_LP64(assert(VM_Version::supports_sse3(), ""));
4424 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
4425 VEX_OPCODE_0F_38, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
4426 emit_int8(0x02);
4427 emit_int8((unsigned char)(0xC0 | encode));
4428 }
4429
4430 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4431 assert(UseAVX > 0, "requires some form of AVX");
4432 emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4433 }
4434
4435 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4436 assert(UseAVX > 0, "requires some form of AVX");
4437 emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4438 }
4439
4440 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4441 _instruction_uses_vl = true;
4442 assert(UseAVX > 0, "requires some form of AVX");
4443 emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);
4444 }
4445
4446 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4447 _instruction_uses_vl = true;
4448 assert(UseAVX > 0, "requires some form of AVX");
4449 if (VM_Version::supports_evex()) {
4450 emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4451 } else {
4452 emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4453 }
4454 }
4455
4456 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4457 assert(UseAVX > 0, "requires some form of AVX");
4458 if (VM_Version::supports_evex()) {
4459 _tuple_type = EVEX_FVM;
4460 }
4461 emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4462 }
4463
4464 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4465 assert(UseAVX > 0, "requires some form of AVX");
4466 if (VM_Version::supports_evex()) {
4467 _tuple_type = EVEX_FVM;
4468 }
4469 emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4470 }
4471
4472 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4473 _instruction_uses_vl = true;
4474 assert(UseAVX > 0, "requires some form of AVX");
4475 if (VM_Version::supports_evex()) {
4476 _tuple_type = EVEX_FV;
4477 _input_size_in_bits = EVEX_32bit;
4478 }
4479 emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector_len);
4480 }
4481
4482 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4483 _instruction_uses_vl = true;
4484 assert(UseAVX > 0, "requires some form of AVX");
4485 if (VM_Version::supports_evex()) {
4486 _tuple_type = EVEX_FV;
4487 _input_size_in_bits = EVEX_64bit;
4488 emit_vex_arith_q(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4489 } else {
4490 emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector_len);
4491 }
4492 }
4493
4494 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
4495 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4496 emit_simd_arith(0xF8, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4497 }
4498
4499 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
4500 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4501 emit_simd_arith(0xF9, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4502 }
4503
4504 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
4505 _instruction_uses_vl = true;
4506 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4507 emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
4508 }
4509
4510 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
4511 _instruction_uses_vl = true;
4512 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4513 if (VM_Version::supports_evex()) {
4514 emit_simd_arith_q(0xFB, dst, src, VEX_SIMD_66);
4515 } else {
4516 emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
4517 }
4518 }
4519
4520 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4521 assert(UseAVX > 0, "requires some form of AVX");
4522 emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4523 }
4524
4525 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4526 assert(UseAVX > 0, "requires some form of AVX");
4527 emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4528 }
4529
4530 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4531 _instruction_uses_vl = true;
4532 assert(UseAVX > 0, "requires some form of AVX");
4533 emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);
4534 }
4535
4536 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4537 _instruction_uses_vl = true;
4538 assert(UseAVX > 0, "requires some form of AVX");
4539 if (VM_Version::supports_evex()) {
4540 emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4541 } else {
4542 emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4543 }
4544 }
4545
4546 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4547 assert(UseAVX > 0, "requires some form of AVX");
4548 if (VM_Version::supports_evex()) {
4549 _tuple_type = EVEX_FVM;
4550 }
4551 emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4552 }
4553
4554 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4555 assert(UseAVX > 0, "requires some form of AVX");
4556 if (VM_Version::supports_evex()) {
4557 _tuple_type = EVEX_FVM;
4558 }
4559 emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4560 }
4561
4562 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4563 _instruction_uses_vl = true;
4564 assert(UseAVX > 0, "requires some form of AVX");
4565 if (VM_Version::supports_evex()) {
4566 _tuple_type = EVEX_FV;
4567 _input_size_in_bits = EVEX_32bit;
4568 }
4569 emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector_len);
4570 }
4571
4572 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4573 _instruction_uses_vl = true;
4574 assert(UseAVX > 0, "requires some form of AVX");
4575 if (VM_Version::supports_evex()) {
4576 _tuple_type = EVEX_FV;
4577 _input_size_in_bits = EVEX_64bit;
4578 emit_vex_arith_q(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4579 } else {
4580 emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector_len);
4581 }
4582 }
4583
4584 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
4585 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4586 emit_simd_arith(0xD5, dst, src, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4587 }
4588
4589 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
4590 _instruction_uses_vl = true;
4591 assert(VM_Version::supports_sse4_1(), "");
4592 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66,
4593 /* no_mask_reg */ false, VEX_OPCODE_0F_38);
4594 emit_int8(0x40);
4595 emit_int8((unsigned char)(0xC0 | encode));
4596 }
4597
4598 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4599 assert(UseAVX > 0, "requires some form of AVX");
4600 emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4601 }
4602
4603 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4604 _instruction_uses_vl = true;
4605 assert(UseAVX > 0, "requires some form of AVX");
4606 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
4607 emit_int8(0x40);
4608 emit_int8((unsigned char)(0xC0 | encode));
4609 }
4610
4611 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4612 assert(UseAVX > 2, "requires some form of AVX");
4613 int src_enc = src->encoding();
4614 int dst_enc = dst->encoding();
4615 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4616 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_38,
4617 /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq, /* no_mask_reg */ false);
4618 emit_int8(0x40);
4619 emit_int8((unsigned char)(0xC0 | encode));
4620 }
4621
4622 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4623 assert(UseAVX > 0, "requires some form of AVX");
4624 if (VM_Version::supports_evex()) {
4625 _tuple_type = EVEX_FVM;
4626 }
4627 emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4628 }
4629
4630 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4631 _instruction_uses_vl = true;
4632 assert(UseAVX > 0, "requires some form of AVX");
4633 if (VM_Version::supports_evex()) {
4634 _tuple_type = EVEX_FV;
4635 _input_size_in_bits = EVEX_32bit;
4636 }
4637 InstructionMark im(this);
4638 int dst_enc = dst->encoding();
4639 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4640 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
4641 VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
4642 emit_int8(0x40);
4643 emit_operand(dst, src);
4644 }
4645
4646 void Assembler::vpmullq(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4647 assert(UseAVX > 0, "requires some form of AVX");
4648 if (VM_Version::supports_evex()) {
4649 _tuple_type = EVEX_FV;
4650 _input_size_in_bits = EVEX_64bit;
4651 }
4652 InstructionMark im(this);
4653 int dst_enc = dst->encoding();
4654 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4655 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66,
4656 VEX_OPCODE_0F_38, /* vex_w */ true, vector_len, /* legacy_mode */ _legacy_mode_dq);
4657 emit_int8(0x40);
4658 emit_operand(dst, src);
4659 }
4660
4661 // Shift packed integers left by specified number of bits.
4662 void Assembler::psllw(XMMRegister dst, int shift) {
4663 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4664 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4665 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F,
4666 /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4667 emit_int8(0x71);
4668 emit_int8((unsigned char)(0xC0 | encode));
4669 emit_int8(shift & 0xFF);
4670 }
4671
4672 void Assembler::pslld(XMMRegister dst, int shift) {
4673 _instruction_uses_vl = true;
4674 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4675 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4676 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4677 emit_int8(0x72);
4678 emit_int8((unsigned char)(0xC0 | encode));
4679 emit_int8(shift & 0xFF);
4680 }
4681
4682 void Assembler::psllq(XMMRegister dst, int shift) {
4683 _instruction_uses_vl = true;
4684 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4685 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4686 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false, VEX_OPCODE_0F, /* rex_w */ true);
4687 emit_int8(0x73);
4688 emit_int8((unsigned char)(0xC0 | encode));
4689 emit_int8(shift & 0xFF);
4690 }
4691
4692 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
4693 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4694 emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4695 }
4696
4697 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
4698 _instruction_uses_vl = true;
4699 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4700 emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
4701 }
4702
4703 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
4704 _instruction_uses_vl = true;
4705 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4706 if (VM_Version::supports_evex()) {
4707 emit_simd_arith_q(0xF3, dst, shift, VEX_SIMD_66);
4708 } else {
4709 emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
4710 }
4711 }
4712
4713 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4714 assert(UseAVX > 0, "requires some form of AVX");
4715 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
4716 emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4717 emit_int8(shift & 0xFF);
4718 }
4719
4720 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4721 _instruction_uses_vl = true;
4722 assert(UseAVX > 0, "requires some form of AVX");
4723 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
4724 emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector_len);
4725 emit_int8(shift & 0xFF);
4726 }
4727
4728 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4729 _instruction_uses_vl = true;
4730 assert(UseAVX > 0, "requires some form of AVX");
4731 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
4732 if (VM_Version::supports_evex()) {
4733 emit_vex_arith_q(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
4734 } else {
4735 emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector_len);
4736 }
4737 emit_int8(shift & 0xFF);
4738 }
4739
4740 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4741 assert(UseAVX > 0, "requires some form of AVX");
4742 emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4743 }
4744
4745 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4746 _instruction_uses_vl = true;
4747 assert(UseAVX > 0, "requires some form of AVX");
4748 emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector_len);
4749 }
4750
4751 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4752 _instruction_uses_vl = true;
4753 assert(UseAVX > 0, "requires some form of AVX");
4754 if (VM_Version::supports_evex()) {
4755 emit_vex_arith_q(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
4756 } else {
4757 emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector_len);
4758 }
4759 }
4760
4761 // Shift packed integers logically right by specified number of bits.
4762 void Assembler::psrlw(XMMRegister dst, int shift) {
4763 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4764 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4765 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4766 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4767 emit_int8(0x71);
4768 emit_int8((unsigned char)(0xC0 | encode));
4769 emit_int8(shift & 0xFF);
4770 }
4771
4772 void Assembler::psrld(XMMRegister dst, int shift) {
4773 _instruction_uses_vl = true;
4774 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4775 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4776 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4777 emit_int8(0x72);
4778 emit_int8((unsigned char)(0xC0 | encode));
4779 emit_int8(shift & 0xFF);
4780 }
4781
4782 void Assembler::psrlq(XMMRegister dst, int shift) {
4783 _instruction_uses_vl = true;
4784 // Do not confuse it with psrldq SSE2 instruction which
4785 // shifts 128 bit value in xmm register by number of bytes.
4786 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4787 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4788 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4789 VEX_OPCODE_0F, /* rex_w */ VM_Version::supports_evex());
4790 emit_int8(0x73);
4791 emit_int8((unsigned char)(0xC0 | encode));
4792 emit_int8(shift & 0xFF);
4793 }
4794
4795 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
4796 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4797 emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4798 }
4799
4800 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
4801 _instruction_uses_vl = true;
4802 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4803 emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
4804 }
4805
4806 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
4807 _instruction_uses_vl = true;
4808 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4809 if (VM_Version::supports_evex()) {
4810 emit_simd_arith_q(0xD3, dst, shift, VEX_SIMD_66);
4811 } else {
4812 emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
4813 }
4814 }
4815
4816 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4817 assert(UseAVX > 0, "requires some form of AVX");
4818 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
4819 emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4820 emit_int8(shift & 0xFF);
4821 }
4822
4823 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4824 _instruction_uses_vl = true;
4825 assert(UseAVX > 0, "requires some form of AVX");
4826 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
4827 emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector_len);
4828 emit_int8(shift & 0xFF);
4829 }
4830
4831 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4832 _instruction_uses_vl = true;
4833 assert(UseAVX > 0, "requires some form of AVX");
4834 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
4835 if (VM_Version::supports_evex()) {
4836 emit_vex_arith_q(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
4837 } else {
4838 emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector_len);
4839 }
4840 emit_int8(shift & 0xFF);
4841 }
4842
4843 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4844 assert(UseAVX > 0, "requires some form of AVX");
4845 emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4846 }
4847
4848 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4849 _instruction_uses_vl = true;
4850 assert(UseAVX > 0, "requires some form of AVX");
4851 emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector_len);
4852 }
4853
4854 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4855 _instruction_uses_vl = true;
4856 assert(UseAVX > 0, "requires some form of AVX");
4857 if (VM_Version::supports_evex()) {
4858 emit_vex_arith_q(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
4859 } else {
4860 emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector_len);
4861 }
4862 }
4863
4864 // Shift packed integers arithmetically right by specified number of bits.
4865 void Assembler::psraw(XMMRegister dst, int shift) {
4866 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4867 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4868 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false,
4869 VEX_OPCODE_0F, /* rex_w */ false, AVX_128bit, /* legacy_mode */ _legacy_mode_bw);
4870 emit_int8(0x71);
4871 emit_int8((unsigned char)(0xC0 | encode));
4872 emit_int8(shift & 0xFF);
4873 }
4874
4875 void Assembler::psrad(XMMRegister dst, int shift) {
4876 _instruction_uses_vl = true;
4877 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4878 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
4879 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66, /* no_mask_reg */ false);
4880 emit_int8(0x72);
4881 emit_int8((unsigned char)(0xC0 | encode));
4882 emit_int8(shift & 0xFF);
4883 }
4884
4885 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
4886 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4887 emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4888 }
4889
4890 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
4891 _instruction_uses_vl = true;
4892 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4893 emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
4894 }
4895
4896 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4897 assert(UseAVX > 0, "requires some form of AVX");
4898 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4899 emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4900 emit_int8(shift & 0xFF);
4901 }
4902
4903 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, int vector_len) {
4904 _instruction_uses_vl = true;
4905 assert(UseAVX > 0, "requires some form of AVX");
4906 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
4907 emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector_len);
4908 emit_int8(shift & 0xFF);
4909 }
4910
4911 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4912 assert(UseAVX > 0, "requires some form of AVX");
4913 emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector_len, /* no_mask_reg */ false, /* legacy_mode */ _legacy_mode_bw);
4914 }
4915
4916 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, int vector_len) {
4917 _instruction_uses_vl = true;
4918 assert(UseAVX > 0, "requires some form of AVX");
4919 emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector_len);
4920 }
4921
4922
4923 // logical operations packed integers
4924 void Assembler::pand(XMMRegister dst, XMMRegister src) {
4925 _instruction_uses_vl = true;
4926 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4927 emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
4928 }
4929
4930 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4931 _instruction_uses_vl = true;
4932 assert(UseAVX > 0, "requires some form of AVX");
4933 emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);
4934 }
4935
4936 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4937 _instruction_uses_vl = true;
4938 assert(UseAVX > 0, "requires some form of AVX");
4939 if (VM_Version::supports_evex()) {
4940 _tuple_type = EVEX_FV;
4941 _input_size_in_bits = EVEX_32bit;
4942 }
4943 emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector_len);
4944 }
4945
4946 void Assembler::pandn(XMMRegister dst, XMMRegister src) {
4947 _instruction_uses_vl = true;
4948 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4949 if (VM_Version::supports_evex()) {
4950 emit_simd_arith_q(0xDF, dst, src, VEX_SIMD_66);
4951 }
4952 else {
4953 emit_simd_arith(0xDF, dst, src, VEX_SIMD_66);
4954 }
4955 }
4956
4957 void Assembler::por(XMMRegister dst, XMMRegister src) {
4958 _instruction_uses_vl = true;
4959 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4960 emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
4961 }
4962
4963 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4964 _instruction_uses_vl = true;
4965 assert(UseAVX > 0, "requires some form of AVX");
4966 emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);
4967 }
4968
4969 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4970 _instruction_uses_vl = true;
4971 assert(UseAVX > 0, "requires some form of AVX");
4972 if (VM_Version::supports_evex()) {
4973 _tuple_type = EVEX_FV;
4974 _input_size_in_bits = EVEX_32bit;
4975 }
4976 emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector_len);
4977 }
4978
4979 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
4980 _instruction_uses_vl = true;
4981 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4982 emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
4983 }
4984
4985 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4986 _instruction_uses_vl = true;
4987 assert(UseAVX > 0, "requires some form of AVX");
4988 emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);
4989 }
4990
4991 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4992 _instruction_uses_vl = true;
4993 assert(UseAVX > 0, "requires some form of AVX");
4994 if (VM_Version::supports_evex()) {
4995 _tuple_type = EVEX_FV;
4996 _input_size_in_bits = EVEX_32bit;
4997 }
4998 emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector_len);
4999 }
5000
5001
5002 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5003 assert(VM_Version::supports_avx(), "");
5004 int vector_len = AVX_256bit;
5005 if (VM_Version::supports_evex()) {
5006 vector_len = AVX_512bit;
5007 }
5008 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5009 emit_int8(0x18);
5010 emit_int8((unsigned char)(0xC0 | encode));
5011 // 0x00 - insert into lower 128 bits
5012 // 0x01 - insert into upper 128 bits
5013 emit_int8(0x01);
5014 }
5015
5016 void Assembler::vinsertf64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5017 assert(VM_Version::supports_evex(), "");
5018 int vector_len = AVX_512bit;
5019 int src_enc = src->encoding();
5020 int dst_enc = dst->encoding();
5021 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5022 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5023 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5024 emit_int8(0x1A);
5025 emit_int8((unsigned char)(0xC0 | encode));
5026 // 0x00 - insert into lower 256 bits
5027 // 0x01 - insert into upper 256 bits
5028 emit_int8(0x01);
5029 }
5030
5031 void Assembler::vinsertf64x4h(XMMRegister dst, Address src) {
5032 assert(VM_Version::supports_evex(), "");
5033 _tuple_type = EVEX_T4;
5034 _input_size_in_bits = EVEX_64bit;
5035 InstructionMark im(this);
5036 int vector_len = AVX_512bit;
5037 assert(dst != xnoreg, "sanity");
5038 int dst_enc = dst->encoding();
5039 // swap src<->dst for encoding
5040 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ true, vector_len);
5041 emit_int8(0x1A);
5042 emit_operand(dst, src);
5043 // 0x01 - insert into upper 128 bits
5044 emit_int8(0x01);
5045 }
5046
5047 void Assembler::vinsertf32x4h(XMMRegister dst, XMMRegister nds, XMMRegister src, int value) {
5048 assert(VM_Version::supports_evex(), "");
5049 int vector_len = AVX_512bit;
5050 int src_enc = src->encoding();
5051 int dst_enc = dst->encoding();
5052 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5053 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5054 /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5055 emit_int8(0x18);
5056 emit_int8((unsigned char)(0xC0 | encode));
5057 // 0x00 - insert into q0 128 bits (0..127)
5058 // 0x01 - insert into q1 128 bits (128..255)
5059 // 0x02 - insert into q2 128 bits (256..383)
5060 // 0x03 - insert into q3 128 bits (384..511)
5061 emit_int8(value & 0x3);
5062 }
5063
5064 void Assembler::vinsertf32x4h(XMMRegister dst, Address src, int value) {
5065 assert(VM_Version::supports_evex(), "");
5066 _tuple_type = EVEX_T4;
5067 _input_size_in_bits = EVEX_32bit;
5068 InstructionMark im(this);
5069 int vector_len = AVX_512bit;
5070 assert(dst != xnoreg, "sanity");
5071 int dst_enc = dst->encoding();
5072 // swap src<->dst for encoding
5073 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5074 emit_int8(0x18);
5075 emit_operand(dst, src);
5076 // 0x00 - insert into q0 128 bits (0..127)
5077 // 0x01 - insert into q1 128 bits (128..255)
5078 // 0x02 - insert into q2 128 bits (256..383)
5079 // 0x03 - insert into q3 128 bits (384..511)
5080 emit_int8(value & 0x3);
5081 }
5082
5083 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
5084 assert(VM_Version::supports_avx(), "");
5085 int vector_len = AVX_256bit;
5086 if (VM_Version::supports_evex()) {
5087 _tuple_type = EVEX_T4;
5088 _input_size_in_bits = EVEX_32bit;
5089 vector_len = AVX_512bit;
5090 }
5091 InstructionMark im(this);
5092 assert(dst != xnoreg, "sanity");
5093 int dst_enc = dst->encoding();
5094 // swap src<->dst for encoding
5095 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5096 emit_int8(0x18);
5097 emit_operand(dst, src);
5098 // 0x01 - insert into upper 128 bits
5099 emit_int8(0x01);
5100 }
5101
5102 void Assembler::vextractf128h(XMMRegister dst, XMMRegister src) {
5103 assert(VM_Version::supports_avx(), "");
5104 int vector_len = AVX_256bit;
5105 if (VM_Version::supports_evex()) {
5106 vector_len = AVX_512bit;
5107 }
5108 int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5109 emit_int8(0x19);
5110 emit_int8((unsigned char)(0xC0 | encode));
5111 // 0x00 - insert into lower 128 bits
5112 // 0x01 - insert into upper 128 bits
5113 emit_int8(0x01);
5114 }
5115
5116 void Assembler::vextractf128h(Address dst, XMMRegister src) {
5117 assert(VM_Version::supports_avx(), "");
5118 int vector_len = AVX_256bit;
5119 if (VM_Version::supports_evex()) {
5120 _tuple_type = EVEX_T4;
5121 _input_size_in_bits = EVEX_32bit;
5122 vector_len = AVX_512bit;
5123 }
5124 InstructionMark im(this);
5125 assert(src != xnoreg, "sanity");
5126 int src_enc = src->encoding();
5127 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5128 emit_int8(0x19);
5129 emit_operand(src, dst);
5130 // 0x01 - extract from upper 128 bits
5131 emit_int8(0x01);
5132 }
5133
5134 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5135 assert(VM_Version::supports_avx2(), "");
5136 int vector_len = AVX_256bit;
5137 if (VM_Version::supports_evex()) {
5138 vector_len = AVX_512bit;
5139 }
5140 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5141 emit_int8(0x38);
5142 emit_int8((unsigned char)(0xC0 | encode));
5143 // 0x00 - insert into lower 128 bits
5144 // 0x01 - insert into upper 128 bits
5145 emit_int8(0x01);
5146 }
5147
5148 void Assembler::vinserti64x4h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
5149 assert(VM_Version::supports_evex(), "");
5150 int vector_len = AVX_512bit;
5151 int src_enc = src->encoding();
5152 int dst_enc = dst->encoding();
5153 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
5154 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5155 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_reg_mask */ false);
5156 emit_int8(0x38);
5157 emit_int8((unsigned char)(0xC0 | encode));
5158 // 0x00 - insert into lower 256 bits
5159 // 0x01 - insert into upper 256 bits
5160 emit_int8(0x01);
5161 }
5162
5163 void Assembler::vinserti128h(XMMRegister dst, Address src) {
5164 assert(VM_Version::supports_avx2(), "");
5165 int vector_len = AVX_256bit;
5166 if (VM_Version::supports_evex()) {
5167 _tuple_type = EVEX_T4;
5168 _input_size_in_bits = EVEX_32bit;
5169 vector_len = AVX_512bit;
5170 }
5171 InstructionMark im(this);
5172 assert(dst != xnoreg, "sanity");
5173 int dst_enc = dst->encoding();
5174 // swap src<->dst for encoding
5175 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5176 emit_int8(0x38);
5177 emit_operand(dst, src);
5178 // 0x01 - insert into upper 128 bits
5179 emit_int8(0x01);
5180 }
5181
5182 void Assembler::vextracti128h(XMMRegister dst, XMMRegister src) {
5183 assert(VM_Version::supports_avx(), "");
5184 int vector_len = AVX_256bit;
5185 if (VM_Version::supports_evex()) {
5186 vector_len = AVX_512bit;
5187 }
5188 int encode = vex_prefix_and_encode(src, xnoreg, dst, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A);
5189 emit_int8(0x39);
5190 emit_int8((unsigned char)(0xC0 | encode));
5191 // 0x00 - insert into lower 128 bits
5192 // 0x01 - insert into upper 128 bits
5193 emit_int8(0x01);
5194 }
5195
5196 void Assembler::vextracti128h(Address dst, XMMRegister src) {
5197 assert(VM_Version::supports_avx2(), "");
5198 int vector_len = AVX_256bit;
5199 if (VM_Version::supports_evex()) {
5200 _tuple_type = EVEX_T4;
5201 _input_size_in_bits = EVEX_32bit;
5202 vector_len = AVX_512bit;
5203 }
5204 InstructionMark im(this);
5205 assert(src != xnoreg, "sanity");
5206 int src_enc = src->encoding();
5207 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5208 emit_int8(0x39);
5209 emit_operand(src, dst);
5210 // 0x01 - extract from upper 128 bits
5211 emit_int8(0x01);
5212 }
5213
5214 void Assembler::vextracti64x4h(XMMRegister dst, XMMRegister src) {
5215 assert(VM_Version::supports_evex(), "");
5216 int vector_len = AVX_512bit;
5217 int src_enc = src->encoding();
5218 int dst_enc = dst->encoding();
5219 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5220 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5221 emit_int8(0x3B);
5222 emit_int8((unsigned char)(0xC0 | encode));
5223 // 0x01 - extract from upper 256 bits
5224 emit_int8(0x01);
5225 }
5226
5227 void Assembler::vextracti64x2h(XMMRegister dst, XMMRegister src, int value) {
5228 assert(VM_Version::supports_evex(), "");
5229 int vector_len = AVX_512bit;
5230 int src_enc = src->encoding();
5231 int dst_enc = dst->encoding();
5232 int encode;
5233 if (VM_Version::supports_avx512dq()) {
5234 encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5235 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5236 } else {
5237 encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5238 /* vex_w */ false, vector_len, /* legacy_mode */ true, /* no_mask_reg */ false);
5239 }
5240 emit_int8(0x39);
5241 emit_int8((unsigned char)(0xC0 | encode));
5242 // 0x01 - extract from bits 255:128
5243 // 0x02 - extract from bits 383:256
5244 // 0x03 - extract from bits 511:384
5245 emit_int8(value & 0x3);
5246 }
5247
5248 void Assembler::vextractf64x4h(XMMRegister dst, XMMRegister src) {
5249 assert(VM_Version::supports_evex(), "");
5250 int vector_len = AVX_512bit;
5251 int src_enc = src->encoding();
5252 int dst_enc = dst->encoding();
5253 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5254 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5255 emit_int8(0x1B);
5256 emit_int8((unsigned char)(0xC0 | encode));
5257 // 0x01 - extract from upper 256 bits
5258 emit_int8(0x01);
5259 }
5260
5261 void Assembler::vextractf64x4h(Address dst, XMMRegister src) {
5262 assert(VM_Version::supports_evex(), "");
5263 _tuple_type = EVEX_T4;
5264 _input_size_in_bits = EVEX_64bit;
5265 InstructionMark im(this);
5266 int vector_len = AVX_512bit;
5267 assert(src != xnoreg, "sanity");
5268 int src_enc = src->encoding();
5269 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5270 /* vex_w */ true, vector_len);
5271 emit_int8(0x1B);
5272 emit_operand(src, dst);
5273 // 0x01 - extract from upper 256 bits
5274 emit_int8(0x01);
5275 }
5276
5277 void Assembler::vextractf32x4h(XMMRegister dst, XMMRegister src, int value) {
5278 assert(VM_Version::supports_evex(), "");
5279 int vector_len = AVX_512bit;
5280 int src_enc = src->encoding();
5281 int dst_enc = dst->encoding();
5282 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5283 /* vex_w */ false, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5284 emit_int8(0x19);
5285 emit_int8((unsigned char)(0xC0 | encode));
5286 // 0x00 - extract from bits 127:0
5287 // 0x01 - extract from bits 255:128
5288 // 0x02 - extract from bits 383:256
5289 // 0x03 - extract from bits 511:384
5290 emit_int8(value & 0x3);
5291 }
5292
5293 void Assembler::vextractf32x4h(Address dst, XMMRegister src, int value) {
5294 assert(VM_Version::supports_evex(), "");
5295 _tuple_type = EVEX_T4;
5296 _input_size_in_bits = EVEX_32bit;
5297 InstructionMark im(this);
5298 int vector_len = AVX_512bit;
5299 assert(src != xnoreg, "sanity");
5300 int src_enc = src->encoding();
5301 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, /* vex_w */ false, vector_len);
5302 emit_int8(0x19);
5303 emit_operand(src, dst);
5304 // 0x00 - extract from bits 127:0
5305 // 0x01 - extract from bits 255:128
5306 // 0x02 - extract from bits 383:256
5307 // 0x03 - extract from bits 511:384
5308 emit_int8(value & 0x3);
5309 }
5310
5311 void Assembler::vextractf64x2h(XMMRegister dst, XMMRegister src, int value) {
5312 assert(VM_Version::supports_evex(), "");
5313 int vector_len = AVX_512bit;
5314 int src_enc = src->encoding();
5315 int dst_enc = dst->encoding();
5316 int encode = vex_prefix_and_encode(src_enc, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A,
5317 /* vex_w */ !_legacy_mode_dq, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5318 emit_int8(0x19);
5319 emit_int8((unsigned char)(0xC0 | encode));
5320 // 0x01 - extract from bits 255:128
5321 // 0x02 - extract from bits 383:256
5322 // 0x03 - extract from bits 511:384
5323 emit_int8(value & 0x3);
5324 }
5325
5326 // duplicate 4-bytes integer data from src into 8 locations in dest
5327 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
5328 _instruction_uses_vl = true;
5329 assert(UseAVX > 1, "");
5330 int vector_len = AVX_256bit;
5331 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5332 emit_int8(0x58);
5333 emit_int8((unsigned char)(0xC0 | encode));
5334 }
5335
5336 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5337 void Assembler::evpbroadcastb(XMMRegister dst, XMMRegister src, int vector_len) {
5338 _instruction_uses_vl = true;
5339 assert(UseAVX > 1, "");
5340 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5341 emit_int8(0x78);
5342 emit_int8((unsigned char)(0xC0 | encode));
5343 }
5344
5345 void Assembler::evpbroadcastb(XMMRegister dst, Address src, int vector_len) {
5346 _instruction_uses_vl = true;
5347 assert(UseAVX > 1, "");
5348 _tuple_type = EVEX_T1S;
5349 _input_size_in_bits = EVEX_8bit;
5350 InstructionMark im(this);
5351 assert(dst != xnoreg, "sanity");
5352 int dst_enc = dst->encoding();
5353 // swap src<->dst for encoding
5354 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5355 emit_int8(0x78);
5356 emit_operand(dst, src);
5357 }
5358
5359 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5360 void Assembler::evpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len) {
5361 _instruction_uses_vl = true;
5362 assert(UseAVX > 1, "");
5363 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5364 emit_int8(0x79);
5365 emit_int8((unsigned char)(0xC0 | encode));
5366 }
5367
5368 void Assembler::evpbroadcastw(XMMRegister dst, Address src, int vector_len) {
5369 _instruction_uses_vl = true;
5370 assert(UseAVX > 1, "");
5371 _tuple_type = EVEX_T1S;
5372 _input_size_in_bits = EVEX_16bit;
5373 InstructionMark im(this);
5374 assert(dst != xnoreg, "sanity");
5375 int dst_enc = dst->encoding();
5376 // swap src<->dst for encoding
5377 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5378 emit_int8(0x79);
5379 emit_operand(dst, src);
5380 }
5381
5382 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5383 void Assembler::evpbroadcastd(XMMRegister dst, XMMRegister src, int vector_len) {
5384 _instruction_uses_vl = true;
5385 assert(UseAVX > 1, "");
5386 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_38);
5387 emit_int8(0x58);
5388 emit_int8((unsigned char)(0xC0 | encode));
5389 }
5390
5391 void Assembler::evpbroadcastd(XMMRegister dst, Address src, int vector_len) {
5392 _instruction_uses_vl = true;
5393 assert(UseAVX > 1, "");
5394 _tuple_type = EVEX_T1S;
5395 _input_size_in_bits = EVEX_32bit;
5396 InstructionMark im(this);
5397 assert(dst != xnoreg, "sanity");
5398 int dst_enc = dst->encoding();
5399 // swap src<->dst for encoding
5400 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5401 emit_int8(0x58);
5402 emit_operand(dst, src);
5403 }
5404
5405 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5406 void Assembler::evpbroadcastq(XMMRegister dst, XMMRegister src, int vector_len) {
5407 _instruction_uses_vl = true;
5408 assert(UseAVX > 1, "");
5409 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5410 /* vex_w */ true, vector_len, /* legacy_mode */ false, /* no_mask_reg */ false);
5411 emit_int8(0x59);
5412 emit_int8((unsigned char)(0xC0 | encode));
5413 }
5414
5415 void Assembler::evpbroadcastq(XMMRegister dst, Address src, int vector_len) {
5416 _instruction_uses_vl = true;
5417 assert(UseAVX > 1, "");
5418 _tuple_type = EVEX_T1S;
5419 _input_size_in_bits = EVEX_64bit;
5420 InstructionMark im(this);
5421 assert(dst != xnoreg, "sanity");
5422 int dst_enc = dst->encoding();
5423 // swap src<->dst for encoding
5424 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
5425 emit_int8(0x59);
5426 emit_operand(dst, src);
5427 }
5428
5429 // duplicate single precision fp from src into 4|8|16 locations in dest : requires AVX512VL
5430 void Assembler::evpbroadcastss(XMMRegister dst, XMMRegister src, int vector_len) {
5431 _instruction_uses_vl = true;
5432 assert(UseAVX > 1, "");
5433 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5434 /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5435 emit_int8(0x18);
5436 emit_int8((unsigned char)(0xC0 | encode));
5437 }
5438
5439 void Assembler::evpbroadcastss(XMMRegister dst, Address src, int vector_len) {
5440 assert(UseAVX > 1, "");
5441 _tuple_type = EVEX_T1S;
5442 _input_size_in_bits = EVEX_32bit;
5443 InstructionMark im(this);
5444 assert(dst != xnoreg, "sanity");
5445 int dst_enc = dst->encoding();
5446 // swap src<->dst for encoding
5447 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ false, vector_len);
5448 emit_int8(0x18);
5449 emit_operand(dst, src);
5450 }
5451
5452 // duplicate double precision fp from src into 2|4|8 locations in dest : requires AVX512VL
5453 void Assembler::evpbroadcastsd(XMMRegister dst, XMMRegister src, int vector_len) {
5454 _instruction_uses_vl = true;
5455 assert(UseAVX > 1, "");
5456 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5457 /*vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5458 emit_int8(0x19);
5459 emit_int8((unsigned char)(0xC0 | encode));
5460 }
5461
5462 void Assembler::evpbroadcastsd(XMMRegister dst, Address src, int vector_len) {
5463 _instruction_uses_vl = true;
5464 assert(UseAVX > 1, "");
5465 _tuple_type = EVEX_T1S;
5466 _input_size_in_bits = EVEX_64bit;
5467 InstructionMark im(this);
5468 assert(dst != xnoreg, "sanity");
5469 int dst_enc = dst->encoding();
5470 // swap src<->dst for encoding
5471 vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, /* vex_w */ true, vector_len);
5472 emit_int8(0x19);
5473 emit_operand(dst, src);
5474 }
5475
5476 // duplicate 1-byte integer data from src into 16||32|64 locations in dest : requires AVX512BW and AVX512VL
5477 void Assembler::evpbroadcastb(XMMRegister dst, Register src, int vector_len) {
5478 _instruction_uses_vl = true;
5479 assert(VM_Version::supports_evex(), "");
5480 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5481 /*vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5482 emit_int8(0x7A);
5483 emit_int8((unsigned char)(0xC0 | encode));
5484 }
5485
5486 // duplicate 2-byte integer data from src into 8|16||32 locations in dest : requires AVX512BW and AVX512VL
5487 void Assembler::evpbroadcastw(XMMRegister dst, Register src, int vector_len) {
5488 _instruction_uses_vl = true;
5489 assert(VM_Version::supports_evex(), "");
5490 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5491 /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5492 emit_int8(0x7B);
5493 emit_int8((unsigned char)(0xC0 | encode));
5494 }
5495
5496 // duplicate 4-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5497 void Assembler::evpbroadcastd(XMMRegister dst, Register src, int vector_len) {
5498 _instruction_uses_vl = true;
5499 assert(VM_Version::supports_evex(), "");
5500 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5501 /* vex_w */ false, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5502 emit_int8(0x7C);
5503 emit_int8((unsigned char)(0xC0 | encode));
5504 }
5505
5506 // duplicate 8-byte integer data from src into 4|8|16 locations in dest : requires AVX512VL
5507 void Assembler::evpbroadcastq(XMMRegister dst, Register src, int vector_len) {
5508 _instruction_uses_vl = true;
5509 assert(VM_Version::supports_evex(), "");
5510 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38,
5511 /* vex_w */ true, vector_len, /* legacy_mode */ false, /*no_mask_reg */ false);
5512 emit_int8(0x7C);
5513 emit_int8((unsigned char)(0xC0 | encode));
5514 }
5515
5516 // Carry-Less Multiplication Quadword
5517 void Assembler::pclmulqdq(XMMRegister dst, XMMRegister src, int mask) {
5518 assert(VM_Version::supports_clmul(), "");
5519 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, /* no_mask_reg */ false,
5520 VEX_OPCODE_0F_3A, /* rex_w */ false, AVX_128bit, /* legacy_mode */ true);
5521 emit_int8(0x44);
5522 emit_int8((unsigned char)(0xC0 | encode));
5523 emit_int8((unsigned char)mask);
5524 }
5525
5526 // Carry-Less Multiplication Quadword
5527 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
5528 assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
5529 int vector_len = AVX_128bit;
5530 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* legacy_mode */ true);
5531 emit_int8(0x44);
5532 emit_int8((unsigned char)(0xC0 | encode));
5533 emit_int8((unsigned char)mask);
5534 }
5535
5536 void Assembler::vzeroupper() {
5537 assert(VM_Version::supports_avx(), "");
5538 if (UseAVX < 3)
5539 {
5540 (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
5541 emit_int8(0x77);
5542 }
5543 }
5544
5545
5546 #ifndef _LP64
5547 // 32bit only pieces of the assembler
5548
5549 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5550 // NO PREFIX AS NEVER 64BIT
5551 InstructionMark im(this);
5552 emit_int8((unsigned char)0x81);
5553 emit_int8((unsigned char)(0xF8 | src1->encoding()));
5554 emit_data(imm32, rspec, 0);
5555 }
5556
5557 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5558 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
5559 InstructionMark im(this);
5560 emit_int8((unsigned char)0x81);
5561 emit_operand(rdi, src1);
5562 emit_data(imm32, rspec, 0);
5563 }
5564
5565 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
5566 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
5567 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
5568 void Assembler::cmpxchg8(Address adr) {
5569 InstructionMark im(this);
5570 emit_int8(0x0F);
5571 emit_int8((unsigned char)0xC7);
5572 emit_operand(rcx, adr);
5573 }
5574
5575 void Assembler::decl(Register dst) {
5576 // Don't use it directly. Use MacroAssembler::decrementl() instead.
5577 emit_int8(0x48 | dst->encoding());
5578 }
5579
5580 #endif // _LP64
5581
5582 // 64bit typically doesn't use the x87 but needs to for the trig funcs
5583
5584 void Assembler::fabs() {
5585 emit_int8((unsigned char)0xD9);
5586 emit_int8((unsigned char)0xE1);
5587 }
5588
5589 void Assembler::fadd(int i) {
5590 emit_farith(0xD8, 0xC0, i);
5591 }
5592
5593 void Assembler::fadd_d(Address src) {
5594 InstructionMark im(this);
5595 emit_int8((unsigned char)0xDC);
5596 emit_operand32(rax, src);
5597 }
5598
5599 void Assembler::fadd_s(Address src) {
5600 InstructionMark im(this);
5601 emit_int8((unsigned char)0xD8);
5602 emit_operand32(rax, src);
5603 }
5604
5605 void Assembler::fadda(int i) {
5606 emit_farith(0xDC, 0xC0, i);
5607 }
5608
5609 void Assembler::faddp(int i) {
5610 emit_farith(0xDE, 0xC0, i);
5611 }
5612
5613 void Assembler::fchs() {
5614 emit_int8((unsigned char)0xD9);
5615 emit_int8((unsigned char)0xE0);
5616 }
5617
5618 void Assembler::fcom(int i) {
5619 emit_farith(0xD8, 0xD0, i);
5620 }
5621
5622 void Assembler::fcomp(int i) {
5623 emit_farith(0xD8, 0xD8, i);
5624 }
5625
5626 void Assembler::fcomp_d(Address src) {
5627 InstructionMark im(this);
5628 emit_int8((unsigned char)0xDC);
5629 emit_operand32(rbx, src);
5630 }
5631
5632 void Assembler::fcomp_s(Address src) {
5633 InstructionMark im(this);
5634 emit_int8((unsigned char)0xD8);
5635 emit_operand32(rbx, src);
5636 }
5637
5638 void Assembler::fcompp() {
5639 emit_int8((unsigned char)0xDE);
5640 emit_int8((unsigned char)0xD9);
5641 }
5642
5643 void Assembler::fcos() {
5644 emit_int8((unsigned char)0xD9);
5645 emit_int8((unsigned char)0xFF);
5646 }
5647
5648 void Assembler::fdecstp() {
5649 emit_int8((unsigned char)0xD9);
5650 emit_int8((unsigned char)0xF6);
5651 }
5652
5653 void Assembler::fdiv(int i) {
5654 emit_farith(0xD8, 0xF0, i);
5655 }
5656
5657 void Assembler::fdiv_d(Address src) {
5658 InstructionMark im(this);
5659 emit_int8((unsigned char)0xDC);
5660 emit_operand32(rsi, src);
5661 }
5662
5663 void Assembler::fdiv_s(Address src) {
5664 InstructionMark im(this);
5665 emit_int8((unsigned char)0xD8);
5666 emit_operand32(rsi, src);
5667 }
5668
5669 void Assembler::fdiva(int i) {
5670 emit_farith(0xDC, 0xF8, i);
5671 }
5672
5673 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
5674 // is erroneous for some of the floating-point instructions below.
5675
5676 void Assembler::fdivp(int i) {
5677 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
5678 }
5679
5680 void Assembler::fdivr(int i) {
5681 emit_farith(0xD8, 0xF8, i);
5682 }
5683
5684 void Assembler::fdivr_d(Address src) {
5685 InstructionMark im(this);
5686 emit_int8((unsigned char)0xDC);
5687 emit_operand32(rdi, src);
5688 }
5689
5690 void Assembler::fdivr_s(Address src) {
5691 InstructionMark im(this);
5692 emit_int8((unsigned char)0xD8);
5693 emit_operand32(rdi, src);
5694 }
5695
5696 void Assembler::fdivra(int i) {
5697 emit_farith(0xDC, 0xF0, i);
5698 }
5699
5700 void Assembler::fdivrp(int i) {
5701 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
5702 }
5703
5704 void Assembler::ffree(int i) {
5705 emit_farith(0xDD, 0xC0, i);
5706 }
5707
5708 void Assembler::fild_d(Address adr) {
5709 InstructionMark im(this);
5710 emit_int8((unsigned char)0xDF);
5711 emit_operand32(rbp, adr);
5712 }
5713
5714 void Assembler::fild_s(Address adr) {
5715 InstructionMark im(this);
5716 emit_int8((unsigned char)0xDB);
5717 emit_operand32(rax, adr);
5718 }
5719
5720 void Assembler::fincstp() {
5721 emit_int8((unsigned char)0xD9);
5722 emit_int8((unsigned char)0xF7);
5723 }
5724
5725 void Assembler::finit() {
5726 emit_int8((unsigned char)0x9B);
5727 emit_int8((unsigned char)0xDB);
5728 emit_int8((unsigned char)0xE3);
5729 }
5730
5731 void Assembler::fist_s(Address adr) {
5732 InstructionMark im(this);
5733 emit_int8((unsigned char)0xDB);
5734 emit_operand32(rdx, adr);
5735 }
5736
5737 void Assembler::fistp_d(Address adr) {
5738 InstructionMark im(this);
5739 emit_int8((unsigned char)0xDF);
5740 emit_operand32(rdi, adr);
5741 }
5742
5743 void Assembler::fistp_s(Address adr) {
5744 InstructionMark im(this);
5745 emit_int8((unsigned char)0xDB);
5746 emit_operand32(rbx, adr);
5747 }
5748
5749 void Assembler::fld1() {
5750 emit_int8((unsigned char)0xD9);
5751 emit_int8((unsigned char)0xE8);
5752 }
5753
5754 void Assembler::fld_d(Address adr) {
5755 InstructionMark im(this);
5756 emit_int8((unsigned char)0xDD);
5757 emit_operand32(rax, adr);
5758 }
5759
5760 void Assembler::fld_s(Address adr) {
5761 InstructionMark im(this);
5762 emit_int8((unsigned char)0xD9);
5763 emit_operand32(rax, adr);
5764 }
5765
5766
5767 void Assembler::fld_s(int index) {
5768 emit_farith(0xD9, 0xC0, index);
5769 }
5770
5771 void Assembler::fld_x(Address adr) {
5772 InstructionMark im(this);
5773 emit_int8((unsigned char)0xDB);
5774 emit_operand32(rbp, adr);
5775 }
5776
5777 void Assembler::fldcw(Address src) {
5778 InstructionMark im(this);
5779 emit_int8((unsigned char)0xD9);
5780 emit_operand32(rbp, src);
5781 }
5782
5783 void Assembler::fldenv(Address src) {
5784 InstructionMark im(this);
5785 emit_int8((unsigned char)0xD9);
5786 emit_operand32(rsp, src);
5787 }
5788
5789 void Assembler::fldlg2() {
5790 emit_int8((unsigned char)0xD9);
5791 emit_int8((unsigned char)0xEC);
5792 }
5793
5794 void Assembler::fldln2() {
5795 emit_int8((unsigned char)0xD9);
5796 emit_int8((unsigned char)0xED);
5797 }
5798
5799 void Assembler::fldz() {
5800 emit_int8((unsigned char)0xD9);
5801 emit_int8((unsigned char)0xEE);
5802 }
5803
5804 void Assembler::flog() {
5805 fldln2();
5806 fxch();
5807 fyl2x();
5808 }
5809
5810 void Assembler::flog10() {
5811 fldlg2();
5812 fxch();
5813 fyl2x();
5814 }
5815
5816 void Assembler::fmul(int i) {
5817 emit_farith(0xD8, 0xC8, i);
5818 }
5819
5820 void Assembler::fmul_d(Address src) {
5821 InstructionMark im(this);
5822 emit_int8((unsigned char)0xDC);
5823 emit_operand32(rcx, src);
5824 }
5825
5826 void Assembler::fmul_s(Address src) {
5827 InstructionMark im(this);
5828 emit_int8((unsigned char)0xD8);
5829 emit_operand32(rcx, src);
5830 }
5831
5832 void Assembler::fmula(int i) {
5833 emit_farith(0xDC, 0xC8, i);
5834 }
5835
5836 void Assembler::fmulp(int i) {
5837 emit_farith(0xDE, 0xC8, i);
5838 }
5839
5840 void Assembler::fnsave(Address dst) {
5841 InstructionMark im(this);
5842 emit_int8((unsigned char)0xDD);
5843 emit_operand32(rsi, dst);
5844 }
5845
5846 void Assembler::fnstcw(Address src) {
5847 InstructionMark im(this);
5848 emit_int8((unsigned char)0x9B);
5849 emit_int8((unsigned char)0xD9);
5850 emit_operand32(rdi, src);
5851 }
5852
5853 void Assembler::fnstsw_ax() {
5854 emit_int8((unsigned char)0xDF);
5855 emit_int8((unsigned char)0xE0);
5856 }
5857
5858 void Assembler::fprem() {
5859 emit_int8((unsigned char)0xD9);
5860 emit_int8((unsigned char)0xF8);
5861 }
5862
5863 void Assembler::fprem1() {
5864 emit_int8((unsigned char)0xD9);
5865 emit_int8((unsigned char)0xF5);
5866 }
5867
5868 void Assembler::frstor(Address src) {
5869 InstructionMark im(this);
5870 emit_int8((unsigned char)0xDD);
5871 emit_operand32(rsp, src);
5872 }
5873
5874 void Assembler::fsin() {
5875 emit_int8((unsigned char)0xD9);
5876 emit_int8((unsigned char)0xFE);
5877 }
5878
5879 void Assembler::fsqrt() {
5880 emit_int8((unsigned char)0xD9);
5881 emit_int8((unsigned char)0xFA);
5882 }
5883
5884 void Assembler::fst_d(Address adr) {
5885 InstructionMark im(this);
5886 emit_int8((unsigned char)0xDD);
5887 emit_operand32(rdx, adr);
5888 }
5889
5890 void Assembler::fst_s(Address adr) {
5891 InstructionMark im(this);
5892 emit_int8((unsigned char)0xD9);
5893 emit_operand32(rdx, adr);
5894 }
5895
5896 void Assembler::fstp_d(Address adr) {
5897 InstructionMark im(this);
5898 emit_int8((unsigned char)0xDD);
5899 emit_operand32(rbx, adr);
5900 }
5901
5902 void Assembler::fstp_d(int index) {
5903 emit_farith(0xDD, 0xD8, index);
5904 }
5905
5906 void Assembler::fstp_s(Address adr) {
5907 InstructionMark im(this);
5908 emit_int8((unsigned char)0xD9);
5909 emit_operand32(rbx, adr);
5910 }
5911
5912 void Assembler::fstp_x(Address adr) {
5913 InstructionMark im(this);
5914 emit_int8((unsigned char)0xDB);
5915 emit_operand32(rdi, adr);
5916 }
5917
5918 void Assembler::fsub(int i) {
5919 emit_farith(0xD8, 0xE0, i);
5920 }
5921
5922 void Assembler::fsub_d(Address src) {
5923 InstructionMark im(this);
5924 emit_int8((unsigned char)0xDC);
5925 emit_operand32(rsp, src);
5926 }
5927
5928 void Assembler::fsub_s(Address src) {
5929 InstructionMark im(this);
5930 emit_int8((unsigned char)0xD8);
5931 emit_operand32(rsp, src);
5932 }
5933
5934 void Assembler::fsuba(int i) {
5935 emit_farith(0xDC, 0xE8, i);
5936 }
5937
5938 void Assembler::fsubp(int i) {
5939 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
5940 }
5941
5942 void Assembler::fsubr(int i) {
5943 emit_farith(0xD8, 0xE8, i);
5944 }
5945
5946 void Assembler::fsubr_d(Address src) {
5947 InstructionMark im(this);
5948 emit_int8((unsigned char)0xDC);
5949 emit_operand32(rbp, src);
5950 }
5951
5952 void Assembler::fsubr_s(Address src) {
5953 InstructionMark im(this);
5954 emit_int8((unsigned char)0xD8);
5955 emit_operand32(rbp, src);
5956 }
5957
5958 void Assembler::fsubra(int i) {
5959 emit_farith(0xDC, 0xE0, i);
5960 }
5961
5962 void Assembler::fsubrp(int i) {
5963 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
5964 }
5965
5966 void Assembler::ftan() {
5967 emit_int8((unsigned char)0xD9);
5968 emit_int8((unsigned char)0xF2);
5969 emit_int8((unsigned char)0xDD);
5970 emit_int8((unsigned char)0xD8);
5971 }
5972
5973 void Assembler::ftst() {
5974 emit_int8((unsigned char)0xD9);
5975 emit_int8((unsigned char)0xE4);
5976 }
5977
5978 void Assembler::fucomi(int i) {
5979 // make sure the instruction is supported (introduced for P6, together with cmov)
5980 guarantee(VM_Version::supports_cmov(), "illegal instruction");
5981 emit_farith(0xDB, 0xE8, i);
5982 }
5983
5984 void Assembler::fucomip(int i) {
5985 // make sure the instruction is supported (introduced for P6, together with cmov)
5986 guarantee(VM_Version::supports_cmov(), "illegal instruction");
5987 emit_farith(0xDF, 0xE8, i);
5988 }
5989
5990 void Assembler::fwait() {
5991 emit_int8((unsigned char)0x9B);
5992 }
5993
5994 void Assembler::fxch(int i) {
5995 emit_farith(0xD9, 0xC8, i);
5996 }
5997
5998 void Assembler::fyl2x() {
5999 emit_int8((unsigned char)0xD9);
6000 emit_int8((unsigned char)0xF1);
6001 }
6002
6003 void Assembler::frndint() {
6004 emit_int8((unsigned char)0xD9);
6005 emit_int8((unsigned char)0xFC);
6006 }
6007
6008 void Assembler::f2xm1() {
6009 emit_int8((unsigned char)0xD9);
6010 emit_int8((unsigned char)0xF0);
6011 }
6012
6013 void Assembler::fldl2e() {
6014 emit_int8((unsigned char)0xD9);
6015 emit_int8((unsigned char)0xEA);
6016 }
6017
6018 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
6019 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
6020 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
6021 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
6022
6023 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
6024 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6025 if (pre > 0) {
6026 emit_int8(simd_pre[pre]);
6027 }
6028 if (rex_w) {
6029 prefixq(adr, xreg);
6030 } else {
6031 prefix(adr, xreg);
6032 }
6033 if (opc > 0) {
6034 emit_int8(0x0F);
6035 int opc2 = simd_opc[opc];
6036 if (opc2 > 0) {
6037 emit_int8(opc2);
6038 }
6039 }
6040 }
6041
6042 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
6043 if (pre > 0) {
6044 emit_int8(simd_pre[pre]);
6045 }
6046 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
6047 prefix_and_encode(dst_enc, src_enc);
6048 if (opc > 0) {
6049 emit_int8(0x0F);
6050 int opc2 = simd_opc[opc];
6051 if (opc2 > 0) {
6052 emit_int8(opc2);
6053 }
6054 }
6055 return encode;
6056 }
6057
6058
6059 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, int vector_len) {
6060 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
6061 prefix(VEX_3bytes);
6062
6063 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
6064 byte1 = (~byte1) & 0xE0;
6065 byte1 |= opc;
6066 emit_int8(byte1);
6067
6068 int byte2 = ((~nds_enc) & 0xf) << 3;
6069 byte2 |= (vex_w ? VEX_W : 0) | ((vector_len > 0) ? 4 : 0) | pre;
6070 emit_int8(byte2);
6071 } else {
6072 prefix(VEX_2bytes);
6073
6074 int byte1 = vex_r ? VEX_R : 0;
6075 byte1 = (~byte1) & 0x80;
6076 byte1 |= ((~nds_enc) & 0xf) << 3;
6077 byte1 |= ((vector_len > 0 ) ? 4 : 0) | pre;
6078 emit_int8(byte1);
6079 }
6080 }
6081
6082 // This is a 4 byte encoding
6083 void Assembler::evex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, bool evex_r, bool evex_v,
6084 int nds_enc, VexSimdPrefix pre, VexOpcode opc,
6085 bool is_extended_context, bool is_merge_context,
6086 int vector_len, bool no_mask_reg ){
6087 // EVEX 0x62 prefix
6088 prefix(EVEX_4bytes);
6089 _evex_encoding = (vex_w ? VEX_W : 0) | (evex_r ? EVEX_Rb : 0);
6090
6091 // P0: byte 2, initialized to RXBR`00mm
6092 // instead of not'd
6093 int byte2 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0) | (evex_r ? EVEX_Rb : 0);
6094 byte2 = (~byte2) & 0xF0;
6095 // confine opc opcode extensions in mm bits to lower two bits
6096 // of form {0F, 0F_38, 0F_3A}
6097 byte2 |= opc;
6098 emit_int8(byte2);
6099
6100 // P1: byte 3 as Wvvvv1pp
6101 int byte3 = ((~nds_enc) & 0xf) << 3;
6102 // p[10] is always 1
6103 byte3 |= EVEX_F;
6104 byte3 |= (vex_w & 1) << 7;
6105 // confine pre opcode extensions in pp bits to lower two bits
6106 // of form {66, F3, F2}
6107 byte3 |= pre;
6108 emit_int8(byte3);
6109
6110 // P2: byte 4 as zL'Lbv'aaa
6111 int byte4 = (no_mask_reg) ? 0 : 1; // kregs are implemented in the low 3 bits as aaa (hard code k1, it will be initialized for now)
6112 // EVEX.v` for extending EVEX.vvvv or VIDX
6113 byte4 |= (evex_v ? 0: EVEX_V);
6114 // third EXEC.b for broadcast actions
6115 byte4 |= (is_extended_context ? EVEX_Rb : 0);
6116 // fourth EVEX.L'L for vector length : 0 is 128, 1 is 256, 2 is 512, currently we do not support 1024
6117 byte4 |= ((vector_len) & 0x3) << 5;
6118 // last is EVEX.z for zero/merge actions
6119 byte4 |= (is_merge_context ? EVEX_Z : 0);
6120 emit_int8(byte4);
6121 }
6122
6123 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre,
6124 VexOpcode opc, bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg) {
6125 bool vex_r = ((xreg_enc & 8) == 8) ? 1 : 0;
6126 bool vex_b = adr.base_needs_rex();
6127 bool vex_x = adr.index_needs_rex();
6128 _avx_vector_len = vector_len;
6129
6130 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6131 if (_legacy_mode_vl && _instruction_uses_vl) {
6132 switch (vector_len) {
6133 case AVX_128bit:
6134 case AVX_256bit:
6135 legacy_mode = true;
6136 break;
6137 }
6138 }
6139
6140 if ((UseAVX > 2) && (legacy_mode == false))
6141 {
6142 bool evex_r = (xreg_enc >= 16);
6143 bool evex_v = (nds_enc >= 16);
6144 _is_evex_instruction = true;
6145 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6146 } else {
6147 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
6148 }
6149 _instruction_uses_vl = false;
6150 }
6151
6152 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc,
6153 bool vex_w, int vector_len, bool legacy_mode, bool no_mask_reg ) {
6154 bool vex_r = ((dst_enc & 8) == 8) ? 1 : 0;
6155 bool vex_b = ((src_enc & 8) == 8) ? 1 : 0;
6156 bool vex_x = false;
6157 _avx_vector_len = vector_len;
6158
6159 // if vector length is turned off, revert to AVX for vectors smaller than 512-bit
6160 if (_legacy_mode_vl && _instruction_uses_vl) {
6161 switch (vector_len) {
6162 case AVX_128bit:
6163 case AVX_256bit:
6164 legacy_mode = true;
6165 break;
6166 }
6167 }
6168
6169 if ((UseAVX > 2) && (legacy_mode == false))
6170 {
6171 bool evex_r = (dst_enc >= 16);
6172 bool evex_v = (nds_enc >= 16);
6173 // can use vex_x as bank extender on rm encoding
6174 vex_x = (src_enc >= 16);
6175 evex_prefix(vex_r, vex_b, vex_x, vex_w, evex_r, evex_v, nds_enc, pre, opc, false, false, vector_len, no_mask_reg);
6176 } else {
6177 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector_len);
6178 }
6179
6180 _instruction_uses_vl = false;
6181
6182 // return modrm byte components for operands
6183 return (((dst_enc & 7) << 3) | (src_enc & 7));
6184 }
6185
6186
6187 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre,
6188 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
6189 if (UseAVX > 0) {
6190 int xreg_enc = xreg->encoding();
6191 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6192 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
6193 } else {
6194 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
6195 rex_prefix(adr, xreg, pre, opc, rex_w);
6196 }
6197 }
6198
6199 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre,
6200 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len, bool legacy_mode) {
6201 int dst_enc = dst->encoding();
6202 int src_enc = src->encoding();
6203 if (UseAVX > 0) {
6204 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6205 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, legacy_mode, no_mask_reg);
6206 } else {
6207 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
6208 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
6209 }
6210 }
6211
6212 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, KRegister src, VexSimdPrefix pre,
6213 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
6214 int dst_enc = dst->encoding();
6215 int src_enc = src->encoding();
6216 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6217 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
6218 }
6219
6220 int Assembler::kreg_prefix_and_encode(KRegister dst, KRegister nds, Register src, VexSimdPrefix pre,
6221 bool no_mask_reg, VexOpcode opc, bool rex_w, int vector_len) {
6222 int dst_enc = dst->encoding();
6223 int src_enc = src->encoding();
6224 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6225 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector_len, true, no_mask_reg);
6226 }
6227
6228 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6229 InstructionMark im(this);
6230 simd_prefix(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6231 emit_int8(opcode);
6232 emit_operand(dst, src);
6233 }
6234
6235 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool no_mask_reg) {
6236 InstructionMark im(this);
6237 simd_prefix_q(dst, dst, src, pre, no_mask_reg);
6238 emit_int8(opcode);
6239 emit_operand(dst, src);
6240 }
6241
6242 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6243 int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6244 emit_int8(opcode);
6245 emit_int8((unsigned char)(0xC0 | encode));
6246 }
6247
6248 void Assembler::emit_simd_arith_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
6249 int encode = simd_prefix_and_encode(dst, dst, src, pre, no_mask_reg, VEX_OPCODE_0F, true, AVX_128bit);
6250 emit_int8(opcode);
6251 emit_int8((unsigned char)(0xC0 | encode));
6252 }
6253
6254 // Versions with no second source register (non-destructive source).
6255 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
6256 InstructionMark im(this);
6257 simd_prefix(dst, xnoreg, src, pre, opNoRegMask);
6258 emit_int8(opcode);
6259 emit_operand(dst, src);
6260 }
6261
6262 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre, bool opNoRegMask) {
6263 InstructionMark im(this);
6264 simd_prefix_q(dst, xnoreg, src, pre, opNoRegMask);
6265 emit_int8(opcode);
6266 emit_operand(dst, src);
6267 }
6268
6269 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg, bool legacy_mode) {
6270 int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, false, AVX_128bit, legacy_mode);
6271 emit_int8(opcode);
6272 emit_int8((unsigned char)(0xC0 | encode));
6273 }
6274
6275 void Assembler::emit_simd_arith_nonds_q(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre, bool no_mask_reg) {
6276 int encode = simd_prefix_and_encode(dst, xnoreg, src, pre, no_mask_reg, VEX_OPCODE_0F, true);
6277 emit_int8(opcode);
6278 emit_int8((unsigned char)(0xC0 | encode));
6279 }
6280
6281 // 3-operands AVX instructions
6282 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, Address src,
6283 VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
6284 InstructionMark im(this);
6285 vex_prefix(dst, nds, src, pre, vector_len, no_mask_reg, legacy_mode);
6286 emit_int8(opcode);
6287 emit_operand(dst, src);
6288 }
6289
6290 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds,
6291 Address src, VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
6292 InstructionMark im(this);
6293 vex_prefix_q(dst, nds, src, pre, vector_len, no_mask_reg);
6294 emit_int8(opcode);
6295 emit_operand(dst, src);
6296 }
6297
6298 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
6299 VexSimdPrefix pre, int vector_len, bool no_mask_reg, bool legacy_mode) {
6300 int encode = vex_prefix_and_encode(dst, nds, src, pre, vector_len, VEX_OPCODE_0F, legacy_mode, no_mask_reg);
6301 emit_int8(opcode);
6302 emit_int8((unsigned char)(0xC0 | encode));
6303 }
6304
6305 void Assembler::emit_vex_arith_q(int opcode, XMMRegister dst, XMMRegister nds, XMMRegister src,
6306 VexSimdPrefix pre, int vector_len, bool no_mask_reg) {
6307 int src_enc = src->encoding();
6308 int dst_enc = dst->encoding();
6309 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
6310 int encode = vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, VEX_OPCODE_0F, true, vector_len, false, no_mask_reg);
6311 emit_int8(opcode);
6312 emit_int8((unsigned char)(0xC0 | encode));
6313 }
6314
6315 void Assembler::cmppd(XMMRegister dst, XMMRegister nds, XMMRegister src, int cop, int vector_len) {
6316 assert(VM_Version::supports_avx(), "");
6317 assert(!VM_Version::supports_evex(), "");
6318 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector_len, VEX_OPCODE_0F, /* no_mask_reg */ false);
6319 emit_int8((unsigned char)0xC2);
6320 emit_int8((unsigned char)(0xC0 | encode));
6321 emit_int8((unsigned char)(0xF & cop));
6322 }
6323
6324 void Assembler::vpblendd(XMMRegister dst, XMMRegister nds, XMMRegister src1, XMMRegister src2, int vector_len) {
6325 assert(VM_Version::supports_avx(), "");
6326 assert(!VM_Version::supports_evex(), "");
6327 int encode = vex_prefix_and_encode(dst, nds, src1, VEX_SIMD_66, vector_len, VEX_OPCODE_0F_3A, /* no_mask_reg */ false);
6328 emit_int8((unsigned char)0x4B);
6329 emit_int8((unsigned char)(0xC0 | encode));
6330 int src2_enc = src2->encoding();
6331 emit_int8((unsigned char)(0xF0 & src2_enc<<4));
6332 }
6333
6334
6335 #ifndef _LP64
6336
6337 void Assembler::incl(Register dst) {
6338 // Don't use it directly. Use MacroAssembler::incrementl() instead.
6339 emit_int8(0x40 | dst->encoding());
6340 }
6341
6342 void Assembler::lea(Register dst, Address src) {
6343 leal(dst, src);
6344 }
6345
6346 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
6347 InstructionMark im(this);
6348 emit_int8((unsigned char)0xC7);
6349 emit_operand(rax, dst);
6350 emit_data((int)imm32, rspec, 0);
6351 }
6352
6353 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
6354 InstructionMark im(this);
6355 int encode = prefix_and_encode(dst->encoding());
6356 emit_int8((unsigned char)(0xB8 | encode));
6357 emit_data((int)imm32, rspec, 0);
6358 }
6359
6360 void Assembler::popa() { // 32bit
6361 emit_int8(0x61);
6362 }
6363
6364 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
6365 InstructionMark im(this);
6366 emit_int8(0x68);
6367 emit_data(imm32, rspec, 0);
6368 }
6369
6370 void Assembler::pusha() { // 32bit
6371 emit_int8(0x60);
6372 }
6373
6374 void Assembler::set_byte_if_not_zero(Register dst) {
6375 emit_int8(0x0F);
6376 emit_int8((unsigned char)0x95);
6377 emit_int8((unsigned char)(0xE0 | dst->encoding()));
6378 }
6379
6380 void Assembler::shldl(Register dst, Register src) {
6381 emit_int8(0x0F);
6382 emit_int8((unsigned char)0xA5);
6383 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6384 }
6385
6386 // 0F A4 / r ib
6387 void Assembler::shldl(Register dst, Register src, int8_t imm8) {
6388 emit_int8(0x0F);
6389 emit_int8((unsigned char)0xA4);
6390 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6391 emit_int8(imm8);
6392 }
6393
6394 void Assembler::shrdl(Register dst, Register src) {
6395 emit_int8(0x0F);
6396 emit_int8((unsigned char)0xAD);
6397 emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
6398 }
6399
6400 #else // LP64
6401
6402 void Assembler::set_byte_if_not_zero(Register dst) {
6403 int enc = prefix_and_encode(dst->encoding(), true);
6404 emit_int8(0x0F);
6405 emit_int8((unsigned char)0x95);
6406 emit_int8((unsigned char)(0xE0 | enc));
6407 }
6408
6409 // 64bit only pieces of the assembler
6410 // This should only be used by 64bit instructions that can use rip-relative
6411 // it cannot be used by instructions that want an immediate value.
6412
6413 bool Assembler::reachable(AddressLiteral adr) {
6414 int64_t disp;
6415 // None will force a 64bit literal to the code stream. Likely a placeholder
6416 // for something that will be patched later and we need to certain it will
6417 // always be reachable.
6418 if (adr.reloc() == relocInfo::none) {
6419 return false;
6420 }
6421 if (adr.reloc() == relocInfo::internal_word_type) {
6422 // This should be rip relative and easily reachable.
6423 return true;
6424 }
6425 if (adr.reloc() == relocInfo::virtual_call_type ||
6426 adr.reloc() == relocInfo::opt_virtual_call_type ||
6427 adr.reloc() == relocInfo::static_call_type ||
6428 adr.reloc() == relocInfo::static_stub_type ) {
6429 // This should be rip relative within the code cache and easily
6430 // reachable until we get huge code caches. (At which point
6431 // ic code is going to have issues).
6432 return true;
6433 }
6434 if (adr.reloc() != relocInfo::external_word_type &&
6435 adr.reloc() != relocInfo::poll_return_type && // these are really external_word but need special
6436 adr.reloc() != relocInfo::poll_type && // relocs to identify them
6437 adr.reloc() != relocInfo::runtime_call_type ) {
6438 return false;
6439 }
6440
6441 // Stress the correction code
6442 if (ForceUnreachable) {
6443 // Must be runtimecall reloc, see if it is in the codecache
6444 // Flipping stuff in the codecache to be unreachable causes issues
6445 // with things like inline caches where the additional instructions
6446 // are not handled.
6447 if (CodeCache::find_blob(adr._target) == NULL) {
6448 return false;
6449 }
6450 }
6451 // For external_word_type/runtime_call_type if it is reachable from where we
6452 // are now (possibly a temp buffer) and where we might end up
6453 // anywhere in the codeCache then we are always reachable.
6454 // This would have to change if we ever save/restore shared code
6455 // to be more pessimistic.
6456 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
6457 if (!is_simm32(disp)) return false;
6458 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
6459 if (!is_simm32(disp)) return false;
6460
6461 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
6462
6463 // Because rip relative is a disp + address_of_next_instruction and we
6464 // don't know the value of address_of_next_instruction we apply a fudge factor
6465 // to make sure we will be ok no matter the size of the instruction we get placed into.
6466 // We don't have to fudge the checks above here because they are already worst case.
6467
6468 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
6469 // + 4 because better safe than sorry.
6470 const int fudge = 12 + 4;
6471 if (disp < 0) {
6472 disp -= fudge;
6473 } else {
6474 disp += fudge;
6475 }
6476 return is_simm32(disp);
6477 }
6478
6479 // Check if the polling page is not reachable from the code cache using rip-relative
6480 // addressing.
6481 bool Assembler::is_polling_page_far() {
6482 intptr_t addr = (intptr_t)os::get_polling_page();
6483 return ForceUnreachable ||
6484 !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
6485 !is_simm32(addr - (intptr_t)CodeCache::high_bound());
6486 }
6487
6488 void Assembler::emit_data64(jlong data,
6489 relocInfo::relocType rtype,
6490 int format) {
6491 if (rtype == relocInfo::none) {
6492 emit_int64(data);
6493 } else {
6494 emit_data64(data, Relocation::spec_simple(rtype), format);
6495 }
6496 }
6497
6498 void Assembler::emit_data64(jlong data,
6499 RelocationHolder const& rspec,
6500 int format) {
6501 assert(imm_operand == 0, "default format must be immediate in this file");
6502 assert(imm_operand == format, "must be immediate");
6503 assert(inst_mark() != NULL, "must be inside InstructionMark");
6504 // Do not use AbstractAssembler::relocate, which is not intended for
6505 // embedded words. Instead, relocate to the enclosing instruction.
6506 code_section()->relocate(inst_mark(), rspec, format);
6507 #ifdef ASSERT
6508 check_relocation(rspec, format);
6509 #endif
6510 emit_int64(data);
6511 }
6512
6513 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
6514 if (reg_enc >= 8) {
6515 prefix(REX_B);
6516 reg_enc -= 8;
6517 } else if (byteinst && reg_enc >= 4) {
6518 prefix(REX);
6519 }
6520 return reg_enc;
6521 }
6522
6523 int Assembler::prefixq_and_encode(int reg_enc) {
6524 if (reg_enc < 8) {
6525 prefix(REX_W);
6526 } else {
6527 prefix(REX_WB);
6528 reg_enc -= 8;
6529 }
6530 return reg_enc;
6531 }
6532
6533 int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
6534 if (dst_enc < 8) {
6535 if (src_enc >= 8) {
6536 prefix(REX_B);
6537 src_enc -= 8;
6538 } else if (byteinst && src_enc >= 4) {
6539 prefix(REX);
6540 }
6541 } else {
6542 if (src_enc < 8) {
6543 prefix(REX_R);
6544 } else {
6545 prefix(REX_RB);
6546 src_enc -= 8;
6547 }
6548 dst_enc -= 8;
6549 }
6550 return dst_enc << 3 | src_enc;
6551 }
6552
6553 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
6554 if (dst_enc < 8) {
6555 if (src_enc < 8) {
6556 prefix(REX_W);
6557 } else {
6558 prefix(REX_WB);
6559 src_enc -= 8;
6560 }
6561 } else {
6562 if (src_enc < 8) {
6563 prefix(REX_WR);
6564 } else {
6565 prefix(REX_WRB);
6566 src_enc -= 8;
6567 }
6568 dst_enc -= 8;
6569 }
6570 return dst_enc << 3 | src_enc;
6571 }
6572
6573 void Assembler::prefix(Register reg) {
6574 if (reg->encoding() >= 8) {
6575 prefix(REX_B);
6576 }
6577 }
6578
6579 void Assembler::prefix(Register dst, Register src, Prefix p) {
6580 if (src->encoding() >= 8) {
6581 p = (Prefix)(p | REX_B);
6582 }
6583 if (dst->encoding() >= 8) {
6584 p = (Prefix)( p | REX_R);
6585 }
6586 if (p != Prefix_EMPTY) {
6587 // do not generate an empty prefix
6588 prefix(p);
6589 }
6590 }
6591
6592 void Assembler::prefix(Register dst, Address adr, Prefix p) {
6593 if (adr.base_needs_rex()) {
6594 if (adr.index_needs_rex()) {
6595 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6596 } else {
6597 prefix(REX_B);
6598 }
6599 } else {
6600 if (adr.index_needs_rex()) {
6601 assert(false, "prefix(Register dst, Address adr, Prefix p) does not support handling of an X");
6602 }
6603 }
6604 if (dst->encoding() >= 8) {
6605 p = (Prefix)(p | REX_R);
6606 }
6607 if (p != Prefix_EMPTY) {
6608 // do not generate an empty prefix
6609 prefix(p);
6610 }
6611 }
6612
6613 void Assembler::prefix(Address adr) {
6614 if (adr.base_needs_rex()) {
6615 if (adr.index_needs_rex()) {
6616 prefix(REX_XB);
6617 } else {
6618 prefix(REX_B);
6619 }
6620 } else {
6621 if (adr.index_needs_rex()) {
6622 prefix(REX_X);
6623 }
6624 }
6625 }
6626
6627 void Assembler::prefixq(Address adr) {
6628 if (adr.base_needs_rex()) {
6629 if (adr.index_needs_rex()) {
6630 prefix(REX_WXB);
6631 } else {
6632 prefix(REX_WB);
6633 }
6634 } else {
6635 if (adr.index_needs_rex()) {
6636 prefix(REX_WX);
6637 } else {
6638 prefix(REX_W);
6639 }
6640 }
6641 }
6642
6643
6644 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
6645 if (reg->encoding() < 8) {
6646 if (adr.base_needs_rex()) {
6647 if (adr.index_needs_rex()) {
6648 prefix(REX_XB);
6649 } else {
6650 prefix(REX_B);
6651 }
6652 } else {
6653 if (adr.index_needs_rex()) {
6654 prefix(REX_X);
6655 } else if (byteinst && reg->encoding() >= 4 ) {
6656 prefix(REX);
6657 }
6658 }
6659 } else {
6660 if (adr.base_needs_rex()) {
6661 if (adr.index_needs_rex()) {
6662 prefix(REX_RXB);
6663 } else {
6664 prefix(REX_RB);
6665 }
6666 } else {
6667 if (adr.index_needs_rex()) {
6668 prefix(REX_RX);
6669 } else {
6670 prefix(REX_R);
6671 }
6672 }
6673 }
6674 }
6675
6676 void Assembler::prefixq(Address adr, Register src) {
6677 if (src->encoding() < 8) {
6678 if (adr.base_needs_rex()) {
6679 if (adr.index_needs_rex()) {
6680 prefix(REX_WXB);
6681 } else {
6682 prefix(REX_WB);
6683 }
6684 } else {
6685 if (adr.index_needs_rex()) {
6686 prefix(REX_WX);
6687 } else {
6688 prefix(REX_W);
6689 }
6690 }
6691 } else {
6692 if (adr.base_needs_rex()) {
6693 if (adr.index_needs_rex()) {
6694 prefix(REX_WRXB);
6695 } else {
6696 prefix(REX_WRB);
6697 }
6698 } else {
6699 if (adr.index_needs_rex()) {
6700 prefix(REX_WRX);
6701 } else {
6702 prefix(REX_WR);
6703 }
6704 }
6705 }
6706 }
6707
6708 void Assembler::prefix(Address adr, XMMRegister reg) {
6709 if (reg->encoding() < 8) {
6710 if (adr.base_needs_rex()) {
6711 if (adr.index_needs_rex()) {
6712 prefix(REX_XB);
6713 } else {
6714 prefix(REX_B);
6715 }
6716 } else {
6717 if (adr.index_needs_rex()) {
6718 prefix(REX_X);
6719 }
6720 }
6721 } else {
6722 if (adr.base_needs_rex()) {
6723 if (adr.index_needs_rex()) {
6724 prefix(REX_RXB);
6725 } else {
6726 prefix(REX_RB);
6727 }
6728 } else {
6729 if (adr.index_needs_rex()) {
6730 prefix(REX_RX);
6731 } else {
6732 prefix(REX_R);
6733 }
6734 }
6735 }
6736 }
6737
6738 void Assembler::prefixq(Address adr, XMMRegister src) {
6739 if (src->encoding() < 8) {
6740 if (adr.base_needs_rex()) {
6741 if (adr.index_needs_rex()) {
6742 prefix(REX_WXB);
6743 } else {
6744 prefix(REX_WB);
6745 }
6746 } else {
6747 if (adr.index_needs_rex()) {
6748 prefix(REX_WX);
6749 } else {
6750 prefix(REX_W);
6751 }
6752 }
6753 } else {
6754 if (adr.base_needs_rex()) {
6755 if (adr.index_needs_rex()) {
6756 prefix(REX_WRXB);
6757 } else {
6758 prefix(REX_WRB);
6759 }
6760 } else {
6761 if (adr.index_needs_rex()) {
6762 prefix(REX_WRX);
6763 } else {
6764 prefix(REX_WR);
6765 }
6766 }
6767 }
6768 }
6769
6770 void Assembler::adcq(Register dst, int32_t imm32) {
6771 (void) prefixq_and_encode(dst->encoding());
6772 emit_arith(0x81, 0xD0, dst, imm32);
6773 }
6774
6775 void Assembler::adcq(Register dst, Address src) {
6776 InstructionMark im(this);
6777 prefixq(src, dst);
6778 emit_int8(0x13);
6779 emit_operand(dst, src);
6780 }
6781
6782 void Assembler::adcq(Register dst, Register src) {
6783 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6784 emit_arith(0x13, 0xC0, dst, src);
6785 }
6786
6787 void Assembler::addq(Address dst, int32_t imm32) {
6788 InstructionMark im(this);
6789 prefixq(dst);
6790 emit_arith_operand(0x81, rax, dst,imm32);
6791 }
6792
6793 void Assembler::addq(Address dst, Register src) {
6794 InstructionMark im(this);
6795 prefixq(dst, src);
6796 emit_int8(0x01);
6797 emit_operand(src, dst);
6798 }
6799
6800 void Assembler::addq(Register dst, int32_t imm32) {
6801 (void) prefixq_and_encode(dst->encoding());
6802 emit_arith(0x81, 0xC0, dst, imm32);
6803 }
6804
6805 void Assembler::addq(Register dst, Address src) {
6806 InstructionMark im(this);
6807 prefixq(src, dst);
6808 emit_int8(0x03);
6809 emit_operand(dst, src);
6810 }
6811
6812 void Assembler::addq(Register dst, Register src) {
6813 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6814 emit_arith(0x03, 0xC0, dst, src);
6815 }
6816
6817 void Assembler::adcxq(Register dst, Register src) {
6818 //assert(VM_Version::supports_adx(), "adx instructions not supported");
6819 emit_int8((unsigned char)0x66);
6820 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6821 emit_int8(0x0F);
6822 emit_int8(0x38);
6823 emit_int8((unsigned char)0xF6);
6824 emit_int8((unsigned char)(0xC0 | encode));
6825 }
6826
6827 void Assembler::adoxq(Register dst, Register src) {
6828 //assert(VM_Version::supports_adx(), "adx instructions not supported");
6829 emit_int8((unsigned char)0xF3);
6830 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6831 emit_int8(0x0F);
6832 emit_int8(0x38);
6833 emit_int8((unsigned char)0xF6);
6834 emit_int8((unsigned char)(0xC0 | encode));
6835 }
6836
6837 void Assembler::andq(Address dst, int32_t imm32) {
6838 InstructionMark im(this);
6839 prefixq(dst);
6840 emit_int8((unsigned char)0x81);
6841 emit_operand(rsp, dst, 4);
6842 emit_int32(imm32);
6843 }
6844
6845 void Assembler::andq(Register dst, int32_t imm32) {
6846 (void) prefixq_and_encode(dst->encoding());
6847 emit_arith(0x81, 0xE0, dst, imm32);
6848 }
6849
6850 void Assembler::andq(Register dst, Address src) {
6851 InstructionMark im(this);
6852 prefixq(src, dst);
6853 emit_int8(0x23);
6854 emit_operand(dst, src);
6855 }
6856
6857 void Assembler::andq(Register dst, Register src) {
6858 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6859 emit_arith(0x23, 0xC0, dst, src);
6860 }
6861
6862 void Assembler::andnq(Register dst, Register src1, Register src2) {
6863 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6864 int encode = vex_prefix_0F38_and_encode_q_legacy(dst, src1, src2);
6865 emit_int8((unsigned char)0xF2);
6866 emit_int8((unsigned char)(0xC0 | encode));
6867 }
6868
6869 void Assembler::andnq(Register dst, Register src1, Address src2) {
6870 InstructionMark im(this);
6871 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6872 vex_prefix_0F38_q_legacy(dst, src1, src2);
6873 emit_int8((unsigned char)0xF2);
6874 emit_operand(dst, src2);
6875 }
6876
6877 void Assembler::bsfq(Register dst, Register src) {
6878 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6879 emit_int8(0x0F);
6880 emit_int8((unsigned char)0xBC);
6881 emit_int8((unsigned char)(0xC0 | encode));
6882 }
6883
6884 void Assembler::bsrq(Register dst, Register src) {
6885 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6886 emit_int8(0x0F);
6887 emit_int8((unsigned char)0xBD);
6888 emit_int8((unsigned char)(0xC0 | encode));
6889 }
6890
6891 void Assembler::bswapq(Register reg) {
6892 int encode = prefixq_and_encode(reg->encoding());
6893 emit_int8(0x0F);
6894 emit_int8((unsigned char)(0xC8 | encode));
6895 }
6896
6897 void Assembler::blsiq(Register dst, Register src) {
6898 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6899 int encode = vex_prefix_0F38_and_encode_q_legacy(rbx, dst, src);
6900 emit_int8((unsigned char)0xF3);
6901 emit_int8((unsigned char)(0xC0 | encode));
6902 }
6903
6904 void Assembler::blsiq(Register dst, Address src) {
6905 InstructionMark im(this);
6906 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6907 vex_prefix_0F38_q_legacy(rbx, dst, src);
6908 emit_int8((unsigned char)0xF3);
6909 emit_operand(rbx, src);
6910 }
6911
6912 void Assembler::blsmskq(Register dst, Register src) {
6913 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6914 int encode = vex_prefix_0F38_and_encode_q_legacy(rdx, dst, src);
6915 emit_int8((unsigned char)0xF3);
6916 emit_int8((unsigned char)(0xC0 | encode));
6917 }
6918
6919 void Assembler::blsmskq(Register dst, Address src) {
6920 InstructionMark im(this);
6921 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6922 vex_prefix_0F38_q_legacy(rdx, dst, src);
6923 emit_int8((unsigned char)0xF3);
6924 emit_operand(rdx, src);
6925 }
6926
6927 void Assembler::blsrq(Register dst, Register src) {
6928 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6929 int encode = vex_prefix_0F38_and_encode_q_legacy(rcx, dst, src);
6930 emit_int8((unsigned char)0xF3);
6931 emit_int8((unsigned char)(0xC0 | encode));
6932 }
6933
6934 void Assembler::blsrq(Register dst, Address src) {
6935 InstructionMark im(this);
6936 assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
6937 vex_prefix_0F38_q_legacy(rcx, dst, src);
6938 emit_int8((unsigned char)0xF3);
6939 emit_operand(rcx, src);
6940 }
6941
6942 void Assembler::cdqq() {
6943 prefix(REX_W);
6944 emit_int8((unsigned char)0x99);
6945 }
6946
6947 void Assembler::clflush(Address adr) {
6948 prefix(adr);
6949 emit_int8(0x0F);
6950 emit_int8((unsigned char)0xAE);
6951 emit_operand(rdi, adr);
6952 }
6953
6954 void Assembler::cmovq(Condition cc, Register dst, Register src) {
6955 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
6956 emit_int8(0x0F);
6957 emit_int8(0x40 | cc);
6958 emit_int8((unsigned char)(0xC0 | encode));
6959 }
6960
6961 void Assembler::cmovq(Condition cc, Register dst, Address src) {
6962 InstructionMark im(this);
6963 prefixq(src, dst);
6964 emit_int8(0x0F);
6965 emit_int8(0x40 | cc);
6966 emit_operand(dst, src);
6967 }
6968
6969 void Assembler::cmpq(Address dst, int32_t imm32) {
6970 InstructionMark im(this);
6971 prefixq(dst);
6972 emit_int8((unsigned char)0x81);
6973 emit_operand(rdi, dst, 4);
6974 emit_int32(imm32);
6975 }
6976
6977 void Assembler::cmpq(Register dst, int32_t imm32) {
6978 (void) prefixq_and_encode(dst->encoding());
6979 emit_arith(0x81, 0xF8, dst, imm32);
6980 }
6981
6982 void Assembler::cmpq(Address dst, Register src) {
6983 InstructionMark im(this);
6984 prefixq(dst, src);
6985 emit_int8(0x3B);
6986 emit_operand(src, dst);
6987 }
6988
6989 void Assembler::cmpq(Register dst, Register src) {
6990 (void) prefixq_and_encode(dst->encoding(), src->encoding());
6991 emit_arith(0x3B, 0xC0, dst, src);
6992 }
6993
6994 void Assembler::cmpq(Register dst, Address src) {
6995 InstructionMark im(this);
6996 prefixq(src, dst);
6997 emit_int8(0x3B);
6998 emit_operand(dst, src);
6999 }
7000
7001 void Assembler::cmpxchgq(Register reg, Address adr) {
7002 InstructionMark im(this);
7003 prefixq(adr, reg);
7004 emit_int8(0x0F);
7005 emit_int8((unsigned char)0xB1);
7006 emit_operand(reg, adr);
7007 }
7008
7009 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
7010 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7011 int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
7012 emit_int8(0x2A);
7013 emit_int8((unsigned char)(0xC0 | encode));
7014 }
7015
7016 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
7017 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7018 if (VM_Version::supports_evex()) {
7019 _tuple_type = EVEX_T1S;
7020 _input_size_in_bits = EVEX_32bit;
7021 }
7022 InstructionMark im(this);
7023 simd_prefix_q(dst, dst, src, VEX_SIMD_F2, /* no_mask_reg */ true);
7024 emit_int8(0x2A);
7025 emit_operand(dst, src);
7026 }
7027
7028 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
7029 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7030 if (VM_Version::supports_evex()) {
7031 _tuple_type = EVEX_T1S;
7032 _input_size_in_bits = EVEX_32bit;
7033 }
7034 InstructionMark im(this);
7035 simd_prefix_q(dst, dst, src, VEX_SIMD_F3, /* no_mask_reg */ true);
7036 emit_int8(0x2A);
7037 emit_operand(dst, src);
7038 }
7039
7040 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
7041 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7042 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2, VEX_OPCODE_0F, /* no_mask_reg */ true);
7043 emit_int8(0x2C);
7044 emit_int8((unsigned char)(0xC0 | encode));
7045 }
7046
7047 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
7048 NOT_LP64(assert(VM_Version::supports_sse(), ""));
7049 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3, VEX_OPCODE_0F, /* no_mask_reg */ true);
7050 emit_int8(0x2C);
7051 emit_int8((unsigned char)(0xC0 | encode));
7052 }
7053
7054 void Assembler::decl(Register dst) {
7055 // Don't use it directly. Use MacroAssembler::decrementl() instead.
7056 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
7057 int encode = prefix_and_encode(dst->encoding());
7058 emit_int8((unsigned char)0xFF);
7059 emit_int8((unsigned char)(0xC8 | encode));
7060 }
7061
7062 void Assembler::decq(Register dst) {
7063 // Don't use it directly. Use MacroAssembler::decrementq() instead.
7064 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7065 int encode = prefixq_and_encode(dst->encoding());
7066 emit_int8((unsigned char)0xFF);
7067 emit_int8(0xC8 | encode);
7068 }
7069
7070 void Assembler::decq(Address dst) {
7071 // Don't use it directly. Use MacroAssembler::decrementq() instead.
7072 InstructionMark im(this);
7073 prefixq(dst);
7074 emit_int8((unsigned char)0xFF);
7075 emit_operand(rcx, dst);
7076 }
7077
7078 void Assembler::fxrstor(Address src) {
7079 prefixq(src);
7080 emit_int8(0x0F);
7081 emit_int8((unsigned char)0xAE);
7082 emit_operand(as_Register(1), src);
7083 }
7084
7085 void Assembler::xrstor(Address src) {
7086 prefixq(src);
7087 emit_int8(0x0F);
7088 emit_int8((unsigned char)0xAE);
7089 emit_operand(as_Register(5), src);
7090 }
7091
7092 void Assembler::fxsave(Address dst) {
7093 prefixq(dst);
7094 emit_int8(0x0F);
7095 emit_int8((unsigned char)0xAE);
7096 emit_operand(as_Register(0), dst);
7097 }
7098
7099 void Assembler::xsave(Address dst) {
7100 prefixq(dst);
7101 emit_int8(0x0F);
7102 emit_int8((unsigned char)0xAE);
7103 emit_operand(as_Register(4), dst);
7104 }
7105
7106 void Assembler::idivq(Register src) {
7107 int encode = prefixq_and_encode(src->encoding());
7108 emit_int8((unsigned char)0xF7);
7109 emit_int8((unsigned char)(0xF8 | encode));
7110 }
7111
7112 void Assembler::imulq(Register dst, Register src) {
7113 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7114 emit_int8(0x0F);
7115 emit_int8((unsigned char)0xAF);
7116 emit_int8((unsigned char)(0xC0 | encode));
7117 }
7118
7119 void Assembler::imulq(Register dst, Register src, int value) {
7120 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7121 if (is8bit(value)) {
7122 emit_int8(0x6B);
7123 emit_int8((unsigned char)(0xC0 | encode));
7124 emit_int8(value & 0xFF);
7125 } else {
7126 emit_int8(0x69);
7127 emit_int8((unsigned char)(0xC0 | encode));
7128 emit_int32(value);
7129 }
7130 }
7131
7132 void Assembler::imulq(Register dst, Address src) {
7133 InstructionMark im(this);
7134 prefixq(src, dst);
7135 emit_int8(0x0F);
7136 emit_int8((unsigned char) 0xAF);
7137 emit_operand(dst, src);
7138 }
7139
7140 void Assembler::incl(Register dst) {
7141 // Don't use it directly. Use MacroAssembler::incrementl() instead.
7142 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7143 int encode = prefix_and_encode(dst->encoding());
7144 emit_int8((unsigned char)0xFF);
7145 emit_int8((unsigned char)(0xC0 | encode));
7146 }
7147
7148 void Assembler::incq(Register dst) {
7149 // Don't use it directly. Use MacroAssembler::incrementq() instead.
7150 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
7151 int encode = prefixq_and_encode(dst->encoding());
7152 emit_int8((unsigned char)0xFF);
7153 emit_int8((unsigned char)(0xC0 | encode));
7154 }
7155
7156 void Assembler::incq(Address dst) {
7157 // Don't use it directly. Use MacroAssembler::incrementq() instead.
7158 InstructionMark im(this);
7159 prefixq(dst);
7160 emit_int8((unsigned char)0xFF);
7161 emit_operand(rax, dst);
7162 }
7163
7164 void Assembler::lea(Register dst, Address src) {
7165 leaq(dst, src);
7166 }
7167
7168 void Assembler::leaq(Register dst, Address src) {
7169 InstructionMark im(this);
7170 prefixq(src, dst);
7171 emit_int8((unsigned char)0x8D);
7172 emit_operand(dst, src);
7173 }
7174
7175 void Assembler::mov64(Register dst, int64_t imm64) {
7176 InstructionMark im(this);
7177 int encode = prefixq_and_encode(dst->encoding());
7178 emit_int8((unsigned char)(0xB8 | encode));
7179 emit_int64(imm64);
7180 }
7181
7182 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
7183 InstructionMark im(this);
7184 int encode = prefixq_and_encode(dst->encoding());
7185 emit_int8(0xB8 | encode);
7186 emit_data64(imm64, rspec);
7187 }
7188
7189 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
7190 InstructionMark im(this);
7191 int encode = prefix_and_encode(dst->encoding());
7192 emit_int8((unsigned char)(0xB8 | encode));
7193 emit_data((int)imm32, rspec, narrow_oop_operand);
7194 }
7195
7196 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
7197 InstructionMark im(this);
7198 prefix(dst);
7199 emit_int8((unsigned char)0xC7);
7200 emit_operand(rax, dst, 4);
7201 emit_data((int)imm32, rspec, narrow_oop_operand);
7202 }
7203
7204 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
7205 InstructionMark im(this);
7206 int encode = prefix_and_encode(src1->encoding());
7207 emit_int8((unsigned char)0x81);
7208 emit_int8((unsigned char)(0xF8 | encode));
7209 emit_data((int)imm32, rspec, narrow_oop_operand);
7210 }
7211
7212 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
7213 InstructionMark im(this);
7214 prefix(src1);
7215 emit_int8((unsigned char)0x81);
7216 emit_operand(rax, src1, 4);
7217 emit_data((int)imm32, rspec, narrow_oop_operand);
7218 }
7219
7220 void Assembler::lzcntq(Register dst, Register src) {
7221 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
7222 emit_int8((unsigned char)0xF3);
7223 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7224 emit_int8(0x0F);
7225 emit_int8((unsigned char)0xBD);
7226 emit_int8((unsigned char)(0xC0 | encode));
7227 }
7228
7229 void Assembler::movdq(XMMRegister dst, Register src) {
7230 // table D-1 says MMX/SSE2
7231 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7232 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66, /* no_mask_reg */ true);
7233 emit_int8(0x6E);
7234 emit_int8((unsigned char)(0xC0 | encode));
7235 }
7236
7237 void Assembler::movdq(Register dst, XMMRegister src) {
7238 // table D-1 says MMX/SSE2
7239 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
7240 // swap src/dst to get correct prefix
7241 int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66, /* no_mask_reg */ true);
7242 emit_int8(0x7E);
7243 emit_int8((unsigned char)(0xC0 | encode));
7244 }
7245
7246 void Assembler::movq(Register dst, Register src) {
7247 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7248 emit_int8((unsigned char)0x8B);
7249 emit_int8((unsigned char)(0xC0 | encode));
7250 }
7251
7252 void Assembler::movq(Register dst, Address src) {
7253 InstructionMark im(this);
7254 prefixq(src, dst);
7255 emit_int8((unsigned char)0x8B);
7256 emit_operand(dst, src);
7257 }
7258
7259 void Assembler::movq(Address dst, Register src) {
7260 InstructionMark im(this);
7261 prefixq(dst, src);
7262 emit_int8((unsigned char)0x89);
7263 emit_operand(src, dst);
7264 }
7265
7266 void Assembler::movsbq(Register dst, Address src) {
7267 InstructionMark im(this);
7268 prefixq(src, dst);
7269 emit_int8(0x0F);
7270 emit_int8((unsigned char)0xBE);
7271 emit_operand(dst, src);
7272 }
7273
7274 void Assembler::movsbq(Register dst, Register src) {
7275 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7276 emit_int8(0x0F);
7277 emit_int8((unsigned char)0xBE);
7278 emit_int8((unsigned char)(0xC0 | encode));
7279 }
7280
7281 void Assembler::movslq(Register dst, int32_t imm32) {
7282 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
7283 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
7284 // as a result we shouldn't use until tested at runtime...
7285 ShouldNotReachHere();
7286 InstructionMark im(this);
7287 int encode = prefixq_and_encode(dst->encoding());
7288 emit_int8((unsigned char)(0xC7 | encode));
7289 emit_int32(imm32);
7290 }
7291
7292 void Assembler::movslq(Address dst, int32_t imm32) {
7293 assert(is_simm32(imm32), "lost bits");
7294 InstructionMark im(this);
7295 prefixq(dst);
7296 emit_int8((unsigned char)0xC7);
7297 emit_operand(rax, dst, 4);
7298 emit_int32(imm32);
7299 }
7300
7301 void Assembler::movslq(Register dst, Address src) {
7302 InstructionMark im(this);
7303 prefixq(src, dst);
7304 emit_int8(0x63);
7305 emit_operand(dst, src);
7306 }
7307
7308 void Assembler::movslq(Register dst, Register src) {
7309 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7310 emit_int8(0x63);
7311 emit_int8((unsigned char)(0xC0 | encode));
7312 }
7313
7314 void Assembler::movswq(Register dst, Address src) {
7315 InstructionMark im(this);
7316 prefixq(src, dst);
7317 emit_int8(0x0F);
7318 emit_int8((unsigned char)0xBF);
7319 emit_operand(dst, src);
7320 }
7321
7322 void Assembler::movswq(Register dst, Register src) {
7323 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7324 emit_int8((unsigned char)0x0F);
7325 emit_int8((unsigned char)0xBF);
7326 emit_int8((unsigned char)(0xC0 | encode));
7327 }
7328
7329 void Assembler::movzbq(Register dst, Address src) {
7330 InstructionMark im(this);
7331 prefixq(src, dst);
7332 emit_int8((unsigned char)0x0F);
7333 emit_int8((unsigned char)0xB6);
7334 emit_operand(dst, src);
7335 }
7336
7337 void Assembler::movzbq(Register dst, Register src) {
7338 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7339 emit_int8(0x0F);
7340 emit_int8((unsigned char)0xB6);
7341 emit_int8(0xC0 | encode);
7342 }
7343
7344 void Assembler::movzwq(Register dst, Address src) {
7345 InstructionMark im(this);
7346 prefixq(src, dst);
7347 emit_int8((unsigned char)0x0F);
7348 emit_int8((unsigned char)0xB7);
7349 emit_operand(dst, src);
7350 }
7351
7352 void Assembler::movzwq(Register dst, Register src) {
7353 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7354 emit_int8((unsigned char)0x0F);
7355 emit_int8((unsigned char)0xB7);
7356 emit_int8((unsigned char)(0xC0 | encode));
7357 }
7358
7359 void Assembler::mulq(Address src) {
7360 InstructionMark im(this);
7361 prefixq(src);
7362 emit_int8((unsigned char)0xF7);
7363 emit_operand(rsp, src);
7364 }
7365
7366 void Assembler::mulq(Register src) {
7367 int encode = prefixq_and_encode(src->encoding());
7368 emit_int8((unsigned char)0xF7);
7369 emit_int8((unsigned char)(0xE0 | encode));
7370 }
7371
7372 void Assembler::mulxq(Register dst1, Register dst2, Register src) {
7373 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7374 int encode = vex_prefix_and_encode(dst1->encoding(), dst2->encoding(), src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_38,
7375 /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
7376 emit_int8((unsigned char)0xF6);
7377 emit_int8((unsigned char)(0xC0 | encode));
7378 }
7379
7380 void Assembler::negq(Register dst) {
7381 int encode = prefixq_and_encode(dst->encoding());
7382 emit_int8((unsigned char)0xF7);
7383 emit_int8((unsigned char)(0xD8 | encode));
7384 }
7385
7386 void Assembler::notq(Register dst) {
7387 int encode = prefixq_and_encode(dst->encoding());
7388 emit_int8((unsigned char)0xF7);
7389 emit_int8((unsigned char)(0xD0 | encode));
7390 }
7391
7392 void Assembler::orq(Address dst, int32_t imm32) {
7393 InstructionMark im(this);
7394 prefixq(dst);
7395 emit_int8((unsigned char)0x81);
7396 emit_operand(rcx, dst, 4);
7397 emit_int32(imm32);
7398 }
7399
7400 void Assembler::orq(Register dst, int32_t imm32) {
7401 (void) prefixq_and_encode(dst->encoding());
7402 emit_arith(0x81, 0xC8, dst, imm32);
7403 }
7404
7405 void Assembler::orq(Register dst, Address src) {
7406 InstructionMark im(this);
7407 prefixq(src, dst);
7408 emit_int8(0x0B);
7409 emit_operand(dst, src);
7410 }
7411
7412 void Assembler::orq(Register dst, Register src) {
7413 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7414 emit_arith(0x0B, 0xC0, dst, src);
7415 }
7416
7417 void Assembler::popa() { // 64bit
7418 movq(r15, Address(rsp, 0));
7419 movq(r14, Address(rsp, wordSize));
7420 movq(r13, Address(rsp, 2 * wordSize));
7421 movq(r12, Address(rsp, 3 * wordSize));
7422 movq(r11, Address(rsp, 4 * wordSize));
7423 movq(r10, Address(rsp, 5 * wordSize));
7424 movq(r9, Address(rsp, 6 * wordSize));
7425 movq(r8, Address(rsp, 7 * wordSize));
7426 movq(rdi, Address(rsp, 8 * wordSize));
7427 movq(rsi, Address(rsp, 9 * wordSize));
7428 movq(rbp, Address(rsp, 10 * wordSize));
7429 // skip rsp
7430 movq(rbx, Address(rsp, 12 * wordSize));
7431 movq(rdx, Address(rsp, 13 * wordSize));
7432 movq(rcx, Address(rsp, 14 * wordSize));
7433 movq(rax, Address(rsp, 15 * wordSize));
7434
7435 addq(rsp, 16 * wordSize);
7436 }
7437
7438 void Assembler::popcntq(Register dst, Address src) {
7439 assert(VM_Version::supports_popcnt(), "must support");
7440 InstructionMark im(this);
7441 emit_int8((unsigned char)0xF3);
7442 prefixq(src, dst);
7443 emit_int8((unsigned char)0x0F);
7444 emit_int8((unsigned char)0xB8);
7445 emit_operand(dst, src);
7446 }
7447
7448 void Assembler::popcntq(Register dst, Register src) {
7449 assert(VM_Version::supports_popcnt(), "must support");
7450 emit_int8((unsigned char)0xF3);
7451 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7452 emit_int8((unsigned char)0x0F);
7453 emit_int8((unsigned char)0xB8);
7454 emit_int8((unsigned char)(0xC0 | encode));
7455 }
7456
7457 void Assembler::popq(Address dst) {
7458 InstructionMark im(this);
7459 prefixq(dst);
7460 emit_int8((unsigned char)0x8F);
7461 emit_operand(rax, dst);
7462 }
7463
7464 void Assembler::pusha() { // 64bit
7465 // we have to store original rsp. ABI says that 128 bytes
7466 // below rsp are local scratch.
7467 movq(Address(rsp, -5 * wordSize), rsp);
7468
7469 subq(rsp, 16 * wordSize);
7470
7471 movq(Address(rsp, 15 * wordSize), rax);
7472 movq(Address(rsp, 14 * wordSize), rcx);
7473 movq(Address(rsp, 13 * wordSize), rdx);
7474 movq(Address(rsp, 12 * wordSize), rbx);
7475 // skip rsp
7476 movq(Address(rsp, 10 * wordSize), rbp);
7477 movq(Address(rsp, 9 * wordSize), rsi);
7478 movq(Address(rsp, 8 * wordSize), rdi);
7479 movq(Address(rsp, 7 * wordSize), r8);
7480 movq(Address(rsp, 6 * wordSize), r9);
7481 movq(Address(rsp, 5 * wordSize), r10);
7482 movq(Address(rsp, 4 * wordSize), r11);
7483 movq(Address(rsp, 3 * wordSize), r12);
7484 movq(Address(rsp, 2 * wordSize), r13);
7485 movq(Address(rsp, wordSize), r14);
7486 movq(Address(rsp, 0), r15);
7487 }
7488
7489 void Assembler::pushq(Address src) {
7490 InstructionMark im(this);
7491 prefixq(src);
7492 emit_int8((unsigned char)0xFF);
7493 emit_operand(rsi, src);
7494 }
7495
7496 void Assembler::rclq(Register dst, int imm8) {
7497 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7498 int encode = prefixq_and_encode(dst->encoding());
7499 if (imm8 == 1) {
7500 emit_int8((unsigned char)0xD1);
7501 emit_int8((unsigned char)(0xD0 | encode));
7502 } else {
7503 emit_int8((unsigned char)0xC1);
7504 emit_int8((unsigned char)(0xD0 | encode));
7505 emit_int8(imm8);
7506 }
7507 }
7508
7509 void Assembler::rcrq(Register dst, int imm8) {
7510 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7511 int encode = prefixq_and_encode(dst->encoding());
7512 if (imm8 == 1) {
7513 emit_int8((unsigned char)0xD1);
7514 emit_int8((unsigned char)(0xD8 | encode));
7515 } else {
7516 emit_int8((unsigned char)0xC1);
7517 emit_int8((unsigned char)(0xD8 | encode));
7518 emit_int8(imm8);
7519 }
7520 }
7521
7522 void Assembler::rorq(Register dst, int imm8) {
7523 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7524 int encode = prefixq_and_encode(dst->encoding());
7525 if (imm8 == 1) {
7526 emit_int8((unsigned char)0xD1);
7527 emit_int8((unsigned char)(0xC8 | encode));
7528 } else {
7529 emit_int8((unsigned char)0xC1);
7530 emit_int8((unsigned char)(0xc8 | encode));
7531 emit_int8(imm8);
7532 }
7533 }
7534
7535 void Assembler::rorxq(Register dst, Register src, int imm8) {
7536 assert(VM_Version::supports_bmi2(), "bit manipulation instructions not supported");
7537 int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F2, VEX_OPCODE_0F_3A,
7538 /* vex_w */ true, AVX_128bit, /* legacy_mode */ true, /* no_mask_reg */ false);
7539 emit_int8((unsigned char)0xF0);
7540 emit_int8((unsigned char)(0xC0 | encode));
7541 emit_int8(imm8);
7542 }
7543
7544 void Assembler::sarq(Register dst, int imm8) {
7545 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7546 int encode = prefixq_and_encode(dst->encoding());
7547 if (imm8 == 1) {
7548 emit_int8((unsigned char)0xD1);
7549 emit_int8((unsigned char)(0xF8 | encode));
7550 } else {
7551 emit_int8((unsigned char)0xC1);
7552 emit_int8((unsigned char)(0xF8 | encode));
7553 emit_int8(imm8);
7554 }
7555 }
7556
7557 void Assembler::sarq(Register dst) {
7558 int encode = prefixq_and_encode(dst->encoding());
7559 emit_int8((unsigned char)0xD3);
7560 emit_int8((unsigned char)(0xF8 | encode));
7561 }
7562
7563 void Assembler::sbbq(Address dst, int32_t imm32) {
7564 InstructionMark im(this);
7565 prefixq(dst);
7566 emit_arith_operand(0x81, rbx, dst, imm32);
7567 }
7568
7569 void Assembler::sbbq(Register dst, int32_t imm32) {
7570 (void) prefixq_and_encode(dst->encoding());
7571 emit_arith(0x81, 0xD8, dst, imm32);
7572 }
7573
7574 void Assembler::sbbq(Register dst, Address src) {
7575 InstructionMark im(this);
7576 prefixq(src, dst);
7577 emit_int8(0x1B);
7578 emit_operand(dst, src);
7579 }
7580
7581 void Assembler::sbbq(Register dst, Register src) {
7582 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7583 emit_arith(0x1B, 0xC0, dst, src);
7584 }
7585
7586 void Assembler::shlq(Register dst, int imm8) {
7587 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7588 int encode = prefixq_and_encode(dst->encoding());
7589 if (imm8 == 1) {
7590 emit_int8((unsigned char)0xD1);
7591 emit_int8((unsigned char)(0xE0 | encode));
7592 } else {
7593 emit_int8((unsigned char)0xC1);
7594 emit_int8((unsigned char)(0xE0 | encode));
7595 emit_int8(imm8);
7596 }
7597 }
7598
7599 void Assembler::shlq(Register dst) {
7600 int encode = prefixq_and_encode(dst->encoding());
7601 emit_int8((unsigned char)0xD3);
7602 emit_int8((unsigned char)(0xE0 | encode));
7603 }
7604
7605 void Assembler::shrq(Register dst, int imm8) {
7606 assert(isShiftCount(imm8 >> 1), "illegal shift count");
7607 int encode = prefixq_and_encode(dst->encoding());
7608 emit_int8((unsigned char)0xC1);
7609 emit_int8((unsigned char)(0xE8 | encode));
7610 emit_int8(imm8);
7611 }
7612
7613 void Assembler::shrq(Register dst) {
7614 int encode = prefixq_and_encode(dst->encoding());
7615 emit_int8((unsigned char)0xD3);
7616 emit_int8(0xE8 | encode);
7617 }
7618
7619 void Assembler::subq(Address dst, int32_t imm32) {
7620 InstructionMark im(this);
7621 prefixq(dst);
7622 emit_arith_operand(0x81, rbp, dst, imm32);
7623 }
7624
7625 void Assembler::subq(Address dst, Register src) {
7626 InstructionMark im(this);
7627 prefixq(dst, src);
7628 emit_int8(0x29);
7629 emit_operand(src, dst);
7630 }
7631
7632 void Assembler::subq(Register dst, int32_t imm32) {
7633 (void) prefixq_and_encode(dst->encoding());
7634 emit_arith(0x81, 0xE8, dst, imm32);
7635 }
7636
7637 // Force generation of a 4 byte immediate value even if it fits into 8bit
7638 void Assembler::subq_imm32(Register dst, int32_t imm32) {
7639 (void) prefixq_and_encode(dst->encoding());
7640 emit_arith_imm32(0x81, 0xE8, dst, imm32);
7641 }
7642
7643 void Assembler::subq(Register dst, Address src) {
7644 InstructionMark im(this);
7645 prefixq(src, dst);
7646 emit_int8(0x2B);
7647 emit_operand(dst, src);
7648 }
7649
7650 void Assembler::subq(Register dst, Register src) {
7651 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7652 emit_arith(0x2B, 0xC0, dst, src);
7653 }
7654
7655 void Assembler::testq(Register dst, int32_t imm32) {
7656 // not using emit_arith because test
7657 // doesn't support sign-extension of
7658 // 8bit operands
7659 int encode = dst->encoding();
7660 if (encode == 0) {
7661 prefix(REX_W);
7662 emit_int8((unsigned char)0xA9);
7663 } else {
7664 encode = prefixq_and_encode(encode);
7665 emit_int8((unsigned char)0xF7);
7666 emit_int8((unsigned char)(0xC0 | encode));
7667 }
7668 emit_int32(imm32);
7669 }
7670
7671 void Assembler::testq(Register dst, Register src) {
7672 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7673 emit_arith(0x85, 0xC0, dst, src);
7674 }
7675
7676 void Assembler::xaddq(Address dst, Register src) {
7677 InstructionMark im(this);
7678 prefixq(dst, src);
7679 emit_int8(0x0F);
7680 emit_int8((unsigned char)0xC1);
7681 emit_operand(src, dst);
7682 }
7683
7684 void Assembler::xchgq(Register dst, Address src) {
7685 InstructionMark im(this);
7686 prefixq(src, dst);
7687 emit_int8((unsigned char)0x87);
7688 emit_operand(dst, src);
7689 }
7690
7691 void Assembler::xchgq(Register dst, Register src) {
7692 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
7693 emit_int8((unsigned char)0x87);
7694 emit_int8((unsigned char)(0xc0 | encode));
7695 }
7696
7697 void Assembler::xorq(Register dst, Register src) {
7698 (void) prefixq_and_encode(dst->encoding(), src->encoding());
7699 emit_arith(0x33, 0xC0, dst, src);
7700 }
7701
7702 void Assembler::xorq(Register dst, Address src) {
7703 InstructionMark im(this);
7704 prefixq(src, dst);
7705 emit_int8(0x33);
7706 emit_operand(dst, src);
7707 }
7708
7709 #endif // !LP64