1 /*
   2  * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef CPU_X86_MACROASSEMBLER_X86_HPP
  26 #define CPU_X86_MACROASSEMBLER_X86_HPP
  27 
  28 #include "asm/assembler.hpp"
  29 #include "utilities/macros.hpp"
  30 #include "runtime/rtmLocking.hpp"
  31 
  32 // MacroAssembler extends Assembler by frequently used macros.
  33 //
  34 // Instructions for which a 'better' code sequence exists depending
  35 // on arguments should also go in here.
  36 
  37 class MacroAssembler: public Assembler {
  38   friend class LIR_Assembler;
  39   friend class Runtime1;      // as_Address()
  40 
  41  public:
  42   // Support for VM calls
  43   //
  44   // This is the base routine called by the different versions of call_VM_leaf. The interpreter
  45   // may customize this version by overriding it for its purposes (e.g., to save/restore
  46   // additional registers when doing a VM call).
  47 
  48   virtual void call_VM_leaf_base(
  49     address entry_point,               // the entry point
  50     int     number_of_arguments        // the number of arguments to pop after the call
  51   );
  52 
  53  protected:
  54   // This is the base routine called by the different versions of call_VM. The interpreter
  55   // may customize this version by overriding it for its purposes (e.g., to save/restore
  56   // additional registers when doing a VM call).
  57   //
  58   // If no java_thread register is specified (noreg) than rdi will be used instead. call_VM_base
  59   // returns the register which contains the thread upon return. If a thread register has been
  60   // specified, the return value will correspond to that register. If no last_java_sp is specified
  61   // (noreg) than rsp will be used instead.
  62   virtual void call_VM_base(           // returns the register containing the thread upon return
  63     Register oop_result,               // where an oop-result ends up if any; use noreg otherwise
  64     Register java_thread,              // the thread if computed before     ; use noreg otherwise
  65     Register last_java_sp,             // to set up last_Java_frame in stubs; use noreg otherwise
  66     address  entry_point,              // the entry point
  67     int      number_of_arguments,      // the number of arguments (w/o thread) to pop after the call
  68     bool     check_exceptions          // whether to check for pending exceptions after return
  69   );
  70 
  71   void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
  72 
  73   // helpers for FPU flag access
  74   // tmp is a temporary register, if none is available use noreg
  75   void save_rax   (Register tmp);
  76   void restore_rax(Register tmp);
  77 
  78  public:
  79   MacroAssembler(CodeBuffer* code) : Assembler(code) {}
  80 
  81  // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
  82  // The implementation is only non-empty for the InterpreterMacroAssembler,
  83  // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
  84  virtual void check_and_handle_popframe(Register java_thread);
  85  virtual void check_and_handle_earlyret(Register java_thread);
  86 
  87   Address as_Address(AddressLiteral adr);
  88   Address as_Address(ArrayAddress adr);
  89 
  90   // Support for NULL-checks
  91   //
  92   // Generates code that causes a NULL OS exception if the content of reg is NULL.
  93   // If the accessed location is M[reg + offset] and the offset is known, provide the
  94   // offset. No explicit code generation is needed if the offset is within a certain
  95   // range (0 <= offset <= page_size).
  96 
  97   void null_check(Register reg, int offset = -1);
  98   static bool needs_explicit_null_check(intptr_t offset);
  99   static bool uses_implicit_null_check(void* address);
 100 
 101   // Required platform-specific helpers for Label::patch_instructions.
 102   // They _shadow_ the declarations in AbstractAssembler, which are undefined.
 103   void pd_patch_instruction(address branch, address target, const char* file, int line) {
 104     unsigned char op = branch[0];
 105     assert(op == 0xE8 /* call */ ||
 106         op == 0xE9 /* jmp */ ||
 107         op == 0xEB /* short jmp */ ||
 108         (op & 0xF0) == 0x70 /* short jcc */ ||
 109         op == 0x0F && (branch[1] & 0xF0) == 0x80 /* jcc */ ||
 110         op == 0xC7 && branch[1] == 0xF8 /* xbegin */,
 111         "Invalid opcode at patch point");
 112 
 113     if (op == 0xEB || (op & 0xF0) == 0x70) {
 114       // short offset operators (jmp and jcc)
 115       char* disp = (char*) &branch[1];
 116       int imm8 = target - (address) &disp[1];
 117       guarantee(this->is8bit(imm8), "Short forward jump exceeds 8-bit offset at %s:%d",
 118                 file == NULL ? "<NULL>" : file, line);
 119       *disp = imm8;
 120     } else {
 121       int* disp = (int*) &branch[(op == 0x0F || op == 0xC7)? 2: 1];
 122       int imm32 = target - (address) &disp[1];
 123       *disp = imm32;
 124     }
 125   }
 126 
 127   // The following 4 methods return the offset of the appropriate move instruction
 128 
 129   // Support for fast byte/short loading with zero extension (depending on particular CPU)
 130   int load_unsigned_byte(Register dst, Address src);
 131   int load_unsigned_short(Register dst, Address src);
 132 
 133   // Support for fast byte/short loading with sign extension (depending on particular CPU)
 134   int load_signed_byte(Register dst, Address src);
 135   int load_signed_short(Register dst, Address src);
 136 
 137   // Support for sign-extension (hi:lo = extend_sign(lo))
 138   void extend_sign(Register hi, Register lo);
 139 
 140   // Load and store values by size and signed-ness
 141   void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
 142   void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
 143 
 144   // Support for inc/dec with optimal instruction selection depending on value
 145 
 146   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
 147   void decrement(Register reg, int value = 1) { LP64_ONLY(decrementq(reg, value)) NOT_LP64(decrementl(reg, value)) ; }
 148 
 149   void decrementl(Address dst, int value = 1);
 150   void decrementl(Register reg, int value = 1);
 151 
 152   void decrementq(Register reg, int value = 1);
 153   void decrementq(Address dst, int value = 1);
 154 
 155   void incrementl(Address dst, int value = 1);
 156   void incrementl(Register reg, int value = 1);
 157 
 158   void incrementq(Register reg, int value = 1);
 159   void incrementq(Address dst, int value = 1);
 160 
 161   // Support optimal SSE move instructions.
 162   void movflt(XMMRegister dst, XMMRegister src) {
 163     if (dst-> encoding() == src->encoding()) return;
 164     if (UseXmmRegToRegMoveAll) { movaps(dst, src); return; }
 165     else                       { movss (dst, src); return; }
 166   }
 167   void movflt(XMMRegister dst, Address src) { movss(dst, src); }
 168   void movflt(XMMRegister dst, AddressLiteral src);
 169   void movflt(Address dst, XMMRegister src) { movss(dst, src); }
 170 
 171   void movdbl(XMMRegister dst, XMMRegister src) {
 172     if (dst-> encoding() == src->encoding()) return;
 173     if (UseXmmRegToRegMoveAll) { movapd(dst, src); return; }
 174     else                       { movsd (dst, src); return; }
 175   }
 176 
 177   void movdbl(XMMRegister dst, AddressLiteral src);
 178 
 179   void movdbl(XMMRegister dst, Address src) {
 180     if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
 181     else                         { movlpd(dst, src); return; }
 182   }
 183   void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
 184 
 185   void incrementl(AddressLiteral dst);
 186   void incrementl(ArrayAddress dst);
 187 
 188   void incrementq(AddressLiteral dst);
 189 
 190   // Alignment
 191   void align(int modulus);
 192   void align(int modulus, int target);
 193 
 194   // A 5 byte nop that is safe for patching (see patch_verified_entry)
 195   void fat_nop();
 196 
 197   // Stack frame creation/removal
 198   void enter();
 199   void leave();
 200 
 201   // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
 202   // The pointer will be loaded into the thread register.
 203   void get_thread(Register thread);
 204 
 205 
 206   // Support for VM calls
 207   //
 208   // It is imperative that all calls into the VM are handled via the call_VM macros.
 209   // They make sure that the stack linkage is setup correctly. call_VM's correspond
 210   // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
 211 
 212 
 213   void call_VM(Register oop_result,
 214                address entry_point,
 215                bool check_exceptions = true);
 216   void call_VM(Register oop_result,
 217                address entry_point,
 218                Register arg_1,
 219                bool check_exceptions = true);
 220   void call_VM(Register oop_result,
 221                address entry_point,
 222                Register arg_1, Register arg_2,
 223                bool check_exceptions = true);
 224   void call_VM(Register oop_result,
 225                address entry_point,
 226                Register arg_1, Register arg_2, Register arg_3,
 227                bool check_exceptions = true);
 228 
 229   // Overloadings with last_Java_sp
 230   void call_VM(Register oop_result,
 231                Register last_java_sp,
 232                address entry_point,
 233                int number_of_arguments = 0,
 234                bool check_exceptions = true);
 235   void call_VM(Register oop_result,
 236                Register last_java_sp,
 237                address entry_point,
 238                Register arg_1, bool
 239                check_exceptions = true);
 240   void call_VM(Register oop_result,
 241                Register last_java_sp,
 242                address entry_point,
 243                Register arg_1, Register arg_2,
 244                bool check_exceptions = true);
 245   void call_VM(Register oop_result,
 246                Register last_java_sp,
 247                address entry_point,
 248                Register arg_1, Register arg_2, Register arg_3,
 249                bool check_exceptions = true);
 250 
 251   void get_vm_result  (Register oop_result, Register thread);
 252   void get_vm_result_2(Register metadata_result, Register thread);
 253 
 254   // These always tightly bind to MacroAssembler::call_VM_base
 255   // bypassing the virtual implementation
 256   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
 257   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
 258   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
 259   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
 260   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
 261 
 262   void call_VM_leaf0(address entry_point);
 263   void call_VM_leaf(address entry_point,
 264                     int number_of_arguments = 0);
 265   void call_VM_leaf(address entry_point,
 266                     Register arg_1);
 267   void call_VM_leaf(address entry_point,
 268                     Register arg_1, Register arg_2);
 269   void call_VM_leaf(address entry_point,
 270                     Register arg_1, Register arg_2, Register arg_3);
 271 
 272   // These always tightly bind to MacroAssembler::call_VM_leaf_base
 273   // bypassing the virtual implementation
 274   void super_call_VM_leaf(address entry_point);
 275   void super_call_VM_leaf(address entry_point, Register arg_1);
 276   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
 277   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
 278   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
 279 
 280   // last Java Frame (fills frame anchor)
 281   void set_last_Java_frame(Register thread,
 282                            Register last_java_sp,
 283                            Register last_java_fp,
 284                            address last_java_pc);
 285 
 286   // thread in the default location (r15_thread on 64bit)
 287   void set_last_Java_frame(Register last_java_sp,
 288                            Register last_java_fp,
 289                            address last_java_pc);
 290 
 291   void reset_last_Java_frame(Register thread, bool clear_fp);
 292 
 293   // thread in the default location (r15_thread on 64bit)
 294   void reset_last_Java_frame(bool clear_fp);
 295 
 296   // jobjects
 297   void clear_jweak_tag(Register possibly_jweak);
 298   void resolve_jobject(Register value, Register thread, Register tmp);
 299 
 300   // C 'boolean' to Java boolean: x == 0 ? 0 : 1
 301   void c2bool(Register x);
 302 
 303   // C++ bool manipulation
 304 
 305   void movbool(Register dst, Address src);
 306   void movbool(Address dst, bool boolconst);
 307   void movbool(Address dst, Register src);
 308   void testbool(Register dst);
 309 
 310   void resolve_oop_handle(Register result, Register tmp = rscratch2);
 311   void resolve_weak_handle(Register result, Register tmp);
 312   void load_mirror(Register mirror, Register method, Register tmp = rscratch2);
 313   void load_method_holder_cld(Register rresult, Register rmethod);
 314 
 315   void load_method_holder(Register holder, Register method);
 316 
 317   // oop manipulations
 318   void load_klass(Register dst, Register src, Register tmp);
 319   void store_klass(Register dst, Register src, Register tmp);
 320 
 321   void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 322                       Register tmp1, Register thread_tmp);
 323   void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
 324                        Register tmp1, Register tmp2);
 325 
 326   // Resolves obj access. Result is placed in the same register.
 327   // All other registers are preserved.
 328   void resolve(DecoratorSet decorators, Register obj);
 329 
 330   void load_heap_oop(Register dst, Address src, Register tmp1 = noreg,
 331                      Register thread_tmp = noreg, DecoratorSet decorators = 0);
 332   void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg,
 333                               Register thread_tmp = noreg, DecoratorSet decorators = 0);
 334   void store_heap_oop(Address dst, Register src, Register tmp1 = noreg,
 335                       Register tmp2 = noreg, DecoratorSet decorators = 0);
 336 
 337   // Used for storing NULL. All other oop constants should be
 338   // stored using routines that take a jobject.
 339   void store_heap_oop_null(Address dst);
 340 
 341   void load_prototype_header(Register dst, Register src, Register tmp);
 342 
 343 #ifdef _LP64
 344   void store_klass_gap(Register dst, Register src);
 345 
 346   // This dummy is to prevent a call to store_heap_oop from
 347   // converting a zero (like NULL) into a Register by giving
 348   // the compiler two choices it can't resolve
 349 
 350   void store_heap_oop(Address dst, void* dummy);
 351 
 352   void encode_heap_oop(Register r);
 353   void decode_heap_oop(Register r);
 354   void encode_heap_oop_not_null(Register r);
 355   void decode_heap_oop_not_null(Register r);
 356   void encode_heap_oop_not_null(Register dst, Register src);
 357   void decode_heap_oop_not_null(Register dst, Register src);
 358 
 359   void set_narrow_oop(Register dst, jobject obj);
 360   void set_narrow_oop(Address dst, jobject obj);
 361   void cmp_narrow_oop(Register dst, jobject obj);
 362   void cmp_narrow_oop(Address dst, jobject obj);
 363 
 364   void encode_klass_not_null(Register r, Register tmp);
 365   void decode_klass_not_null(Register r, Register tmp);
 366   void encode_and_move_klass_not_null(Register dst, Register src);
 367   void decode_and_move_klass_not_null(Register dst, Register src);
 368   void set_narrow_klass(Register dst, Klass* k);
 369   void set_narrow_klass(Address dst, Klass* k);
 370   void cmp_narrow_klass(Register dst, Klass* k);
 371   void cmp_narrow_klass(Address dst, Klass* k);
 372 
 373   // if heap base register is used - reinit it with the correct value
 374   void reinit_heapbase();
 375 
 376   DEBUG_ONLY(void verify_heapbase(const char* msg);)
 377 
 378 #endif // _LP64
 379 
 380   // Int division/remainder for Java
 381   // (as idivl, but checks for special case as described in JVM spec.)
 382   // returns idivl instruction offset for implicit exception handling
 383   int corrected_idivl(Register reg);
 384 
 385   // Long division/remainder for Java
 386   // (as idivq, but checks for special case as described in JVM spec.)
 387   // returns idivq instruction offset for implicit exception handling
 388   int corrected_idivq(Register reg);
 389 
 390   void int3();
 391 
 392   // Long operation macros for a 32bit cpu
 393   // Long negation for Java
 394   void lneg(Register hi, Register lo);
 395 
 396   // Long multiplication for Java
 397   // (destroys contents of eax, ebx, ecx and edx)
 398   void lmul(int x_rsp_offset, int y_rsp_offset); // rdx:rax = x * y
 399 
 400   // Long shifts for Java
 401   // (semantics as described in JVM spec.)
 402   void lshl(Register hi, Register lo);                               // hi:lo << (rcx & 0x3f)
 403   void lshr(Register hi, Register lo, bool sign_extension = false);  // hi:lo >> (rcx & 0x3f)
 404 
 405   // Long compare for Java
 406   // (semantics as described in JVM spec.)
 407   void lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo); // x_hi = lcmp(x, y)
 408 
 409 
 410   // misc
 411 
 412   // Sign extension
 413   void sign_extend_short(Register reg);
 414   void sign_extend_byte(Register reg);
 415 
 416   // Division by power of 2, rounding towards 0
 417   void division_with_shift(Register reg, int shift_value);
 418 
 419 #ifndef _LP64
 420   // Compares the top-most stack entries on the FPU stack and sets the eflags as follows:
 421   //
 422   // CF (corresponds to C0) if x < y
 423   // PF (corresponds to C2) if unordered
 424   // ZF (corresponds to C3) if x = y
 425   //
 426   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 427   // tmp is a temporary register, if none is available use noreg (only matters for non-P6 code)
 428   void fcmp(Register tmp);
 429   // Variant of the above which allows y to be further down the stack
 430   // and which only pops x and y if specified. If pop_right is
 431   // specified then pop_left must also be specified.
 432   void fcmp(Register tmp, int index, bool pop_left, bool pop_right);
 433 
 434   // Floating-point comparison for Java
 435   // Compares the top-most stack entries on the FPU stack and stores the result in dst.
 436   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 437   // (semantics as described in JVM spec.)
 438   void fcmp2int(Register dst, bool unordered_is_less);
 439   // Variant of the above which allows y to be further down the stack
 440   // and which only pops x and y if specified. If pop_right is
 441   // specified then pop_left must also be specified.
 442   void fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right);
 443 
 444   // Floating-point remainder for Java (ST0 = ST0 fremr ST1, ST1 is empty afterwards)
 445   // tmp is a temporary register, if none is available use noreg
 446   void fremr(Register tmp);
 447 
 448   // only if +VerifyFPU
 449   void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
 450 #endif // !LP64
 451 
 452   // dst = c = a * b + c
 453   void fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 454   void fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 455 
 456   void vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 457   void vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 458   void vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 459   void vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 460 
 461 
 462   // same as fcmp2int, but using SSE2
 463   void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 464   void cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 465 
 466   // branch to L if FPU flag C2 is set/not set
 467   // tmp is a temporary register, if none is available use noreg
 468   void jC2 (Register tmp, Label& L);
 469   void jnC2(Register tmp, Label& L);
 470 
 471   // Load float value from 'address'. If UseSSE >= 1, the value is loaded into
 472   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 473   void load_float(Address src);
 474 
 475   // Store float value to 'address'. If UseSSE >= 1, the value is stored
 476   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 477   void store_float(Address dst);
 478 
 479   // Load double value from 'address'. If UseSSE >= 2, the value is loaded into
 480   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 481   void load_double(Address src);
 482 
 483   // Store double value to 'address'. If UseSSE >= 2, the value is stored
 484   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 485   void store_double(Address dst);
 486 
 487 #ifndef _LP64
 488   // Pop ST (ffree & fincstp combined)
 489   void fpop();
 490 
 491   void empty_FPU_stack();
 492 #endif // !_LP64
 493 
 494   void push_IU_state();
 495   void pop_IU_state();
 496 
 497   void push_FPU_state();
 498   void pop_FPU_state();
 499 
 500   void push_CPU_state();
 501   void pop_CPU_state();
 502 
 503   // Round up to a power of two
 504   void round_to(Register reg, int modulus);
 505 
 506   // Callee saved registers handling
 507   void push_callee_saved_registers();
 508   void pop_callee_saved_registers();
 509 
 510   // allocation
 511   void eden_allocate(
 512     Register thread,                   // Current thread
 513     Register obj,                      // result: pointer to object after successful allocation
 514     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
 515     int      con_size_in_bytes,        // object size in bytes if   known at compile time
 516     Register t1,                       // temp register
 517     Label&   slow_case                 // continuation point if fast allocation fails
 518   );
 519   void tlab_allocate(
 520     Register thread,                   // Current thread
 521     Register obj,                      // result: pointer to object after successful allocation
 522     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
 523     int      con_size_in_bytes,        // object size in bytes if   known at compile time
 524     Register t1,                       // temp register
 525     Register t2,                       // temp register
 526     Label&   slow_case                 // continuation point if fast allocation fails
 527   );
 528   void zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp);
 529 
 530   // interface method calling
 531   void lookup_interface_method(Register recv_klass,
 532                                Register intf_klass,
 533                                RegisterOrConstant itable_index,
 534                                Register method_result,
 535                                Register scan_temp,
 536                                Label& no_such_interface,
 537                                bool return_method = true);
 538 
 539   // virtual method calling
 540   void lookup_virtual_method(Register recv_klass,
 541                              RegisterOrConstant vtable_index,
 542                              Register method_result);
 543 
 544   // Test sub_klass against super_klass, with fast and slow paths.
 545 
 546   // The fast path produces a tri-state answer: yes / no / maybe-slow.
 547   // One of the three labels can be NULL, meaning take the fall-through.
 548   // If super_check_offset is -1, the value is loaded up from super_klass.
 549   // No registers are killed, except temp_reg.
 550   void check_klass_subtype_fast_path(Register sub_klass,
 551                                      Register super_klass,
 552                                      Register temp_reg,
 553                                      Label* L_success,
 554                                      Label* L_failure,
 555                                      Label* L_slow_path,
 556                 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
 557 
 558   // The rest of the type check; must be wired to a corresponding fast path.
 559   // It does not repeat the fast path logic, so don't use it standalone.
 560   // The temp_reg and temp2_reg can be noreg, if no temps are available.
 561   // Updates the sub's secondary super cache as necessary.
 562   // If set_cond_codes, condition codes will be Z on success, NZ on failure.
 563   void check_klass_subtype_slow_path(Register sub_klass,
 564                                      Register super_klass,
 565                                      Register temp_reg,
 566                                      Register temp2_reg,
 567                                      Label* L_success,
 568                                      Label* L_failure,
 569                                      bool set_cond_codes = false);
 570 
 571   // Simplified, combined version, good for typical uses.
 572   // Falls through on failure.
 573   void check_klass_subtype(Register sub_klass,
 574                            Register super_klass,
 575                            Register temp_reg,
 576                            Label& L_success);
 577 
 578   void clinit_barrier(Register klass,
 579                       Register thread,
 580                       Label* L_fast_path = NULL,
 581                       Label* L_slow_path = NULL);
 582 
 583   // method handles (JSR 292)
 584   Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
 585 
 586   //----
 587   void set_word_if_not_zero(Register reg); // sets reg to 1 if not zero, otherwise 0
 588 
 589   // Debugging
 590 
 591   // only if +VerifyOops
 592   void _verify_oop(Register reg, const char* s, const char* file, int line);
 593   void _verify_oop_addr(Address addr, const char* s, const char* file, int line);
 594 
 595   // TODO: verify method and klass metadata (compare against vptr?)
 596   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
 597   void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
 598 
 599 #define verify_oop(reg) _verify_oop(reg, "broken oop " #reg, __FILE__, __LINE__)
 600 #define verify_oop_msg(reg, msg) _verify_oop(reg, "broken oop " #reg ", " #msg, __FILE__, __LINE__)
 601 #define verify_oop_addr(addr) _verify_oop_addr(addr, "broken oop addr " #addr, __FILE__, __LINE__)
 602 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
 603 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
 604 
 605   // Verify or restore cpu control state after JNI call
 606   void restore_cpu_control_state_after_jni();
 607 
 608   // prints msg, dumps registers and stops execution
 609   void stop(const char* msg);
 610 
 611   // prints msg and continues
 612   void warn(const char* msg);
 613 
 614   // dumps registers and other state
 615   void print_state();
 616 
 617   static void debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg);
 618   static void debug64(char* msg, int64_t pc, int64_t regs[]);
 619   static void print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip);
 620   static void print_state64(int64_t pc, int64_t regs[]);
 621 
 622   void os_breakpoint();
 623 
 624   void untested()                                { stop("untested"); }
 625 
 626   void unimplemented(const char* what = "");
 627 
 628   void should_not_reach_here()                   { stop("should not reach here"); }
 629 
 630   void print_CPU_state();
 631 
 632   // Stack overflow checking
 633   void bang_stack_with_offset(int offset) {
 634     // stack grows down, caller passes positive offset
 635     assert(offset > 0, "must bang with negative offset");
 636     movl(Address(rsp, (-offset)), rax);
 637   }
 638 
 639   // Writes to stack successive pages until offset reached to check for
 640   // stack overflow + shadow pages.  Also, clobbers tmp
 641   void bang_stack_size(Register size, Register tmp);
 642 
 643   // Check for reserved stack access in method being exited (for JIT)
 644   void reserved_stack_check();
 645 
 646   virtual RegisterOrConstant delayed_value_impl(intptr_t* delayed_value_addr,
 647                                                 Register tmp,
 648                                                 int offset);
 649 
 650   // If thread_reg is != noreg the code assumes the register passed contains
 651   // the thread (required on 64 bit).
 652   void safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg);
 653 
 654   void verify_tlab();
 655 
 656   // Biased locking support
 657   // lock_reg and obj_reg must be loaded up with the appropriate values.
 658   // swap_reg must be rax, and is killed.
 659   // tmp_reg is optional. If it is supplied (i.e., != noreg) it will
 660   // be killed; if not supplied, push/pop will be used internally to
 661   // allocate a temporary (inefficient, avoid if possible).
 662   // Optional slow case is for implementations (interpreter and C1) which branch to
 663   // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
 664   // Returns offset of first potentially-faulting instruction for null
 665   // check info (currently consumed only by C1). If
 666   // swap_reg_contains_mark is true then returns -1 as it is assumed
 667   // the calling code has already passed any potential faults.
 668   int biased_locking_enter(Register lock_reg, Register obj_reg,
 669                            Register swap_reg, Register tmp_reg,
 670                            Register tmp_reg2, bool swap_reg_contains_mark,
 671                            Label& done, Label* slow_case = NULL,
 672                            BiasedLockingCounters* counters = NULL);
 673   void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
 674 
 675   Condition negate_condition(Condition cond);
 676 
 677   // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
 678   // operands. In general the names are modified to avoid hiding the instruction in Assembler
 679   // so that we don't need to implement all the varieties in the Assembler with trivial wrappers
 680   // here in MacroAssembler. The major exception to this rule is call
 681 
 682   // Arithmetics
 683 
 684 
 685   void addptr(Address dst, int32_t src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)) ; }
 686   void addptr(Address dst, Register src);
 687 
 688   void addptr(Register dst, Address src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); }
 689   void addptr(Register dst, int32_t src);
 690   void addptr(Register dst, Register src);
 691   void addptr(Register dst, RegisterOrConstant src) {
 692     if (src.is_constant()) addptr(dst, (int) src.as_constant());
 693     else                   addptr(dst,       src.as_register());
 694   }
 695 
 696   void andptr(Register dst, int32_t src);
 697   void andptr(Register src1, Register src2) { LP64_ONLY(andq(src1, src2)) NOT_LP64(andl(src1, src2)) ; }
 698 
 699   void cmp8(AddressLiteral src1, int imm);
 700 
 701   // renamed to drag out the casting of address to int32_t/intptr_t
 702   void cmp32(Register src1, int32_t imm);
 703 
 704   void cmp32(AddressLiteral src1, int32_t imm);
 705   // compare reg - mem, or reg - &mem
 706   void cmp32(Register src1, AddressLiteral src2);
 707 
 708   void cmp32(Register src1, Address src2);
 709 
 710 #ifndef _LP64
 711   void cmpklass(Address dst, Metadata* obj);
 712   void cmpklass(Register dst, Metadata* obj);
 713   void cmpoop(Address dst, jobject obj);
 714   void cmpoop_raw(Address dst, jobject obj);
 715 #endif // _LP64
 716 
 717   void cmpoop(Register src1, Register src2);
 718   void cmpoop(Register src1, Address src2);
 719   void cmpoop(Register dst, jobject obj);
 720   void cmpoop_raw(Register dst, jobject obj);
 721 
 722   // NOTE src2 must be the lval. This is NOT an mem-mem compare
 723   void cmpptr(Address src1, AddressLiteral src2);
 724 
 725   void cmpptr(Register src1, AddressLiteral src2);
 726 
 727   void cmpptr(Register src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 728   void cmpptr(Register src1, Address src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 729   // void cmpptr(Address src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 730 
 731   void cmpptr(Register src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 732   void cmpptr(Address src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 733 
 734   // cmp64 to avoild hiding cmpq
 735   void cmp64(Register src1, AddressLiteral src);
 736 
 737   void cmpxchgptr(Register reg, Address adr);
 738 
 739   void locked_cmpxchgptr(Register reg, AddressLiteral adr);
 740 
 741 
 742   void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
 743   void imulptr(Register dst, Register src, int imm32) { LP64_ONLY(imulq(dst, src, imm32)) NOT_LP64(imull(dst, src, imm32)); }
 744 
 745 
 746   void negptr(Register dst) { LP64_ONLY(negq(dst)) NOT_LP64(negl(dst)); }
 747 
 748   void notptr(Register dst) { LP64_ONLY(notq(dst)) NOT_LP64(notl(dst)); }
 749 
 750   void shlptr(Register dst, int32_t shift);
 751   void shlptr(Register dst) { LP64_ONLY(shlq(dst)) NOT_LP64(shll(dst)); }
 752 
 753   void shrptr(Register dst, int32_t shift);
 754   void shrptr(Register dst) { LP64_ONLY(shrq(dst)) NOT_LP64(shrl(dst)); }
 755 
 756   void sarptr(Register dst) { LP64_ONLY(sarq(dst)) NOT_LP64(sarl(dst)); }
 757   void sarptr(Register dst, int32_t src) { LP64_ONLY(sarq(dst, src)) NOT_LP64(sarl(dst, src)); }
 758 
 759   void subptr(Address dst, int32_t src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 760 
 761   void subptr(Register dst, Address src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 762   void subptr(Register dst, int32_t src);
 763   // Force generation of a 4 byte immediate value even if it fits into 8bit
 764   void subptr_imm32(Register dst, int32_t src);
 765   void subptr(Register dst, Register src);
 766   void subptr(Register dst, RegisterOrConstant src) {
 767     if (src.is_constant()) subptr(dst, (int) src.as_constant());
 768     else                   subptr(dst,       src.as_register());
 769   }
 770 
 771   void sbbptr(Address dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 772   void sbbptr(Register dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 773 
 774   void xchgptr(Register src1, Register src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 775   void xchgptr(Register src1, Address src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 776 
 777   void xaddptr(Address src1, Register src2) { LP64_ONLY(xaddq(src1, src2)) NOT_LP64(xaddl(src1, src2)) ; }
 778 
 779 
 780 
 781   // Helper functions for statistics gathering.
 782   // Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
 783   void cond_inc32(Condition cond, AddressLiteral counter_addr);
 784   // Unconditional atomic increment.
 785   void atomic_incl(Address counter_addr);
 786   void atomic_incl(AddressLiteral counter_addr, Register scr = rscratch1);
 787 #ifdef _LP64
 788   void atomic_incq(Address counter_addr);
 789   void atomic_incq(AddressLiteral counter_addr, Register scr = rscratch1);
 790 #endif
 791   void atomic_incptr(AddressLiteral counter_addr, Register scr = rscratch1) { LP64_ONLY(atomic_incq(counter_addr, scr)) NOT_LP64(atomic_incl(counter_addr, scr)) ; }
 792   void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; }
 793 
 794   void lea(Register dst, AddressLiteral adr);
 795   void lea(Address dst, AddressLiteral adr);
 796   void lea(Register dst, Address adr) { Assembler::lea(dst, adr); }
 797 
 798   void leal32(Register dst, Address src) { leal(dst, src); }
 799 
 800   // Import other testl() methods from the parent class or else
 801   // they will be hidden by the following overriding declaration.
 802   using Assembler::testl;
 803   void testl(Register dst, AddressLiteral src);
 804 
 805   void orptr(Register dst, Address src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 806   void orptr(Register dst, Register src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 807   void orptr(Register dst, int32_t src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 808   void orptr(Address dst, int32_t imm32) { LP64_ONLY(orq(dst, imm32)) NOT_LP64(orl(dst, imm32)); }
 809 
 810   void testptr(Register src, int32_t imm32) {  LP64_ONLY(testq(src, imm32)) NOT_LP64(testl(src, imm32)); }
 811   void testptr(Register src1, Address src2) { LP64_ONLY(testq(src1, src2)) NOT_LP64(testl(src1, src2)); }
 812   void testptr(Register src1, Register src2);
 813 
 814   void xorptr(Register dst, Register src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 815   void xorptr(Register dst, Address src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 816 
 817   // Calls
 818 
 819   void call(Label& L, relocInfo::relocType rtype);
 820   void call(Register entry);
 821 
 822   // NOTE: this call transfers to the effective address of entry NOT
 823   // the address contained by entry. This is because this is more natural
 824   // for jumps/calls.
 825   void call(AddressLiteral entry);
 826 
 827   // Emit the CompiledIC call idiom
 828   void ic_call(address entry, jint method_index = 0);
 829 
 830   // Jumps
 831 
 832   // NOTE: these jumps tranfer to the effective address of dst NOT
 833   // the address contained by dst. This is because this is more natural
 834   // for jumps/calls.
 835   void jump(AddressLiteral dst);
 836   void jump_cc(Condition cc, AddressLiteral dst);
 837 
 838   // 32bit can do a case table jump in one instruction but we no longer allow the base
 839   // to be installed in the Address class. This jump will tranfers to the address
 840   // contained in the location described by entry (not the address of entry)
 841   void jump(ArrayAddress entry);
 842 
 843   // Floating
 844 
 845   void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); }
 846   void andpd(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
 847   void andpd(XMMRegister dst, XMMRegister src) { Assembler::andpd(dst, src); }
 848 
 849   void andps(XMMRegister dst, XMMRegister src) { Assembler::andps(dst, src); }
 850   void andps(XMMRegister dst, Address src) { Assembler::andps(dst, src); }
 851   void andps(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
 852 
 853   void comiss(XMMRegister dst, XMMRegister src) { Assembler::comiss(dst, src); }
 854   void comiss(XMMRegister dst, Address src) { Assembler::comiss(dst, src); }
 855   void comiss(XMMRegister dst, AddressLiteral src);
 856 
 857   void comisd(XMMRegister dst, XMMRegister src) { Assembler::comisd(dst, src); }
 858   void comisd(XMMRegister dst, Address src) { Assembler::comisd(dst, src); }
 859   void comisd(XMMRegister dst, AddressLiteral src);
 860 
 861 #ifndef _LP64
 862   void fadd_s(Address src)        { Assembler::fadd_s(src); }
 863   void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
 864 
 865   void fldcw(Address src) { Assembler::fldcw(src); }
 866   void fldcw(AddressLiteral src);
 867 
 868   void fld_s(int index)   { Assembler::fld_s(index); }
 869   void fld_s(Address src) { Assembler::fld_s(src); }
 870   void fld_s(AddressLiteral src);
 871 
 872   void fld_d(Address src) { Assembler::fld_d(src); }
 873   void fld_d(AddressLiteral src);
 874 
 875   void fld_x(Address src) { Assembler::fld_x(src); }
 876   void fld_x(AddressLiteral src);
 877 
 878   void fmul_s(Address src)        { Assembler::fmul_s(src); }
 879   void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
 880 #endif // _LP64
 881 
 882   void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
 883   void ldmxcsr(AddressLiteral src);
 884 
 885 #ifdef _LP64
 886  private:
 887   void sha256_AVX2_one_round_compute(
 888     Register  reg_old_h,
 889     Register  reg_a,
 890     Register  reg_b,
 891     Register  reg_c,
 892     Register  reg_d,
 893     Register  reg_e,
 894     Register  reg_f,
 895     Register  reg_g,
 896     Register  reg_h,
 897     int iter);
 898   void sha256_AVX2_four_rounds_compute_first(int start);
 899   void sha256_AVX2_four_rounds_compute_last(int start);
 900   void sha256_AVX2_one_round_and_sched(
 901         XMMRegister xmm_0,     /* == ymm4 on 0, 1, 2, 3 iterations, then rotate 4 registers left on 4, 8, 12 iterations */
 902         XMMRegister xmm_1,     /* ymm5 */  /* full cycle is 16 iterations */
 903         XMMRegister xmm_2,     /* ymm6 */
 904         XMMRegister xmm_3,     /* ymm7 */
 905         Register    reg_a,      /* == eax on 0 iteration, then rotate 8 register right on each next iteration */
 906         Register    reg_b,      /* ebx */    /* full cycle is 8 iterations */
 907         Register    reg_c,      /* edi */
 908         Register    reg_d,      /* esi */
 909         Register    reg_e,      /* r8d */
 910         Register    reg_f,      /* r9d */
 911         Register    reg_g,      /* r10d */
 912         Register    reg_h,      /* r11d */
 913         int iter);
 914 
 915   void addm(int disp, Register r1, Register r2);
 916   void gfmul(XMMRegister tmp0, XMMRegister t);
 917   void schoolbookAAD(int i, Register subkeyH, XMMRegister data, XMMRegister tmp0,
 918                      XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3);
 919   void generateHtbl_one_block(Register htbl);
 920   void generateHtbl_eight_blocks(Register htbl);
 921  public:
 922   void sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 923                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 924                    Register buf, Register state, Register ofs, Register limit, Register rsp,
 925                    bool multi_block, XMMRegister shuf_mask);
 926   void avx_ghash(Register state, Register htbl, Register data, Register blocks);
 927 #endif
 928 
 929 #ifdef _LP64
 930  private:
 931   void sha512_AVX2_one_round_compute(Register old_h, Register a, Register b, Register c, Register d,
 932                                      Register e, Register f, Register g, Register h, int iteration);
 933 
 934   void sha512_AVX2_one_round_and_schedule(XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
 935                                           Register a, Register b, Register c, Register d, Register e, Register f,
 936                                           Register g, Register h, int iteration);
 937 
 938   void addmq(int disp, Register r1, Register r2);
 939  public:
 940   void sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 941                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 942                    Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block,
 943                    XMMRegister shuf_mask);
 944 private:
 945   void roundEnc(XMMRegister key, int rnum);
 946   void lastroundEnc(XMMRegister key, int rnum);
 947   void roundDec(XMMRegister key, int rnum);
 948   void lastroundDec(XMMRegister key, int rnum);
 949   void ev_load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask);
 950 
 951 public:
 952   void aesecb_encrypt(Register source_addr, Register dest_addr, Register key, Register len);
 953   void aesecb_decrypt(Register source_addr, Register dest_addr, Register key, Register len);
 954   void aesctr_encrypt(Register src_addr, Register dest_addr, Register key, Register counter,
 955                       Register len_reg, Register used, Register used_addr, Register saved_encCounter_start);
 956 
 957 #endif
 958 
 959   void fast_sha1(XMMRegister abcd, XMMRegister e0, XMMRegister e1, XMMRegister msg0,
 960                  XMMRegister msg1, XMMRegister msg2, XMMRegister msg3, XMMRegister shuf_mask,
 961                  Register buf, Register state, Register ofs, Register limit, Register rsp,
 962                  bool multi_block);
 963 
 964 #ifdef _LP64
 965   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 966                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 967                    Register buf, Register state, Register ofs, Register limit, Register rsp,
 968                    bool multi_block, XMMRegister shuf_mask);
 969 #else
 970   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 971                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 972                    Register buf, Register state, Register ofs, Register limit, Register rsp,
 973                    bool multi_block);
 974 #endif
 975 
 976   void fast_exp(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
 977                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
 978                 Register rax, Register rcx, Register rdx, Register tmp);
 979 
 980 #ifdef _LP64
 981   void fast_log(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
 982                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
 983                 Register rax, Register rcx, Register rdx, Register tmp1, Register tmp2);
 984 
 985   void fast_log10(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
 986                   XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
 987                   Register rax, Register rcx, Register rdx, Register r11);
 988 
 989   void fast_pow(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
 990                 XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7, Register rax, Register rcx,
 991                 Register rdx, Register tmp1, Register tmp2, Register tmp3, Register tmp4);
 992 
 993   void fast_sin(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
 994                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
 995                 Register rax, Register rbx, Register rcx, Register rdx, Register tmp1, Register tmp2,
 996                 Register tmp3, Register tmp4);
 997 
 998   void fast_cos(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
 999                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1000                 Register rax, Register rcx, Register rdx, Register tmp1,
1001                 Register tmp2, Register tmp3, Register tmp4);
1002   void fast_tan(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1003                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1004                 Register rax, Register rcx, Register rdx, Register tmp1,
1005                 Register tmp2, Register tmp3, Register tmp4);
1006 #else
1007   void fast_log(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1008                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1009                 Register rax, Register rcx, Register rdx, Register tmp1);
1010 
1011   void fast_log10(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1012                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1013                 Register rax, Register rcx, Register rdx, Register tmp);
1014 
1015   void fast_pow(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
1016                 XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7, Register rax, Register rcx,
1017                 Register rdx, Register tmp);
1018 
1019   void fast_sin(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1020                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1021                 Register rax, Register rbx, Register rdx);
1022 
1023   void fast_cos(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1024                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1025                 Register rax, Register rcx, Register rdx, Register tmp);
1026 
1027   void libm_sincos_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1028                         Register edx, Register ebx, Register esi, Register edi,
1029                         Register ebp, Register esp);
1030 
1031   void libm_reduce_pi04l(Register eax, Register ecx, Register edx, Register ebx,
1032                          Register esi, Register edi, Register ebp, Register esp);
1033 
1034   void libm_tancot_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1035                         Register edx, Register ebx, Register esi, Register edi,
1036                         Register ebp, Register esp);
1037 
1038   void fast_tan(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1039                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1040                 Register rax, Register rcx, Register rdx, Register tmp);
1041 #endif
1042 
1043 private:
1044 
1045   // these are private because users should be doing movflt/movdbl
1046 
1047   void movss(XMMRegister dst, XMMRegister src) { Assembler::movss(dst, src); }
1048   void movss(Address dst, XMMRegister src)     { Assembler::movss(dst, src); }
1049   void movss(XMMRegister dst, Address src)     { Assembler::movss(dst, src); }
1050   void movss(XMMRegister dst, AddressLiteral src);
1051 
1052   void movlpd(XMMRegister dst, Address src)    {Assembler::movlpd(dst, src); }
1053   void movlpd(XMMRegister dst, AddressLiteral src);
1054 
1055 public:
1056 
1057   void addsd(XMMRegister dst, XMMRegister src)    { Assembler::addsd(dst, src); }
1058   void addsd(XMMRegister dst, Address src)        { Assembler::addsd(dst, src); }
1059   void addsd(XMMRegister dst, AddressLiteral src);
1060 
1061   void addss(XMMRegister dst, XMMRegister src)    { Assembler::addss(dst, src); }
1062   void addss(XMMRegister dst, Address src)        { Assembler::addss(dst, src); }
1063   void addss(XMMRegister dst, AddressLiteral src);
1064 
1065   void addpd(XMMRegister dst, XMMRegister src)    { Assembler::addpd(dst, src); }
1066   void addpd(XMMRegister dst, Address src)        { Assembler::addpd(dst, src); }
1067   void addpd(XMMRegister dst, AddressLiteral src);
1068 
1069   void divsd(XMMRegister dst, XMMRegister src)    { Assembler::divsd(dst, src); }
1070   void divsd(XMMRegister dst, Address src)        { Assembler::divsd(dst, src); }
1071   void divsd(XMMRegister dst, AddressLiteral src);
1072 
1073   void divss(XMMRegister dst, XMMRegister src)    { Assembler::divss(dst, src); }
1074   void divss(XMMRegister dst, Address src)        { Assembler::divss(dst, src); }
1075   void divss(XMMRegister dst, AddressLiteral src);
1076 
1077   // Move Unaligned Double Quadword
1078   void movdqu(Address     dst, XMMRegister src);
1079   void movdqu(XMMRegister dst, Address src);
1080   void movdqu(XMMRegister dst, XMMRegister src);
1081   void movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg = rscratch1);
1082 
1083   void kmovwl(KRegister dst, Register src) { Assembler::kmovwl(dst, src); }
1084   void kmovwl(Register dst, KRegister src) { Assembler::kmovwl(dst, src); }
1085   void kmovwl(KRegister dst, Address src) { Assembler::kmovwl(dst, src); }
1086   void kmovwl(KRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
1087 
1088   // AVX Unaligned forms
1089   void vmovdqu(Address     dst, XMMRegister src);
1090   void vmovdqu(XMMRegister dst, Address src);
1091   void vmovdqu(XMMRegister dst, XMMRegister src);
1092   void vmovdqu(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
1093 
1094   // AVX512 Unaligned
1095   void evmovdqub(Address dst, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, src, merge, vector_len); }
1096   void evmovdqub(XMMRegister dst, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, src, merge, vector_len); }
1097   void evmovdqub(XMMRegister dst, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqub(dst, src, merge, vector_len); }
1098   void evmovdqub(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqub(dst, mask, src, merge, vector_len); }
1099   void evmovdqub(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
1100 
1101   void evmovdquw(Address dst, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquw(dst, src, merge, vector_len); }
1102   void evmovdquw(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1103   void evmovdquw(XMMRegister dst, Address src, bool merge, int vector_len) { Assembler::evmovdquw(dst, src, merge, vector_len); }
1104   void evmovdquw(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquw(dst, mask, src, merge, vector_len); }
1105   void evmovdquw(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
1106 
1107   void evmovdqul(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1108   void evmovdqul(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdqul(dst, src, vector_len); }
1109   void evmovdqul(XMMRegister dst, XMMRegister src, int vector_len) {
1110      if (dst->encoding() == src->encoding()) return;
1111      Assembler::evmovdqul(dst, src, vector_len);
1112   }
1113   void evmovdqul(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1114   void evmovdqul(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdqul(dst, mask, src, merge, vector_len); }
1115   void evmovdqul(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1116     if (dst->encoding() == src->encoding() && mask == k0) return;
1117     Assembler::evmovdqul(dst, mask, src, merge, vector_len);
1118    }
1119   void evmovdqul(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
1120 
1121   void evmovdquq(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1122   void evmovdquq(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1123   void evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch);
1124   void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) {
1125     if (dst->encoding() == src->encoding()) return;
1126     Assembler::evmovdquq(dst, src, vector_len);
1127   }
1128   void evmovdquq(Address dst, KRegister mask, XMMRegister src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1129   void evmovdquq(XMMRegister dst, KRegister mask, Address src, bool merge, int vector_len) { Assembler::evmovdquq(dst, mask, src, merge, vector_len); }
1130   void evmovdquq(XMMRegister dst, KRegister mask, XMMRegister src, bool merge, int vector_len) {
1131     if (dst->encoding() == src->encoding() && mask == k0) return;
1132     Assembler::evmovdquq(dst, mask, src, merge, vector_len);
1133   }
1134   void evmovdquq(XMMRegister dst, KRegister mask, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
1135 
1136   // Move Aligned Double Quadword
1137   void movdqa(XMMRegister dst, Address src)       { Assembler::movdqa(dst, src); }
1138   void movdqa(XMMRegister dst, XMMRegister src)   { Assembler::movdqa(dst, src); }
1139   void movdqa(XMMRegister dst, AddressLiteral src);
1140 
1141   void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); }
1142   void movsd(Address dst, XMMRegister src)     { Assembler::movsd(dst, src); }
1143   void movsd(XMMRegister dst, Address src)     { Assembler::movsd(dst, src); }
1144   void movsd(XMMRegister dst, AddressLiteral src);
1145 
1146   void mulpd(XMMRegister dst, XMMRegister src)    { Assembler::mulpd(dst, src); }
1147   void mulpd(XMMRegister dst, Address src)        { Assembler::mulpd(dst, src); }
1148   void mulpd(XMMRegister dst, AddressLiteral src);
1149 
1150   void mulsd(XMMRegister dst, XMMRegister src)    { Assembler::mulsd(dst, src); }
1151   void mulsd(XMMRegister dst, Address src)        { Assembler::mulsd(dst, src); }
1152   void mulsd(XMMRegister dst, AddressLiteral src);
1153 
1154   void mulss(XMMRegister dst, XMMRegister src)    { Assembler::mulss(dst, src); }
1155   void mulss(XMMRegister dst, Address src)        { Assembler::mulss(dst, src); }
1156   void mulss(XMMRegister dst, AddressLiteral src);
1157 
1158   // Carry-Less Multiplication Quadword
1159   void pclmulldq(XMMRegister dst, XMMRegister src) {
1160     // 0x00 - multiply lower 64 bits [0:63]
1161     Assembler::pclmulqdq(dst, src, 0x00);
1162   }
1163   void pclmulhdq(XMMRegister dst, XMMRegister src) {
1164     // 0x11 - multiply upper 64 bits [64:127]
1165     Assembler::pclmulqdq(dst, src, 0x11);
1166   }
1167 
1168   void pcmpeqb(XMMRegister dst, XMMRegister src);
1169   void pcmpeqw(XMMRegister dst, XMMRegister src);
1170 
1171   void pcmpestri(XMMRegister dst, Address src, int imm8);
1172   void pcmpestri(XMMRegister dst, XMMRegister src, int imm8);
1173 
1174   void pmovzxbw(XMMRegister dst, XMMRegister src);
1175   void pmovzxbw(XMMRegister dst, Address src);
1176 
1177   void pmovmskb(Register dst, XMMRegister src);
1178 
1179   void ptest(XMMRegister dst, XMMRegister src);
1180 
1181   void sqrtsd(XMMRegister dst, XMMRegister src)    { Assembler::sqrtsd(dst, src); }
1182   void sqrtsd(XMMRegister dst, Address src)        { Assembler::sqrtsd(dst, src); }
1183   void sqrtsd(XMMRegister dst, AddressLiteral src);
1184 
1185   void roundsd(XMMRegister dst, XMMRegister src, int32_t rmode)    { Assembler::roundsd(dst, src, rmode); }
1186   void roundsd(XMMRegister dst, Address src, int32_t rmode)        { Assembler::roundsd(dst, src, rmode); }
1187   void roundsd(XMMRegister dst, AddressLiteral src, int32_t rmode, Register scratch_reg);
1188 
1189   void sqrtss(XMMRegister dst, XMMRegister src)    { Assembler::sqrtss(dst, src); }
1190   void sqrtss(XMMRegister dst, Address src)        { Assembler::sqrtss(dst, src); }
1191   void sqrtss(XMMRegister dst, AddressLiteral src);
1192 
1193   void subsd(XMMRegister dst, XMMRegister src)    { Assembler::subsd(dst, src); }
1194   void subsd(XMMRegister dst, Address src)        { Assembler::subsd(dst, src); }
1195   void subsd(XMMRegister dst, AddressLiteral src);
1196 
1197   void subss(XMMRegister dst, XMMRegister src)    { Assembler::subss(dst, src); }
1198   void subss(XMMRegister dst, Address src)        { Assembler::subss(dst, src); }
1199   void subss(XMMRegister dst, AddressLiteral src);
1200 
1201   void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
1202   void ucomiss(XMMRegister dst, Address src)     { Assembler::ucomiss(dst, src); }
1203   void ucomiss(XMMRegister dst, AddressLiteral src);
1204 
1205   void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
1206   void ucomisd(XMMRegister dst, Address src)     { Assembler::ucomisd(dst, src); }
1207   void ucomisd(XMMRegister dst, AddressLiteral src);
1208 
1209   // Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
1210   void xorpd(XMMRegister dst, XMMRegister src);
1211   void xorpd(XMMRegister dst, Address src)     { Assembler::xorpd(dst, src); }
1212   void xorpd(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
1213 
1214   // Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
1215   void xorps(XMMRegister dst, XMMRegister src);
1216   void xorps(XMMRegister dst, Address src)     { Assembler::xorps(dst, src); }
1217   void xorps(XMMRegister dst, AddressLiteral src, Register scratch_reg = rscratch1);
1218 
1219   // Shuffle Bytes
1220   void pshufb(XMMRegister dst, XMMRegister src) { Assembler::pshufb(dst, src); }
1221   void pshufb(XMMRegister dst, Address src)     { Assembler::pshufb(dst, src); }
1222   void pshufb(XMMRegister dst, AddressLiteral src);
1223   // AVX 3-operands instructions
1224 
1225   void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddsd(dst, nds, src); }
1226   void vaddsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vaddsd(dst, nds, src); }
1227   void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1228 
1229   void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddss(dst, nds, src); }
1230   void vaddss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vaddss(dst, nds, src); }
1231   void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1232 
1233   void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len);
1234   void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len);
1235 
1236   void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1237   void vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1238 
1239   void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1240   void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1241 
1242   void vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1243   void vpaddd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpaddd(dst, nds, src, vector_len); }
1244   void vpaddd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register rscratch);
1245 
1246   void vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1247   void vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1248   void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
1249 
1250   void vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len);
1251   void vpbroadcastw(XMMRegister dst, Address src, int vector_len) { Assembler::vpbroadcastw(dst, src, vector_len); }
1252 
1253   void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1254 
1255   void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1256   void evpcmpeqd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg);
1257 
1258   // Vector compares
1259   void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
1260                int comparison, int vector_len) { Assembler::evpcmpd(kdst, mask, nds, src, comparison, vector_len); }
1261   void evpcmpd(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
1262                int comparison, int vector_len, Register scratch_reg);
1263   void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
1264                int comparison, int vector_len) { Assembler::evpcmpq(kdst, mask, nds, src, comparison, vector_len); }
1265   void evpcmpq(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
1266                int comparison, int vector_len, Register scratch_reg);
1267   void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
1268                int comparison, int vector_len) { Assembler::evpcmpb(kdst, mask, nds, src, comparison, vector_len); }
1269   void evpcmpb(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
1270                int comparison, int vector_len, Register scratch_reg);
1271   void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, XMMRegister src,
1272                int comparison, int vector_len) { Assembler::evpcmpw(kdst, mask, nds, src, comparison, vector_len); }
1273   void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, AddressLiteral src,
1274                int comparison, int vector_len, Register scratch_reg);
1275 
1276 
1277   // Emit comparison instruction for the specified comparison predicate.
1278   void vpcmpCCW(XMMRegister dst, XMMRegister nds, XMMRegister src, ComparisonPredicate cond, Width width, int vector_len, Register scratch_reg);
1279   void vpcmpCC(XMMRegister dst, XMMRegister nds, XMMRegister src, int cond_encoding, Width width, int vector_len);
1280 
1281   void vpmovzxbw(XMMRegister dst, Address src, int vector_len);
1282   void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vpmovzxbw(dst, src, vector_len); }
1283 
1284   void vpmovmskb(Register dst, XMMRegister src);
1285 
1286   void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1287   void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1288 
1289   void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1290   void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1291 
1292   void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1293   void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1294 
1295   void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1296   void vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1297 
1298   void evpsraq(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1299   void evpsraq(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1300 
1301   void vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1302   void vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1303 
1304   void vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1305   void vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1306 
1307   void vptest(XMMRegister dst, XMMRegister src);
1308   void vptest(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vptest(dst, src, vector_len); }
1309 
1310   void punpcklbw(XMMRegister dst, XMMRegister src);
1311   void punpcklbw(XMMRegister dst, Address src) { Assembler::punpcklbw(dst, src); }
1312 
1313   void pshufd(XMMRegister dst, Address src, int mode);
1314   void pshufd(XMMRegister dst, XMMRegister src, int mode) { Assembler::pshufd(dst, src, mode); }
1315 
1316   void pshuflw(XMMRegister dst, XMMRegister src, int mode);
1317   void pshuflw(XMMRegister dst, Address src, int mode) { Assembler::pshuflw(dst, src, mode); }
1318 
1319   void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1320   void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len)     { Assembler::vandpd(dst, nds, src, vector_len); }
1321   void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
1322 
1323   void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1324   void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len)     { Assembler::vandps(dst, nds, src, vector_len); }
1325   void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
1326 
1327   void evpord(XMMRegister dst, KRegister mask, XMMRegister nds, AddressLiteral src, bool merge, int vector_len, Register scratch_reg);
1328 
1329   void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivsd(dst, nds, src); }
1330   void vdivsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vdivsd(dst, nds, src); }
1331   void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1332 
1333   void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivss(dst, nds, src); }
1334   void vdivss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vdivss(dst, nds, src); }
1335   void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1336 
1337   void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulsd(dst, nds, src); }
1338   void vmulsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vmulsd(dst, nds, src); }
1339   void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1340 
1341   void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulss(dst, nds, src); }
1342   void vmulss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vmulss(dst, nds, src); }
1343   void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1344 
1345   void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubsd(dst, nds, src); }
1346   void vsubsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vsubsd(dst, nds, src); }
1347   void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1348 
1349   void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubss(dst, nds, src); }
1350   void vsubss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vsubss(dst, nds, src); }
1351   void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1352 
1353   void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1354   void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1355 
1356   // AVX Vector instructions
1357 
1358   void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1359   void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1360   void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
1361 
1362   void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1363   void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1364   void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
1365 
1366   void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1367     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1368       Assembler::vpxor(dst, nds, src, vector_len);
1369     else
1370       Assembler::vxorpd(dst, nds, src, vector_len);
1371   }
1372   void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
1373     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1374       Assembler::vpxor(dst, nds, src, vector_len);
1375     else
1376       Assembler::vxorpd(dst, nds, src, vector_len);
1377   }
1378   void vpxor(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg = rscratch1);
1379 
1380   // Simple version for AVX2 256bit vectors
1381   void vpxor(XMMRegister dst, XMMRegister src) { Assembler::vpxor(dst, dst, src, true); }
1382   void vpxor(XMMRegister dst, Address src) { Assembler::vpxor(dst, dst, src, true); }
1383 
1384   void vpermd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpermd(dst, nds, src, vector_len); }
1385   void vpermd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len, Register scratch_reg);
1386 
1387   void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
1388     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1389       Assembler::vinserti32x4(dst, dst, src, imm8);
1390     } else if (UseAVX > 1) {
1391       // vinserti128 is available only in AVX2
1392       Assembler::vinserti128(dst, nds, src, imm8);
1393     } else {
1394       Assembler::vinsertf128(dst, nds, src, imm8);
1395     }
1396   }
1397 
1398   void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
1399     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1400       Assembler::vinserti32x4(dst, dst, src, imm8);
1401     } else if (UseAVX > 1) {
1402       // vinserti128 is available only in AVX2
1403       Assembler::vinserti128(dst, nds, src, imm8);
1404     } else {
1405       Assembler::vinsertf128(dst, nds, src, imm8);
1406     }
1407   }
1408 
1409   void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
1410     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1411       Assembler::vextracti32x4(dst, src, imm8);
1412     } else if (UseAVX > 1) {
1413       // vextracti128 is available only in AVX2
1414       Assembler::vextracti128(dst, src, imm8);
1415     } else {
1416       Assembler::vextractf128(dst, src, imm8);
1417     }
1418   }
1419 
1420   void vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
1421     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1422       Assembler::vextracti32x4(dst, src, imm8);
1423     } else if (UseAVX > 1) {
1424       // vextracti128 is available only in AVX2
1425       Assembler::vextracti128(dst, src, imm8);
1426     } else {
1427       Assembler::vextractf128(dst, src, imm8);
1428     }
1429   }
1430 
1431   // 128bit copy to/from high 128 bits of 256bit (YMM) vector registers
1432   void vinserti128_high(XMMRegister dst, XMMRegister src) {
1433     vinserti128(dst, dst, src, 1);
1434   }
1435   void vinserti128_high(XMMRegister dst, Address src) {
1436     vinserti128(dst, dst, src, 1);
1437   }
1438   void vextracti128_high(XMMRegister dst, XMMRegister src) {
1439     vextracti128(dst, src, 1);
1440   }
1441   void vextracti128_high(Address dst, XMMRegister src) {
1442     vextracti128(dst, src, 1);
1443   }
1444 
1445   void vinsertf128_high(XMMRegister dst, XMMRegister src) {
1446     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1447       Assembler::vinsertf32x4(dst, dst, src, 1);
1448     } else {
1449       Assembler::vinsertf128(dst, dst, src, 1);
1450     }
1451   }
1452 
1453   void vinsertf128_high(XMMRegister dst, Address src) {
1454     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1455       Assembler::vinsertf32x4(dst, dst, src, 1);
1456     } else {
1457       Assembler::vinsertf128(dst, dst, src, 1);
1458     }
1459   }
1460 
1461   void vextractf128_high(XMMRegister dst, XMMRegister src) {
1462     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1463       Assembler::vextractf32x4(dst, src, 1);
1464     } else {
1465       Assembler::vextractf128(dst, src, 1);
1466     }
1467   }
1468 
1469   void vextractf128_high(Address dst, XMMRegister src) {
1470     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1471       Assembler::vextractf32x4(dst, src, 1);
1472     } else {
1473       Assembler::vextractf128(dst, src, 1);
1474     }
1475   }
1476 
1477   // 256bit copy to/from high 256 bits of 512bit (ZMM) vector registers
1478   void vinserti64x4_high(XMMRegister dst, XMMRegister src) {
1479     Assembler::vinserti64x4(dst, dst, src, 1);
1480   }
1481   void vinsertf64x4_high(XMMRegister dst, XMMRegister src) {
1482     Assembler::vinsertf64x4(dst, dst, src, 1);
1483   }
1484   void vextracti64x4_high(XMMRegister dst, XMMRegister src) {
1485     Assembler::vextracti64x4(dst, src, 1);
1486   }
1487   void vextractf64x4_high(XMMRegister dst, XMMRegister src) {
1488     Assembler::vextractf64x4(dst, src, 1);
1489   }
1490   void vextractf64x4_high(Address dst, XMMRegister src) {
1491     Assembler::vextractf64x4(dst, src, 1);
1492   }
1493   void vinsertf64x4_high(XMMRegister dst, Address src) {
1494     Assembler::vinsertf64x4(dst, dst, src, 1);
1495   }
1496 
1497   // 128bit copy to/from low 128 bits of 256bit (YMM) vector registers
1498   void vinserti128_low(XMMRegister dst, XMMRegister src) {
1499     vinserti128(dst, dst, src, 0);
1500   }
1501   void vinserti128_low(XMMRegister dst, Address src) {
1502     vinserti128(dst, dst, src, 0);
1503   }
1504   void vextracti128_low(XMMRegister dst, XMMRegister src) {
1505     vextracti128(dst, src, 0);
1506   }
1507   void vextracti128_low(Address dst, XMMRegister src) {
1508     vextracti128(dst, src, 0);
1509   }
1510 
1511   void vinsertf128_low(XMMRegister dst, XMMRegister src) {
1512     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1513       Assembler::vinsertf32x4(dst, dst, src, 0);
1514     } else {
1515       Assembler::vinsertf128(dst, dst, src, 0);
1516     }
1517   }
1518 
1519   void vinsertf128_low(XMMRegister dst, Address src) {
1520     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1521       Assembler::vinsertf32x4(dst, dst, src, 0);
1522     } else {
1523       Assembler::vinsertf128(dst, dst, src, 0);
1524     }
1525   }
1526 
1527   void vextractf128_low(XMMRegister dst, XMMRegister src) {
1528     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1529       Assembler::vextractf32x4(dst, src, 0);
1530     } else {
1531       Assembler::vextractf128(dst, src, 0);
1532     }
1533   }
1534 
1535   void vextractf128_low(Address dst, XMMRegister src) {
1536     if (UseAVX > 2 && VM_Version::supports_avx512novl()) {
1537       Assembler::vextractf32x4(dst, src, 0);
1538     } else {
1539       Assembler::vextractf128(dst, src, 0);
1540     }
1541   }
1542 
1543   // 256bit copy to/from low 256 bits of 512bit (ZMM) vector registers
1544   void vinserti64x4_low(XMMRegister dst, XMMRegister src) {
1545     Assembler::vinserti64x4(dst, dst, src, 0);
1546   }
1547   void vinsertf64x4_low(XMMRegister dst, XMMRegister src) {
1548     Assembler::vinsertf64x4(dst, dst, src, 0);
1549   }
1550   void vextracti64x4_low(XMMRegister dst, XMMRegister src) {
1551     Assembler::vextracti64x4(dst, src, 0);
1552   }
1553   void vextractf64x4_low(XMMRegister dst, XMMRegister src) {
1554     Assembler::vextractf64x4(dst, src, 0);
1555   }
1556   void vextractf64x4_low(Address dst, XMMRegister src) {
1557     Assembler::vextractf64x4(dst, src, 0);
1558   }
1559   void vinsertf64x4_low(XMMRegister dst, Address src) {
1560     Assembler::vinsertf64x4(dst, dst, src, 0);
1561   }
1562 
1563   // Carry-Less Multiplication Quadword
1564   void vpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1565     // 0x00 - multiply lower 64 bits [0:63]
1566     Assembler::vpclmulqdq(dst, nds, src, 0x00);
1567   }
1568   void vpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1569     // 0x11 - multiply upper 64 bits [64:127]
1570     Assembler::vpclmulqdq(dst, nds, src, 0x11);
1571   }
1572   void vpclmullqhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1573     // 0x10 - multiply nds[0:63] and src[64:127]
1574     Assembler::vpclmulqdq(dst, nds, src, 0x10);
1575   }
1576   void vpclmulhqlqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1577     //0x01 - multiply nds[64:127] and src[0:63]
1578     Assembler::vpclmulqdq(dst, nds, src, 0x01);
1579   }
1580 
1581   void evpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1582     // 0x00 - multiply lower 64 bits [0:63]
1583     Assembler::evpclmulqdq(dst, nds, src, 0x00, vector_len);
1584   }
1585   void evpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1586     // 0x11 - multiply upper 64 bits [64:127]
1587     Assembler::evpclmulqdq(dst, nds, src, 0x11, vector_len);
1588   }
1589 
1590   // Data
1591 
1592   void cmov32( Condition cc, Register dst, Address  src);
1593   void cmov32( Condition cc, Register dst, Register src);
1594 
1595   void cmov(   Condition cc, Register dst, Register src) { cmovptr(cc, dst, src); }
1596 
1597   void cmovptr(Condition cc, Register dst, Address  src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1598   void cmovptr(Condition cc, Register dst, Register src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1599 
1600   void movoop(Register dst, jobject obj);
1601   void movoop(Address dst, jobject obj);
1602 
1603   void mov_metadata(Register dst, Metadata* obj);
1604   void mov_metadata(Address dst, Metadata* obj);
1605 
1606   void movptr(ArrayAddress dst, Register src);
1607   // can this do an lea?
1608   void movptr(Register dst, ArrayAddress src);
1609 
1610   void movptr(Register dst, Address src);
1611 
1612 #ifdef _LP64
1613   void movptr(Register dst, AddressLiteral src, Register scratch=rscratch1);
1614 #else
1615   void movptr(Register dst, AddressLiteral src, Register scratch=noreg); // Scratch reg is ignored in 32-bit
1616 #endif
1617 
1618   void movptr(Register dst, intptr_t src);
1619   void movptr(Register dst, Register src);
1620   void movptr(Address dst, intptr_t src);
1621 
1622   void movptr(Address dst, Register src);
1623 
1624   void movptr(Register dst, RegisterOrConstant src) {
1625     if (src.is_constant()) movptr(dst, src.as_constant());
1626     else                   movptr(dst, src.as_register());
1627   }
1628 
1629 #ifdef _LP64
1630   // Generally the next two are only used for moving NULL
1631   // Although there are situations in initializing the mark word where
1632   // they could be used. They are dangerous.
1633 
1634   // They only exist on LP64 so that int32_t and intptr_t are not the same
1635   // and we have ambiguous declarations.
1636 
1637   void movptr(Address dst, int32_t imm32);
1638   void movptr(Register dst, int32_t imm32);
1639 #endif // _LP64
1640 
1641   // to avoid hiding movl
1642   void mov32(AddressLiteral dst, Register src);
1643   void mov32(Register dst, AddressLiteral src);
1644 
1645   // to avoid hiding movb
1646   void movbyte(ArrayAddress dst, int src);
1647 
1648   // Import other mov() methods from the parent class or else
1649   // they will be hidden by the following overriding declaration.
1650   using Assembler::movdl;
1651   using Assembler::movq;
1652   void movdl(XMMRegister dst, AddressLiteral src);
1653   void movq(XMMRegister dst, AddressLiteral src);
1654 
1655   // Can push value or effective address
1656   void pushptr(AddressLiteral src);
1657 
1658   void pushptr(Address src) { LP64_ONLY(pushq(src)) NOT_LP64(pushl(src)); }
1659   void popptr(Address src) { LP64_ONLY(popq(src)) NOT_LP64(popl(src)); }
1660 
1661   void pushoop(jobject obj);
1662   void pushklass(Metadata* obj);
1663 
1664   // sign extend as need a l to ptr sized element
1665   void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
1666   void movl2ptr(Register dst, Register src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(if (dst != src) movl(dst, src)); }
1667 
1668 
1669  public:
1670   // C2 compiled method's prolog code.
1671   void verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub);
1672 
1673   // clear memory of size 'cnt' qwords, starting at 'base';
1674   // if 'is_large' is set, do not try to produce short loop
1675   void clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, bool is_large);
1676 
1677   // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
1678   void xmm_clear_mem(Register base, Register cnt, XMMRegister xtmp);
1679 
1680   // Fill primitive arrays
1681   void generate_fill(BasicType t, bool aligned,
1682                      Register to, Register value, Register count,
1683                      Register rtmp, XMMRegister xtmp);
1684 
1685   void encode_iso_array(Register src, Register dst, Register len,
1686                         XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1687                         XMMRegister tmp4, Register tmp5, Register result);
1688 
1689 #ifdef _LP64
1690   void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
1691   void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1692                              Register y, Register y_idx, Register z,
1693                              Register carry, Register product,
1694                              Register idx, Register kdx);
1695   void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
1696                               Register yz_idx, Register idx,
1697                               Register carry, Register product, int offset);
1698   void multiply_128_x_128_bmi2_loop(Register y, Register z,
1699                                     Register carry, Register carry2,
1700                                     Register idx, Register jdx,
1701                                     Register yz_idx1, Register yz_idx2,
1702                                     Register tmp, Register tmp3, Register tmp4);
1703   void multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
1704                                Register yz_idx, Register idx, Register jdx,
1705                                Register carry, Register product,
1706                                Register carry2);
1707   void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
1708                        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
1709   void square_rshift(Register x, Register len, Register z, Register tmp1, Register tmp3,
1710                      Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1711   void multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry,
1712                             Register tmp2);
1713   void multiply_add_64(Register sum, Register op1, Register op2, Register carry,
1714                        Register rdxReg, Register raxReg);
1715   void add_one_64(Register z, Register zlen, Register carry, Register tmp1);
1716   void lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1717                        Register tmp3, Register tmp4);
1718   void square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1719                      Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1720 
1721   void mul_add_128_x_32_loop(Register out, Register in, Register offset, Register len, Register tmp1,
1722                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1723                Register raxReg);
1724   void mul_add(Register out, Register in, Register offset, Register len, Register k, Register tmp1,
1725                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1726                Register raxReg);
1727   void vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
1728                            Register result, Register tmp1, Register tmp2,
1729                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3);
1730 #endif
1731 
1732   // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
1733   void update_byte_crc32(Register crc, Register val, Register table);
1734   void kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp);
1735 
1736 
1737 #ifdef _LP64
1738   void kernel_crc32_avx512(Register crc, Register buf, Register len, Register table, Register tmp1, Register tmp2);
1739   void kernel_crc32_avx512_256B(Register crc, Register buf, Register len, Register key, Register pos,
1740                                 Register tmp1, Register tmp2, Label& L_barrett, Label& L_16B_reduction_loop,
1741                                 Label& L_get_last_two_xmms, Label& L_128_done, Label& L_cleanup);
1742 #endif // _LP64
1743 
1744   // CRC32C code for java.util.zip.CRC32C::updateBytes() intrinsic
1745   // Note on a naming convention:
1746   // Prefix w = register only used on a Westmere+ architecture
1747   // Prefix n = register only used on a Nehalem architecture
1748 #ifdef _LP64
1749   void crc32c_ipl_alg4(Register in_out, uint32_t n,
1750                        Register tmp1, Register tmp2, Register tmp3);
1751 #else
1752   void crc32c_ipl_alg4(Register in_out, uint32_t n,
1753                        Register tmp1, Register tmp2, Register tmp3,
1754                        XMMRegister xtmp1, XMMRegister xtmp2);
1755 #endif
1756   void crc32c_pclmulqdq(XMMRegister w_xtmp1,
1757                         Register in_out,
1758                         uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
1759                         XMMRegister w_xtmp2,
1760                         Register tmp1,
1761                         Register n_tmp2, Register n_tmp3);
1762   void crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
1763                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1764                        Register tmp1, Register tmp2,
1765                        Register n_tmp3);
1766   void crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
1767                          Register in_out1, Register in_out2, Register in_out3,
1768                          Register tmp1, Register tmp2, Register tmp3,
1769                          XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1770                          Register tmp4, Register tmp5,
1771                          Register n_tmp6);
1772   void crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
1773                             Register tmp1, Register tmp2, Register tmp3,
1774                             Register tmp4, Register tmp5, Register tmp6,
1775                             XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1776                             bool is_pclmulqdq_supported);
1777   // Fold 128-bit data chunk
1778   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
1779   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf);
1780 #ifdef _LP64
1781   // Fold 512-bit data chunk
1782   void fold512bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, Register pos, int offset);
1783 #endif // _LP64
1784   // Fold 8-bit data
1785   void fold_8bit_crc32(Register crc, Register table, Register tmp);
1786   void fold_8bit_crc32(XMMRegister crc, Register table, XMMRegister xtmp, Register tmp);
1787 
1788   // Compress char[] array to byte[].
1789   void char_array_compress(Register src, Register dst, Register len,
1790                            XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1791                            XMMRegister tmp4, Register tmp5, Register result);
1792 
1793   // Inflate byte[] array to char[].
1794   void byte_array_inflate(Register src, Register dst, Register len,
1795                           XMMRegister tmp1, Register tmp2);
1796 
1797 #ifdef _LP64
1798   void convert_f2i(Register dst, XMMRegister src);
1799   void convert_d2i(Register dst, XMMRegister src);
1800   void convert_f2l(Register dst, XMMRegister src);
1801   void convert_d2l(Register dst, XMMRegister src);
1802 
1803   void cache_wb(Address line);
1804   void cache_wbsync(bool is_pre);
1805 #endif // _LP64
1806 
1807   void vallones(XMMRegister dst, int vector_len);
1808 };
1809 
1810 /**
1811  * class SkipIfEqual:
1812  *
1813  * Instantiating this class will result in assembly code being output that will
1814  * jump around any code emitted between the creation of the instance and it's
1815  * automatic destruction at the end of a scope block, depending on the value of
1816  * the flag passed to the constructor, which will be checked at run-time.
1817  */
1818 class SkipIfEqual {
1819  private:
1820   MacroAssembler* _masm;
1821   Label _label;
1822 
1823  public:
1824    SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
1825    ~SkipIfEqual();
1826 };
1827 
1828 #endif // CPU_X86_MACROASSEMBLER_X86_HPP