1 /*
   2  * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef CPU_X86_MACROASSEMBLER_X86_HPP
  26 #define CPU_X86_MACROASSEMBLER_X86_HPP
  27 
  28 #include "asm/assembler.hpp"
  29 #include "utilities/macros.hpp"
  30 #include "runtime/rtmLocking.hpp"
  31 #include "runtime/signature.hpp"
  32 
  33 // MacroAssembler extends Assembler by frequently used macros.
  34 //
  35 // Instructions for which a 'better' code sequence exists depending
  36 // on arguments should also go in here.
  37 
  38 class MacroAssembler: public Assembler {
  39   friend class LIR_Assembler;
  40   friend class Runtime1;      // as_Address()
  41 
  42  public:
  43   // Support for VM calls
  44   //
  45   // This is the base routine called by the different versions of call_VM_leaf. The interpreter
  46   // may customize this version by overriding it for its purposes (e.g., to save/restore
  47   // additional registers when doing a VM call).
  48 
  49   virtual void call_VM_leaf_base(
  50     address entry_point,               // the entry point
  51     int     number_of_arguments        // the number of arguments to pop after the call
  52   );
  53 
  54  protected:
  55   // This is the base routine called by the different versions of call_VM. The interpreter
  56   // may customize this version by overriding it for its purposes (e.g., to save/restore
  57   // additional registers when doing a VM call).
  58   //
  59   // If no java_thread register is specified (noreg) than rdi will be used instead. call_VM_base
  60   // returns the register which contains the thread upon return. If a thread register has been
  61   // specified, the return value will correspond to that register. If no last_java_sp is specified
  62   // (noreg) than rsp will be used instead.
  63   virtual void call_VM_base(           // returns the register containing the thread upon return
  64     Register oop_result,               // where an oop-result ends up if any; use noreg otherwise
  65     Register java_thread,              // the thread if computed before     ; use noreg otherwise
  66     Register last_java_sp,             // to set up last_Java_frame in stubs; use noreg otherwise
  67     address  entry_point,              // the entry point
  68     int      number_of_arguments,      // the number of arguments (w/o thread) to pop after the call
  69     bool     check_exceptions          // whether to check for pending exceptions after return
  70   );
  71 
  72   void call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions = true);
  73 
  74   // helpers for FPU flag access
  75   // tmp is a temporary register, if none is available use noreg
  76   void save_rax   (Register tmp);
  77   void restore_rax(Register tmp);
  78 
  79  public:
  80   MacroAssembler(CodeBuffer* code) : Assembler(code) {}
  81 
  82  // These routines should emit JVMTI PopFrame and ForceEarlyReturn handling code.
  83  // The implementation is only non-empty for the InterpreterMacroAssembler,
  84  // as only the interpreter handles PopFrame and ForceEarlyReturn requests.
  85  virtual void check_and_handle_popframe(Register java_thread);
  86  virtual void check_and_handle_earlyret(Register java_thread);
  87 
  88   Address as_Address(AddressLiteral adr);
  89   Address as_Address(ArrayAddress adr);
  90 
  91   // Support for NULL-checks
  92   //
  93   // Generates code that causes a NULL OS exception if the content of reg is NULL.
  94   // If the accessed location is M[reg + offset] and the offset is known, provide the
  95   // offset. No explicit code generation is needed if the offset is within a certain
  96   // range (0 <= offset <= page_size).
  97 
  98   void null_check(Register reg, int offset = -1);
  99   static bool needs_explicit_null_check(intptr_t offset);
 100   static bool uses_implicit_null_check(void* address);
 101 
 102   void test_klass_is_value(Register klass, Register temp_reg, Label& is_value);
 103 
 104   void test_field_is_flattenable(Register flags, Register temp_reg, Label& is_flattenable);
 105   void test_field_is_not_flattenable(Register flags, Register temp_reg, Label& notFlattenable);
 106   void test_field_is_flattened(Register flags, Register temp_reg, Label& is_flattened);
 107 
 108   // Check oops array storage properties, i.e. flattened and/or null-free
 109   void test_flattened_array_oop(Register oop, Register temp_reg, Label&is_flattened_array);
 110   void test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array);
 111 
 112   // Required platform-specific helpers for Label::patch_instructions.
 113   // They _shadow_ the declarations in AbstractAssembler, which are undefined.
 114   void pd_patch_instruction(address branch, address target, const char* file, int line) {
 115     unsigned char op = branch[0];
 116     assert(op == 0xE8 /* call */ ||
 117         op == 0xE9 /* jmp */ ||
 118         op == 0xEB /* short jmp */ ||
 119         (op & 0xF0) == 0x70 /* short jcc */ ||
 120         op == 0x0F && (branch[1] & 0xF0) == 0x80 /* jcc */ ||
 121         op == 0xC7 && branch[1] == 0xF8 /* xbegin */,
 122         "Invalid opcode at patch point");
 123 
 124     if (op == 0xEB || (op & 0xF0) == 0x70) {
 125       // short offset operators (jmp and jcc)
 126       char* disp = (char*) &branch[1];
 127       int imm8 = target - (address) &disp[1];
 128       guarantee(this->is8bit(imm8), "Short forward jump exceeds 8-bit offset at %s:%d", file, line);
 129       *disp = imm8;
 130     } else {
 131       int* disp = (int*) &branch[(op == 0x0F || op == 0xC7)? 2: 1];
 132       int imm32 = target - (address) &disp[1];
 133       *disp = imm32;
 134     }
 135   }
 136 
 137   // The following 4 methods return the offset of the appropriate move instruction
 138 
 139   // Support for fast byte/short loading with zero extension (depending on particular CPU)
 140   int load_unsigned_byte(Register dst, Address src);
 141   int load_unsigned_short(Register dst, Address src);
 142 
 143   // Support for fast byte/short loading with sign extension (depending on particular CPU)
 144   int load_signed_byte(Register dst, Address src);
 145   int load_signed_short(Register dst, Address src);
 146 
 147   // Support for sign-extension (hi:lo = extend_sign(lo))
 148   void extend_sign(Register hi, Register lo);
 149 
 150   // Load and store values by size and signed-ness
 151   void load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2 = noreg);
 152   void store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2 = noreg);
 153 
 154   // Support for inc/dec with optimal instruction selection depending on value
 155 
 156   void increment(Register reg, int value = 1) { LP64_ONLY(incrementq(reg, value)) NOT_LP64(incrementl(reg, value)) ; }
 157   void decrement(Register reg, int value = 1) { LP64_ONLY(decrementq(reg, value)) NOT_LP64(decrementl(reg, value)) ; }
 158 
 159   void decrementl(Address dst, int value = 1);
 160   void decrementl(Register reg, int value = 1);
 161 
 162   void decrementq(Register reg, int value = 1);
 163   void decrementq(Address dst, int value = 1);
 164 
 165   void incrementl(Address dst, int value = 1);
 166   void incrementl(Register reg, int value = 1);
 167 
 168   void incrementq(Register reg, int value = 1);
 169   void incrementq(Address dst, int value = 1);
 170 
 171 #ifdef COMPILER2
 172   // special instructions for EVEX
 173   void setvectmask(Register dst, Register src);
 174   void restorevectmask();
 175 #endif
 176 
 177   // Support optimal SSE move instructions.
 178   void movflt(XMMRegister dst, XMMRegister src) {
 179     if (dst-> encoding() == src->encoding()) return;
 180     if (UseXmmRegToRegMoveAll) { movaps(dst, src); return; }
 181     else                       { movss (dst, src); return; }
 182   }
 183   void movflt(XMMRegister dst, Address src) { movss(dst, src); }
 184   void movflt(XMMRegister dst, AddressLiteral src);
 185   void movflt(Address dst, XMMRegister src) { movss(dst, src); }
 186 
 187   void movdbl(XMMRegister dst, XMMRegister src) {
 188     if (dst-> encoding() == src->encoding()) return;
 189     if (UseXmmRegToRegMoveAll) { movapd(dst, src); return; }
 190     else                       { movsd (dst, src); return; }
 191   }
 192 
 193   void movdbl(XMMRegister dst, AddressLiteral src);
 194 
 195   void movdbl(XMMRegister dst, Address src) {
 196     if (UseXmmLoadAndClearUpper) { movsd (dst, src); return; }
 197     else                         { movlpd(dst, src); return; }
 198   }
 199   void movdbl(Address dst, XMMRegister src) { movsd(dst, src); }
 200 
 201   void incrementl(AddressLiteral dst);
 202   void incrementl(ArrayAddress dst);
 203 
 204   void incrementq(AddressLiteral dst);
 205 
 206   // Alignment
 207   void align(int modulus);
 208   void align(int modulus, int target);
 209 
 210   // A 5 byte nop that is safe for patching (see patch_verified_entry)
 211   void fat_nop();
 212 
 213   // Stack frame creation/removal
 214   void enter();
 215   void leave();
 216 
 217   // Support for getting the JavaThread pointer (i.e.; a reference to thread-local information)
 218   // The pointer will be loaded into the thread register.
 219   void get_thread(Register thread);
 220 
 221 
 222   // Support for VM calls
 223   //
 224   // It is imperative that all calls into the VM are handled via the call_VM macros.
 225   // They make sure that the stack linkage is setup correctly. call_VM's correspond
 226   // to ENTRY/ENTRY_X entry points while call_VM_leaf's correspond to LEAF entry points.
 227 
 228 
 229   void call_VM(Register oop_result,
 230                address entry_point,
 231                bool check_exceptions = true);
 232   void call_VM(Register oop_result,
 233                address entry_point,
 234                Register arg_1,
 235                bool check_exceptions = true);
 236   void call_VM(Register oop_result,
 237                address entry_point,
 238                Register arg_1, Register arg_2,
 239                bool check_exceptions = true);
 240   void call_VM(Register oop_result,
 241                address entry_point,
 242                Register arg_1, Register arg_2, Register arg_3,
 243                bool check_exceptions = true);
 244 
 245   // Overloadings with last_Java_sp
 246   void call_VM(Register oop_result,
 247                Register last_java_sp,
 248                address entry_point,
 249                int number_of_arguments = 0,
 250                bool check_exceptions = true);
 251   void call_VM(Register oop_result,
 252                Register last_java_sp,
 253                address entry_point,
 254                Register arg_1, bool
 255                check_exceptions = true);
 256   void call_VM(Register oop_result,
 257                Register last_java_sp,
 258                address entry_point,
 259                Register arg_1, Register arg_2,
 260                bool check_exceptions = true);
 261   void call_VM(Register oop_result,
 262                Register last_java_sp,
 263                address entry_point,
 264                Register arg_1, Register arg_2, Register arg_3,
 265                bool check_exceptions = true);
 266 
 267   void get_vm_result  (Register oop_result, Register thread);
 268   void get_vm_result_2(Register metadata_result, Register thread);
 269 
 270   // These always tightly bind to MacroAssembler::call_VM_base
 271   // bypassing the virtual implementation
 272   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, int number_of_arguments = 0, bool check_exceptions = true);
 273   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, bool check_exceptions = true);
 274   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
 275   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, bool check_exceptions = true);
 276   void super_call_VM(Register oop_result, Register last_java_sp, address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4, bool check_exceptions = true);
 277 
 278   void call_VM_leaf0(address entry_point);
 279   void call_VM_leaf(address entry_point,
 280                     int number_of_arguments = 0);
 281   void call_VM_leaf(address entry_point,
 282                     Register arg_1);
 283   void call_VM_leaf(address entry_point,
 284                     Register arg_1, Register arg_2);
 285   void call_VM_leaf(address entry_point,
 286                     Register arg_1, Register arg_2, Register arg_3);
 287 
 288   // These always tightly bind to MacroAssembler::call_VM_leaf_base
 289   // bypassing the virtual implementation
 290   void super_call_VM_leaf(address entry_point);
 291   void super_call_VM_leaf(address entry_point, Register arg_1);
 292   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
 293   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
 294   void super_call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3, Register arg_4);
 295 
 296   // last Java Frame (fills frame anchor)
 297   void set_last_Java_frame(Register thread,
 298                            Register last_java_sp,
 299                            Register last_java_fp,
 300                            address last_java_pc);
 301 
 302   // thread in the default location (r15_thread on 64bit)
 303   void set_last_Java_frame(Register last_java_sp,
 304                            Register last_java_fp,
 305                            address last_java_pc);
 306 
 307   void reset_last_Java_frame(Register thread, bool clear_fp);
 308 
 309   // thread in the default location (r15_thread on 64bit)
 310   void reset_last_Java_frame(bool clear_fp);
 311 
 312   // jobjects
 313   void clear_jweak_tag(Register possibly_jweak);
 314   void resolve_jobject(Register value, Register thread, Register tmp);
 315 
 316   // C 'boolean' to Java boolean: x == 0 ? 0 : 1
 317   void c2bool(Register x);
 318 
 319   // C++ bool manipulation
 320 
 321   void movbool(Register dst, Address src);
 322   void movbool(Address dst, bool boolconst);
 323   void movbool(Address dst, Register src);
 324   void testbool(Register dst);
 325 
 326   void resolve_oop_handle(Register result, Register tmp = rscratch2);
 327   void load_mirror(Register mirror, Register method, Register tmp = rscratch2);
 328 
 329   // oop manipulations
 330   void load_metadata(Register dst, Register src);
 331   void load_storage_props(Register dst, Register src);
 332   void load_klass(Register dst, Register src);
 333   void store_klass(Register dst, Register src);
 334 
 335   void access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 336                       Register tmp1, Register thread_tmp);
 337   void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
 338                        Register tmp1, Register tmp2);
 339 
 340   // Resolves obj access. Result is placed in the same register.
 341   // All other registers are preserved.
 342   void resolve(DecoratorSet decorators, Register obj);
 343 
 344   void load_heap_oop(Register dst, Address src, Register tmp1 = noreg,
 345                      Register thread_tmp = noreg, DecoratorSet decorators = 0);
 346   void load_heap_oop_not_null(Register dst, Address src, Register tmp1 = noreg,
 347                               Register thread_tmp = noreg, DecoratorSet decorators = 0);
 348   void store_heap_oop(Address dst, Register src, Register tmp1 = noreg,
 349                       Register tmp2 = noreg, DecoratorSet decorators = 0);
 350 
 351   // Used for storing NULL. All other oop constants should be
 352   // stored using routines that take a jobject.
 353   void store_heap_oop_null(Address dst);
 354 
 355   void load_prototype_header(Register dst, Register src);
 356 
 357 #ifdef _LP64
 358   void store_klass_gap(Register dst, Register src);
 359 
 360   // This dummy is to prevent a call to store_heap_oop from
 361   // converting a zero (like NULL) into a Register by giving
 362   // the compiler two choices it can't resolve
 363 
 364   void store_heap_oop(Address dst, void* dummy);
 365 
 366   void encode_heap_oop(Register r);
 367   void decode_heap_oop(Register r);
 368   void encode_heap_oop_not_null(Register r);
 369   void decode_heap_oop_not_null(Register r);
 370   void encode_heap_oop_not_null(Register dst, Register src);
 371   void decode_heap_oop_not_null(Register dst, Register src);
 372 
 373   void set_narrow_oop(Register dst, jobject obj);
 374   void set_narrow_oop(Address dst, jobject obj);
 375   void cmp_narrow_oop(Register dst, jobject obj);
 376   void cmp_narrow_oop(Address dst, jobject obj);
 377 
 378   void encode_klass_not_null(Register r);
 379   void decode_klass_not_null(Register r);
 380   void encode_klass_not_null(Register dst, Register src);
 381   void decode_klass_not_null(Register dst, Register src);
 382   void set_narrow_klass(Register dst, Klass* k);
 383   void set_narrow_klass(Address dst, Klass* k);
 384   void cmp_narrow_klass(Register dst, Klass* k);
 385   void cmp_narrow_klass(Address dst, Klass* k);
 386 
 387   // Returns the byte size of the instructions generated by decode_klass_not_null()
 388   // when compressed klass pointers are being used.
 389   static int instr_size_for_decode_klass_not_null();
 390 
 391   // if heap base register is used - reinit it with the correct value
 392   void reinit_heapbase();
 393 
 394   DEBUG_ONLY(void verify_heapbase(const char* msg);)
 395 
 396 #endif // _LP64
 397 
 398   // Int division/remainder for Java
 399   // (as idivl, but checks for special case as described in JVM spec.)
 400   // returns idivl instruction offset for implicit exception handling
 401   int corrected_idivl(Register reg);
 402 
 403   // Long division/remainder for Java
 404   // (as idivq, but checks for special case as described in JVM spec.)
 405   // returns idivq instruction offset for implicit exception handling
 406   int corrected_idivq(Register reg);
 407 
 408   void int3();
 409 
 410   // Long operation macros for a 32bit cpu
 411   // Long negation for Java
 412   void lneg(Register hi, Register lo);
 413 
 414   // Long multiplication for Java
 415   // (destroys contents of eax, ebx, ecx and edx)
 416   void lmul(int x_rsp_offset, int y_rsp_offset); // rdx:rax = x * y
 417 
 418   // Long shifts for Java
 419   // (semantics as described in JVM spec.)
 420   void lshl(Register hi, Register lo);                               // hi:lo << (rcx & 0x3f)
 421   void lshr(Register hi, Register lo, bool sign_extension = false);  // hi:lo >> (rcx & 0x3f)
 422 
 423   // Long compare for Java
 424   // (semantics as described in JVM spec.)
 425   void lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo); // x_hi = lcmp(x, y)
 426 
 427 
 428   // misc
 429 
 430   // Sign extension
 431   void sign_extend_short(Register reg);
 432   void sign_extend_byte(Register reg);
 433 
 434   // Division by power of 2, rounding towards 0
 435   void division_with_shift(Register reg, int shift_value);
 436 
 437   // Compares the top-most stack entries on the FPU stack and sets the eflags as follows:
 438   //
 439   // CF (corresponds to C0) if x < y
 440   // PF (corresponds to C2) if unordered
 441   // ZF (corresponds to C3) if x = y
 442   //
 443   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 444   // tmp is a temporary register, if none is available use noreg (only matters for non-P6 code)
 445   void fcmp(Register tmp);
 446   // Variant of the above which allows y to be further down the stack
 447   // and which only pops x and y if specified. If pop_right is
 448   // specified then pop_left must also be specified.
 449   void fcmp(Register tmp, int index, bool pop_left, bool pop_right);
 450 
 451   // Floating-point comparison for Java
 452   // Compares the top-most stack entries on the FPU stack and stores the result in dst.
 453   // The arguments are in reversed order on the stack (i.e., top of stack is first argument).
 454   // (semantics as described in JVM spec.)
 455   void fcmp2int(Register dst, bool unordered_is_less);
 456   // Variant of the above which allows y to be further down the stack
 457   // and which only pops x and y if specified. If pop_right is
 458   // specified then pop_left must also be specified.
 459   void fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right);
 460 
 461   // Floating-point remainder for Java (ST0 = ST0 fremr ST1, ST1 is empty afterwards)
 462   // tmp is a temporary register, if none is available use noreg
 463   void fremr(Register tmp);
 464 
 465   // dst = c = a * b + c
 466   void fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 467   void fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c);
 468 
 469   void vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 470   void vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len);
 471   void vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 472   void vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len);
 473 
 474 
 475   // same as fcmp2int, but using SSE2
 476   void cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 477   void cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less);
 478 
 479   // branch to L if FPU flag C2 is set/not set
 480   // tmp is a temporary register, if none is available use noreg
 481   void jC2 (Register tmp, Label& L);
 482   void jnC2(Register tmp, Label& L);
 483 
 484   // Pop ST (ffree & fincstp combined)
 485   void fpop();
 486 
 487   // Load float value from 'address'. If UseSSE >= 1, the value is loaded into
 488   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 489   void load_float(Address src);
 490 
 491   // Store float value to 'address'. If UseSSE >= 1, the value is stored
 492   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 493   void store_float(Address dst);
 494 
 495   // Load double value from 'address'. If UseSSE >= 2, the value is loaded into
 496   // register xmm0. Otherwise, the value is loaded onto the FPU stack.
 497   void load_double(Address src);
 498 
 499   // Store double value to 'address'. If UseSSE >= 2, the value is stored
 500   // from register xmm0. Otherwise, the value is stored from the FPU stack.
 501   void store_double(Address dst);
 502 
 503   // pushes double TOS element of FPU stack on CPU stack; pops from FPU stack
 504   void push_fTOS();
 505 
 506   // pops double TOS element from CPU stack and pushes on FPU stack
 507   void pop_fTOS();
 508 
 509   void empty_FPU_stack();
 510 
 511   void push_IU_state();
 512   void pop_IU_state();
 513 
 514   void push_FPU_state();
 515   void pop_FPU_state();
 516 
 517   void push_CPU_state();
 518   void pop_CPU_state();
 519 
 520   // Round up to a power of two
 521   void round_to(Register reg, int modulus);
 522 
 523   // Callee saved registers handling
 524   void push_callee_saved_registers();
 525   void pop_callee_saved_registers();
 526 
 527   // allocation
 528   void eden_allocate(
 529     Register thread,                   // Current thread
 530     Register obj,                      // result: pointer to object after successful allocation
 531     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
 532     int      con_size_in_bytes,        // object size in bytes if   known at compile time
 533     Register t1,                       // temp register
 534     Label&   slow_case                 // continuation point if fast allocation fails
 535   );
 536   void tlab_allocate(
 537     Register thread,                   // Current thread
 538     Register obj,                      // result: pointer to object after successful allocation
 539     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
 540     int      con_size_in_bytes,        // object size in bytes if   known at compile time
 541     Register t1,                       // temp register
 542     Register t2,                       // temp register
 543     Label&   slow_case                 // continuation point if fast allocation fails
 544   );
 545   void zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp);
 546 
 547   // interface method calling
 548   void lookup_interface_method(Register recv_klass,
 549                                Register intf_klass,
 550                                RegisterOrConstant itable_index,
 551                                Register method_result,
 552                                Register scan_temp,
 553                                Label& no_such_interface,
 554                                bool return_method = true);
 555 
 556   // virtual method calling
 557   void lookup_virtual_method(Register recv_klass,
 558                              RegisterOrConstant vtable_index,
 559                              Register method_result);
 560 
 561   // Test sub_klass against super_klass, with fast and slow paths.
 562 
 563   // The fast path produces a tri-state answer: yes / no / maybe-slow.
 564   // One of the three labels can be NULL, meaning take the fall-through.
 565   // If super_check_offset is -1, the value is loaded up from super_klass.
 566   // No registers are killed, except temp_reg.
 567   void check_klass_subtype_fast_path(Register sub_klass,
 568                                      Register super_klass,
 569                                      Register temp_reg,
 570                                      Label* L_success,
 571                                      Label* L_failure,
 572                                      Label* L_slow_path,
 573                 RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
 574 
 575   // The rest of the type check; must be wired to a corresponding fast path.
 576   // It does not repeat the fast path logic, so don't use it standalone.
 577   // The temp_reg and temp2_reg can be noreg, if no temps are available.
 578   // Updates the sub's secondary super cache as necessary.
 579   // If set_cond_codes, condition codes will be Z on success, NZ on failure.
 580   void check_klass_subtype_slow_path(Register sub_klass,
 581                                      Register super_klass,
 582                                      Register temp_reg,
 583                                      Register temp2_reg,
 584                                      Label* L_success,
 585                                      Label* L_failure,
 586                                      bool set_cond_codes = false);
 587 
 588   // Simplified, combined version, good for typical uses.
 589   // Falls through on failure.
 590   void check_klass_subtype(Register sub_klass,
 591                            Register super_klass,
 592                            Register temp_reg,
 593                            Label& L_success);
 594 
 595   // method handles (JSR 292)
 596   Address argument_address(RegisterOrConstant arg_slot, int extra_slot_offset = 0);
 597 
 598   //----
 599   void set_word_if_not_zero(Register reg); // sets reg to 1 if not zero, otherwise 0
 600 
 601   // Debugging
 602 
 603   // only if +VerifyOops
 604   // TODO: Make these macros with file and line like sparc version!
 605   void verify_oop(Register reg, const char* s = "broken oop");
 606   void verify_oop_addr(Address addr, const char * s = "broken oop addr");
 607 
 608   // TODO: verify method and klass metadata (compare against vptr?)
 609   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
 610   void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line){}
 611 
 612 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
 613 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
 614 
 615   // only if +VerifyFPU
 616   void verify_FPU(int stack_depth, const char* s = "illegal FPU state");
 617 
 618   // Verify or restore cpu control state after JNI call
 619   void restore_cpu_control_state_after_jni();
 620 
 621   // prints msg, dumps registers and stops execution
 622   void stop(const char* msg);
 623 
 624   // prints msg and continues
 625   void warn(const char* msg);
 626 
 627   // dumps registers and other state
 628   void print_state();
 629 
 630   static void debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg);
 631   static void debug64(char* msg, int64_t pc, int64_t regs[]);
 632   static void print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip);
 633   static void print_state64(int64_t pc, int64_t regs[]);
 634 
 635   void os_breakpoint();
 636 
 637   void untested()                                { stop("untested"); }
 638 
 639   void unimplemented(const char* what = "");
 640 
 641   void should_not_reach_here()                   { stop("should not reach here"); }
 642 
 643   void print_CPU_state();
 644 
 645   // Stack overflow checking
 646   void bang_stack_with_offset(int offset) {
 647     // stack grows down, caller passes positive offset
 648     assert(offset > 0, "must bang with negative offset");
 649     movl(Address(rsp, (-offset)), rax);
 650   }
 651 
 652   // Writes to stack successive pages until offset reached to check for
 653   // stack overflow + shadow pages.  Also, clobbers tmp
 654   void bang_stack_size(Register size, Register tmp);
 655 
 656   // Check for reserved stack access in method being exited (for JIT)
 657   void reserved_stack_check();
 658 
 659   virtual RegisterOrConstant delayed_value_impl(intptr_t* delayed_value_addr,
 660                                                 Register tmp,
 661                                                 int offset);
 662 
 663   // If thread_reg is != noreg the code assumes the register passed contains
 664   // the thread (required on 64 bit).
 665   void safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg);
 666 
 667   void verify_tlab();
 668 
 669   // Biased locking support
 670   // lock_reg and obj_reg must be loaded up with the appropriate values.
 671   // swap_reg must be rax, and is killed.
 672   // tmp_reg is optional. If it is supplied (i.e., != noreg) it will
 673   // be killed; if not supplied, push/pop will be used internally to
 674   // allocate a temporary (inefficient, avoid if possible).
 675   // Optional slow case is for implementations (interpreter and C1) which branch to
 676   // slow case directly. Leaves condition codes set for C2's Fast_Lock node.
 677   // Returns offset of first potentially-faulting instruction for null
 678   // check info (currently consumed only by C1). If
 679   // swap_reg_contains_mark is true then returns -1 as it is assumed
 680   // the calling code has already passed any potential faults.
 681   int biased_locking_enter(Register lock_reg, Register obj_reg,
 682                            Register swap_reg, Register tmp_reg,
 683                            bool swap_reg_contains_mark,
 684                            Label& done, Label* slow_case = NULL,
 685                            BiasedLockingCounters* counters = NULL);
 686   void biased_locking_exit (Register obj_reg, Register temp_reg, Label& done);
 687 #ifdef COMPILER2
 688   // Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
 689   // See full desription in macroAssembler_x86.cpp.
 690   void fast_lock(Register obj, Register box, Register tmp,
 691                  Register scr, Register cx1, Register cx2,
 692                  BiasedLockingCounters* counters,
 693                  RTMLockingCounters* rtm_counters,
 694                  RTMLockingCounters* stack_rtm_counters,
 695                  Metadata* method_data,
 696                  bool use_rtm, bool profile_rtm);
 697   void fast_unlock(Register obj, Register box, Register tmp, bool use_rtm);
 698 #if INCLUDE_RTM_OPT
 699   void rtm_counters_update(Register abort_status, Register rtm_counters);
 700   void branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel);
 701   void rtm_abort_ratio_calculation(Register tmp, Register rtm_counters_reg,
 702                                    RTMLockingCounters* rtm_counters,
 703                                    Metadata* method_data);
 704   void rtm_profiling(Register abort_status_Reg, Register rtm_counters_Reg,
 705                      RTMLockingCounters* rtm_counters, Metadata* method_data, bool profile_rtm);
 706   void rtm_retry_lock_on_abort(Register retry_count, Register abort_status, Label& retryLabel);
 707   void rtm_retry_lock_on_busy(Register retry_count, Register box, Register tmp, Register scr, Label& retryLabel);
 708   void rtm_stack_locking(Register obj, Register tmp, Register scr,
 709                          Register retry_on_abort_count,
 710                          RTMLockingCounters* stack_rtm_counters,
 711                          Metadata* method_data, bool profile_rtm,
 712                          Label& DONE_LABEL, Label& IsInflated);
 713   void rtm_inflated_locking(Register obj, Register box, Register tmp,
 714                             Register scr, Register retry_on_busy_count,
 715                             Register retry_on_abort_count,
 716                             RTMLockingCounters* rtm_counters,
 717                             Metadata* method_data, bool profile_rtm,
 718                             Label& DONE_LABEL);
 719 #endif
 720 #endif
 721 
 722   Condition negate_condition(Condition cond);
 723 
 724   // Instructions that use AddressLiteral operands. These instruction can handle 32bit/64bit
 725   // operands. In general the names are modified to avoid hiding the instruction in Assembler
 726   // so that we don't need to implement all the varieties in the Assembler with trivial wrappers
 727   // here in MacroAssembler. The major exception to this rule is call
 728 
 729   // Arithmetics
 730 
 731 
 732   void addptr(Address dst, int32_t src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)) ; }
 733   void addptr(Address dst, Register src);
 734 
 735   void addptr(Register dst, Address src) { LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); }
 736   void addptr(Register dst, int32_t src);
 737   void addptr(Register dst, Register src);
 738   void addptr(Register dst, RegisterOrConstant src) {
 739     if (src.is_constant()) addptr(dst, (int) src.as_constant());
 740     else                   addptr(dst,       src.as_register());
 741   }
 742 
 743   void andptr(Register dst, int32_t src);
 744   void andptr(Register src1, Register src2) { LP64_ONLY(andq(src1, src2)) NOT_LP64(andl(src1, src2)) ; }
 745 
 746   void cmp8(AddressLiteral src1, int imm);
 747 
 748   // renamed to drag out the casting of address to int32_t/intptr_t
 749   void cmp32(Register src1, int32_t imm);
 750 
 751   void cmp32(AddressLiteral src1, int32_t imm);
 752   // compare reg - mem, or reg - &mem
 753   void cmp32(Register src1, AddressLiteral src2);
 754 
 755   void cmp32(Register src1, Address src2);
 756 
 757 #ifndef _LP64
 758   void cmpklass(Address dst, Metadata* obj);
 759   void cmpklass(Register dst, Metadata* obj);
 760   void cmpoop(Address dst, jobject obj);
 761   void cmpoop_raw(Address dst, jobject obj);
 762 #endif // _LP64
 763 
 764   void cmpoop(Register src1, Register src2);
 765   void cmpoop(Register src1, Address src2);
 766   void cmpoop(Register dst, jobject obj);
 767   void cmpoop_raw(Register dst, jobject obj);
 768 
 769   // NOTE src2 must be the lval. This is NOT an mem-mem compare
 770   void cmpptr(Address src1, AddressLiteral src2);
 771 
 772   void cmpptr(Register src1, AddressLiteral src2);
 773 
 774   void cmpptr(Register src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 775   void cmpptr(Register src1, Address src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 776   // void cmpptr(Address src1, Register src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 777 
 778   void cmpptr(Register src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 779   void cmpptr(Address src1, int32_t src2) { LP64_ONLY(cmpq(src1, src2)) NOT_LP64(cmpl(src1, src2)) ; }
 780 
 781   // cmp64 to avoild hiding cmpq
 782   void cmp64(Register src1, AddressLiteral src);
 783 
 784   void cmpxchgptr(Register reg, Address adr);
 785 
 786   void locked_cmpxchgptr(Register reg, AddressLiteral adr);
 787 
 788 
 789   void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
 790   void imulptr(Register dst, Register src, int imm32) { LP64_ONLY(imulq(dst, src, imm32)) NOT_LP64(imull(dst, src, imm32)); }
 791 
 792 
 793   void negptr(Register dst) { LP64_ONLY(negq(dst)) NOT_LP64(negl(dst)); }
 794 
 795   void notptr(Register dst) { LP64_ONLY(notq(dst)) NOT_LP64(notl(dst)); }
 796 
 797   void shlptr(Register dst, int32_t shift);
 798   void shlptr(Register dst) { LP64_ONLY(shlq(dst)) NOT_LP64(shll(dst)); }
 799 
 800   void shrptr(Register dst, int32_t shift);
 801   void shrptr(Register dst) { LP64_ONLY(shrq(dst)) NOT_LP64(shrl(dst)); }
 802 
 803   void sarptr(Register dst) { LP64_ONLY(sarq(dst)) NOT_LP64(sarl(dst)); }
 804   void sarptr(Register dst, int32_t src) { LP64_ONLY(sarq(dst, src)) NOT_LP64(sarl(dst, src)); }
 805 
 806   void subptr(Address dst, int32_t src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 807 
 808   void subptr(Register dst, Address src) { LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); }
 809   void subptr(Register dst, int32_t src);
 810   // Force generation of a 4 byte immediate value even if it fits into 8bit
 811   void subptr_imm32(Register dst, int32_t src);
 812   void subptr(Register dst, Register src);
 813   void subptr(Register dst, RegisterOrConstant src) {
 814     if (src.is_constant()) subptr(dst, (int) src.as_constant());
 815     else                   subptr(dst,       src.as_register());
 816   }
 817 
 818   void sbbptr(Address dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 819   void sbbptr(Register dst, int32_t src) { LP64_ONLY(sbbq(dst, src)) NOT_LP64(sbbl(dst, src)); }
 820 
 821   void xchgptr(Register src1, Register src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 822   void xchgptr(Register src1, Address src2) { LP64_ONLY(xchgq(src1, src2)) NOT_LP64(xchgl(src1, src2)) ; }
 823 
 824   void xaddptr(Address src1, Register src2) { LP64_ONLY(xaddq(src1, src2)) NOT_LP64(xaddl(src1, src2)) ; }
 825 
 826 
 827 
 828   // Helper functions for statistics gathering.
 829   // Conditionally (atomically, on MPs) increments passed counter address, preserving condition codes.
 830   void cond_inc32(Condition cond, AddressLiteral counter_addr);
 831   // Unconditional atomic increment.
 832   void atomic_incl(Address counter_addr);
 833   void atomic_incl(AddressLiteral counter_addr, Register scr = rscratch1);
 834 #ifdef _LP64
 835   void atomic_incq(Address counter_addr);
 836   void atomic_incq(AddressLiteral counter_addr, Register scr = rscratch1);
 837 #endif
 838   void atomic_incptr(AddressLiteral counter_addr, Register scr = rscratch1) { LP64_ONLY(atomic_incq(counter_addr, scr)) NOT_LP64(atomic_incl(counter_addr, scr)) ; }
 839   void atomic_incptr(Address counter_addr) { LP64_ONLY(atomic_incq(counter_addr)) NOT_LP64(atomic_incl(counter_addr)) ; }
 840 
 841   void lea(Register dst, AddressLiteral adr);
 842   void lea(Address dst, AddressLiteral adr);
 843   void lea(Register dst, Address adr) { Assembler::lea(dst, adr); }
 844 
 845   void leal32(Register dst, Address src) { leal(dst, src); }
 846 
 847   // Import other testl() methods from the parent class or else
 848   // they will be hidden by the following overriding declaration.
 849   using Assembler::testl;
 850   void testl(Register dst, AddressLiteral src);
 851 
 852   void orptr(Register dst, Address src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 853   void orptr(Register dst, Register src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 854   void orptr(Register dst, int32_t src) { LP64_ONLY(orq(dst, src)) NOT_LP64(orl(dst, src)); }
 855   void orptr(Address dst, int32_t imm32) { LP64_ONLY(orq(dst, imm32)) NOT_LP64(orl(dst, imm32)); }
 856 
 857   void testptr(Register src, int32_t imm32) {  LP64_ONLY(testq(src, imm32)) NOT_LP64(testl(src, imm32)); }
 858   void testptr(Register src1, Address src2) { LP64_ONLY(testq(src1, src2)) NOT_LP64(testl(src1, src2)); }
 859   void testptr(Register src1, Register src2);
 860 
 861   void xorptr(Register dst, Register src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 862   void xorptr(Register dst, Address src) { LP64_ONLY(xorq(dst, src)) NOT_LP64(xorl(dst, src)); }
 863 
 864   // Calls
 865 
 866   void call(Label& L, relocInfo::relocType rtype);
 867   void call(Register entry);
 868 
 869   // NOTE: this call transfers to the effective address of entry NOT
 870   // the address contained by entry. This is because this is more natural
 871   // for jumps/calls.
 872   void call(AddressLiteral entry);
 873 
 874   // Emit the CompiledIC call idiom
 875   void ic_call(address entry, jint method_index = 0);
 876 
 877   // Jumps
 878 
 879   // NOTE: these jumps tranfer to the effective address of dst NOT
 880   // the address contained by dst. This is because this is more natural
 881   // for jumps/calls.
 882   void jump(AddressLiteral dst);
 883   void jump_cc(Condition cc, AddressLiteral dst);
 884 
 885   // 32bit can do a case table jump in one instruction but we no longer allow the base
 886   // to be installed in the Address class. This jump will tranfers to the address
 887   // contained in the location described by entry (not the address of entry)
 888   void jump(ArrayAddress entry);
 889 
 890   // Floating
 891 
 892   void andpd(XMMRegister dst, Address src) { Assembler::andpd(dst, src); }
 893   void andpd(XMMRegister dst, AddressLiteral src);
 894   void andpd(XMMRegister dst, XMMRegister src) { Assembler::andpd(dst, src); }
 895 
 896   void andps(XMMRegister dst, XMMRegister src) { Assembler::andps(dst, src); }
 897   void andps(XMMRegister dst, Address src) { Assembler::andps(dst, src); }
 898   void andps(XMMRegister dst, AddressLiteral src);
 899 
 900   void comiss(XMMRegister dst, XMMRegister src) { Assembler::comiss(dst, src); }
 901   void comiss(XMMRegister dst, Address src) { Assembler::comiss(dst, src); }
 902   void comiss(XMMRegister dst, AddressLiteral src);
 903 
 904   void comisd(XMMRegister dst, XMMRegister src) { Assembler::comisd(dst, src); }
 905   void comisd(XMMRegister dst, Address src) { Assembler::comisd(dst, src); }
 906   void comisd(XMMRegister dst, AddressLiteral src);
 907 
 908   void fadd_s(Address src)        { Assembler::fadd_s(src); }
 909   void fadd_s(AddressLiteral src) { Assembler::fadd_s(as_Address(src)); }
 910 
 911   void fldcw(Address src) { Assembler::fldcw(src); }
 912   void fldcw(AddressLiteral src);
 913 
 914   void fld_s(int index)   { Assembler::fld_s(index); }
 915   void fld_s(Address src) { Assembler::fld_s(src); }
 916   void fld_s(AddressLiteral src);
 917 
 918   void fld_d(Address src) { Assembler::fld_d(src); }
 919   void fld_d(AddressLiteral src);
 920 
 921   void fld_x(Address src) { Assembler::fld_x(src); }
 922   void fld_x(AddressLiteral src);
 923 
 924   void fmul_s(Address src)        { Assembler::fmul_s(src); }
 925   void fmul_s(AddressLiteral src) { Assembler::fmul_s(as_Address(src)); }
 926 
 927   void ldmxcsr(Address src) { Assembler::ldmxcsr(src); }
 928   void ldmxcsr(AddressLiteral src);
 929 
 930 #ifdef _LP64
 931  private:
 932   void sha256_AVX2_one_round_compute(
 933     Register  reg_old_h,
 934     Register  reg_a,
 935     Register  reg_b,
 936     Register  reg_c,
 937     Register  reg_d,
 938     Register  reg_e,
 939     Register  reg_f,
 940     Register  reg_g,
 941     Register  reg_h,
 942     int iter);
 943   void sha256_AVX2_four_rounds_compute_first(int start);
 944   void sha256_AVX2_four_rounds_compute_last(int start);
 945   void sha256_AVX2_one_round_and_sched(
 946         XMMRegister xmm_0,     /* == ymm4 on 0, 1, 2, 3 iterations, then rotate 4 registers left on 4, 8, 12 iterations */
 947         XMMRegister xmm_1,     /* ymm5 */  /* full cycle is 16 iterations */
 948         XMMRegister xmm_2,     /* ymm6 */
 949         XMMRegister xmm_3,     /* ymm7 */
 950         Register    reg_a,      /* == eax on 0 iteration, then rotate 8 register right on each next iteration */
 951         Register    reg_b,      /* ebx */    /* full cycle is 8 iterations */
 952         Register    reg_c,      /* edi */
 953         Register    reg_d,      /* esi */
 954         Register    reg_e,      /* r8d */
 955         Register    reg_f,      /* r9d */
 956         Register    reg_g,      /* r10d */
 957         Register    reg_h,      /* r11d */
 958         int iter);
 959 
 960   void addm(int disp, Register r1, Register r2);
 961   void gfmul(XMMRegister tmp0, XMMRegister t);
 962   void schoolbookAAD(int i, Register subkeyH, XMMRegister data, XMMRegister tmp0,
 963                      XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3);
 964   void generateHtbl_one_block(Register htbl);
 965   void generateHtbl_eight_blocks(Register htbl);
 966  public:
 967   void sha256_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 968                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 969                    Register buf, Register state, Register ofs, Register limit, Register rsp,
 970                    bool multi_block, XMMRegister shuf_mask);
 971   void avx_ghash(Register state, Register htbl, Register data, Register blocks);
 972 #endif
 973 
 974 #ifdef _LP64
 975  private:
 976   void sha512_AVX2_one_round_compute(Register old_h, Register a, Register b, Register c, Register d,
 977                                      Register e, Register f, Register g, Register h, int iteration);
 978 
 979   void sha512_AVX2_one_round_and_schedule(XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
 980                                           Register a, Register b, Register c, Register d, Register e, Register f,
 981                                           Register g, Register h, int iteration);
 982 
 983   void addmq(int disp, Register r1, Register r2);
 984  public:
 985   void sha512_AVX2(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 986                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 987                    Register buf, Register state, Register ofs, Register limit, Register rsp, bool multi_block,
 988                    XMMRegister shuf_mask);
 989 #endif
 990 
 991   void fast_sha1(XMMRegister abcd, XMMRegister e0, XMMRegister e1, XMMRegister msg0,
 992                  XMMRegister msg1, XMMRegister msg2, XMMRegister msg3, XMMRegister shuf_mask,
 993                  Register buf, Register state, Register ofs, Register limit, Register rsp,
 994                  bool multi_block);
 995 
 996 #ifdef _LP64
 997   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
 998                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
 999                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1000                    bool multi_block, XMMRegister shuf_mask);
1001 #else
1002   void fast_sha256(XMMRegister msg, XMMRegister state0, XMMRegister state1, XMMRegister msgtmp0,
1003                    XMMRegister msgtmp1, XMMRegister msgtmp2, XMMRegister msgtmp3, XMMRegister msgtmp4,
1004                    Register buf, Register state, Register ofs, Register limit, Register rsp,
1005                    bool multi_block);
1006 #endif
1007 
1008   void fast_exp(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1009                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1010                 Register rax, Register rcx, Register rdx, Register tmp);
1011 
1012 #ifdef _LP64
1013   void fast_log(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1014                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1015                 Register rax, Register rcx, Register rdx, Register tmp1, Register tmp2);
1016 
1017   void fast_log10(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1018                   XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1019                   Register rax, Register rcx, Register rdx, Register r11);
1020 
1021   void fast_pow(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
1022                 XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7, Register rax, Register rcx,
1023                 Register rdx, Register tmp1, Register tmp2, Register tmp3, Register tmp4);
1024 
1025   void fast_sin(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1026                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1027                 Register rax, Register rbx, Register rcx, Register rdx, Register tmp1, Register tmp2,
1028                 Register tmp3, Register tmp4);
1029 
1030   void fast_cos(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1031                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1032                 Register rax, Register rcx, Register rdx, Register tmp1,
1033                 Register tmp2, Register tmp3, Register tmp4);
1034   void fast_tan(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1035                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1036                 Register rax, Register rcx, Register rdx, Register tmp1,
1037                 Register tmp2, Register tmp3, Register tmp4);
1038 #else
1039   void fast_log(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1040                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1041                 Register rax, Register rcx, Register rdx, Register tmp1);
1042 
1043   void fast_log10(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1044                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1045                 Register rax, Register rcx, Register rdx, Register tmp);
1046 
1047   void fast_pow(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3, XMMRegister xmm4,
1048                 XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7, Register rax, Register rcx,
1049                 Register rdx, Register tmp);
1050 
1051   void fast_sin(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1052                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1053                 Register rax, Register rbx, Register rdx);
1054 
1055   void fast_cos(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1056                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1057                 Register rax, Register rcx, Register rdx, Register tmp);
1058 
1059   void libm_sincos_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1060                         Register edx, Register ebx, Register esi, Register edi,
1061                         Register ebp, Register esp);
1062 
1063   void libm_reduce_pi04l(Register eax, Register ecx, Register edx, Register ebx,
1064                          Register esi, Register edi, Register ebp, Register esp);
1065 
1066   void libm_tancot_huge(XMMRegister xmm0, XMMRegister xmm1, Register eax, Register ecx,
1067                         Register edx, Register ebx, Register esi, Register edi,
1068                         Register ebp, Register esp);
1069 
1070   void fast_tan(XMMRegister xmm0, XMMRegister xmm1, XMMRegister xmm2, XMMRegister xmm3,
1071                 XMMRegister xmm4, XMMRegister xmm5, XMMRegister xmm6, XMMRegister xmm7,
1072                 Register rax, Register rcx, Register rdx, Register tmp);
1073 #endif
1074 
1075   void increase_precision();
1076   void restore_precision();
1077 
1078 private:
1079 
1080   // these are private because users should be doing movflt/movdbl
1081 
1082   void movss(Address dst, XMMRegister src)     { Assembler::movss(dst, src); }
1083   void movss(XMMRegister dst, XMMRegister src) { Assembler::movss(dst, src); }
1084   void movss(XMMRegister dst, Address src)     { Assembler::movss(dst, src); }
1085   void movss(XMMRegister dst, AddressLiteral src);
1086 
1087   void movlpd(XMMRegister dst, Address src)    {Assembler::movlpd(dst, src); }
1088   void movlpd(XMMRegister dst, AddressLiteral src);
1089 
1090 public:
1091 
1092   void addsd(XMMRegister dst, XMMRegister src)    { Assembler::addsd(dst, src); }
1093   void addsd(XMMRegister dst, Address src)        { Assembler::addsd(dst, src); }
1094   void addsd(XMMRegister dst, AddressLiteral src);
1095 
1096   void addss(XMMRegister dst, XMMRegister src)    { Assembler::addss(dst, src); }
1097   void addss(XMMRegister dst, Address src)        { Assembler::addss(dst, src); }
1098   void addss(XMMRegister dst, AddressLiteral src);
1099 
1100   void addpd(XMMRegister dst, XMMRegister src)    { Assembler::addpd(dst, src); }
1101   void addpd(XMMRegister dst, Address src)        { Assembler::addpd(dst, src); }
1102   void addpd(XMMRegister dst, AddressLiteral src);
1103 
1104   void divsd(XMMRegister dst, XMMRegister src)    { Assembler::divsd(dst, src); }
1105   void divsd(XMMRegister dst, Address src)        { Assembler::divsd(dst, src); }
1106   void divsd(XMMRegister dst, AddressLiteral src);
1107 
1108   void divss(XMMRegister dst, XMMRegister src)    { Assembler::divss(dst, src); }
1109   void divss(XMMRegister dst, Address src)        { Assembler::divss(dst, src); }
1110   void divss(XMMRegister dst, AddressLiteral src);
1111 
1112   // Move Unaligned Double Quadword
1113   void movdqu(Address     dst, XMMRegister src);
1114   void movdqu(XMMRegister dst, Address src);
1115   void movdqu(XMMRegister dst, XMMRegister src);
1116   void movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg = rscratch1);
1117   // AVX Unaligned forms
1118   void vmovdqu(Address     dst, XMMRegister src);
1119   void vmovdqu(XMMRegister dst, Address src);
1120   void vmovdqu(XMMRegister dst, XMMRegister src);
1121   void vmovdqu(XMMRegister dst, AddressLiteral src);
1122   void evmovdquq(XMMRegister dst, Address src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1123   void evmovdquq(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1124   void evmovdquq(Address dst, XMMRegister src, int vector_len) { Assembler::evmovdquq(dst, src, vector_len); }
1125   void evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch);
1126 
1127   // Move Aligned Double Quadword
1128   void movdqa(XMMRegister dst, Address src)       { Assembler::movdqa(dst, src); }
1129   void movdqa(XMMRegister dst, XMMRegister src)   { Assembler::movdqa(dst, src); }
1130   void movdqa(XMMRegister dst, AddressLiteral src);
1131 
1132   void movsd(XMMRegister dst, XMMRegister src) { Assembler::movsd(dst, src); }
1133   void movsd(Address dst, XMMRegister src)     { Assembler::movsd(dst, src); }
1134   void movsd(XMMRegister dst, Address src)     { Assembler::movsd(dst, src); }
1135   void movsd(XMMRegister dst, AddressLiteral src);
1136 
1137   void mulpd(XMMRegister dst, XMMRegister src)    { Assembler::mulpd(dst, src); }
1138   void mulpd(XMMRegister dst, Address src)        { Assembler::mulpd(dst, src); }
1139   void mulpd(XMMRegister dst, AddressLiteral src);
1140 
1141   void mulsd(XMMRegister dst, XMMRegister src)    { Assembler::mulsd(dst, src); }
1142   void mulsd(XMMRegister dst, Address src)        { Assembler::mulsd(dst, src); }
1143   void mulsd(XMMRegister dst, AddressLiteral src);
1144 
1145   void mulss(XMMRegister dst, XMMRegister src)    { Assembler::mulss(dst, src); }
1146   void mulss(XMMRegister dst, Address src)        { Assembler::mulss(dst, src); }
1147   void mulss(XMMRegister dst, AddressLiteral src);
1148 
1149   // Carry-Less Multiplication Quadword
1150   void pclmulldq(XMMRegister dst, XMMRegister src) {
1151     // 0x00 - multiply lower 64 bits [0:63]
1152     Assembler::pclmulqdq(dst, src, 0x00);
1153   }
1154   void pclmulhdq(XMMRegister dst, XMMRegister src) {
1155     // 0x11 - multiply upper 64 bits [64:127]
1156     Assembler::pclmulqdq(dst, src, 0x11);
1157   }
1158 
1159   void pcmpeqb(XMMRegister dst, XMMRegister src);
1160   void pcmpeqw(XMMRegister dst, XMMRegister src);
1161 
1162   void pcmpestri(XMMRegister dst, Address src, int imm8);
1163   void pcmpestri(XMMRegister dst, XMMRegister src, int imm8);
1164 
1165   void pmovzxbw(XMMRegister dst, XMMRegister src);
1166   void pmovzxbw(XMMRegister dst, Address src);
1167 
1168   void pmovmskb(Register dst, XMMRegister src);
1169 
1170   void ptest(XMMRegister dst, XMMRegister src);
1171 
1172   void sqrtsd(XMMRegister dst, XMMRegister src)    { Assembler::sqrtsd(dst, src); }
1173   void sqrtsd(XMMRegister dst, Address src)        { Assembler::sqrtsd(dst, src); }
1174   void sqrtsd(XMMRegister dst, AddressLiteral src);
1175 
1176   void sqrtss(XMMRegister dst, XMMRegister src)    { Assembler::sqrtss(dst, src); }
1177   void sqrtss(XMMRegister dst, Address src)        { Assembler::sqrtss(dst, src); }
1178   void sqrtss(XMMRegister dst, AddressLiteral src);
1179 
1180   void subsd(XMMRegister dst, XMMRegister src)    { Assembler::subsd(dst, src); }
1181   void subsd(XMMRegister dst, Address src)        { Assembler::subsd(dst, src); }
1182   void subsd(XMMRegister dst, AddressLiteral src);
1183 
1184   void subss(XMMRegister dst, XMMRegister src)    { Assembler::subss(dst, src); }
1185   void subss(XMMRegister dst, Address src)        { Assembler::subss(dst, src); }
1186   void subss(XMMRegister dst, AddressLiteral src);
1187 
1188   void ucomiss(XMMRegister dst, XMMRegister src) { Assembler::ucomiss(dst, src); }
1189   void ucomiss(XMMRegister dst, Address src)     { Assembler::ucomiss(dst, src); }
1190   void ucomiss(XMMRegister dst, AddressLiteral src);
1191 
1192   void ucomisd(XMMRegister dst, XMMRegister src) { Assembler::ucomisd(dst, src); }
1193   void ucomisd(XMMRegister dst, Address src)     { Assembler::ucomisd(dst, src); }
1194   void ucomisd(XMMRegister dst, AddressLiteral src);
1195 
1196   // Bitwise Logical XOR of Packed Double-Precision Floating-Point Values
1197   void xorpd(XMMRegister dst, XMMRegister src);
1198   void xorpd(XMMRegister dst, Address src)     { Assembler::xorpd(dst, src); }
1199   void xorpd(XMMRegister dst, AddressLiteral src);
1200 
1201   // Bitwise Logical XOR of Packed Single-Precision Floating-Point Values
1202   void xorps(XMMRegister dst, XMMRegister src);
1203   void xorps(XMMRegister dst, Address src)     { Assembler::xorps(dst, src); }
1204   void xorps(XMMRegister dst, AddressLiteral src);
1205 
1206   // Shuffle Bytes
1207   void pshufb(XMMRegister dst, XMMRegister src) { Assembler::pshufb(dst, src); }
1208   void pshufb(XMMRegister dst, Address src)     { Assembler::pshufb(dst, src); }
1209   void pshufb(XMMRegister dst, AddressLiteral src);
1210   // AVX 3-operands instructions
1211 
1212   void vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddsd(dst, nds, src); }
1213   void vaddsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vaddsd(dst, nds, src); }
1214   void vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1215 
1216   void vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vaddss(dst, nds, src); }
1217   void vaddss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vaddss(dst, nds, src); }
1218   void vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1219 
1220   void vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len);
1221   void vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len);
1222 
1223   void vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1224   void vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1225 
1226   void vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1227   void vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1228 
1229   void vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1230   void vpand(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vpand(dst, nds, src, vector_len); }
1231   void vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len);
1232 
1233   void vpbroadcastw(XMMRegister dst, XMMRegister src, int vector_len);
1234   void vpbroadcastw(XMMRegister dst, Address src, int vector_len) { Assembler::vpbroadcastw(dst, src, vector_len); }
1235 
1236   void vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1237 
1238   void vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1239 
1240   void vpmovzxbw(XMMRegister dst, Address src, int vector_len);
1241   void vpmovzxbw(XMMRegister dst, XMMRegister src, int vector_len) { Assembler::vpmovzxbw(dst, src, vector_len); }
1242 
1243   void vpmovmskb(Register dst, XMMRegister src);
1244 
1245   void vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1246   void vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1247 
1248   void vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1249   void vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1250 
1251   void vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len);
1252   void vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len);
1253 
1254   void vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1255   void vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1256 
1257   void vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1258   void vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1259 
1260   void vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len);
1261   void vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len);
1262 
1263   void vptest(XMMRegister dst, XMMRegister src);
1264 
1265   void punpcklbw(XMMRegister dst, XMMRegister src);
1266   void punpcklbw(XMMRegister dst, Address src) { Assembler::punpcklbw(dst, src); }
1267 
1268   void pshufd(XMMRegister dst, Address src, int mode);
1269   void pshufd(XMMRegister dst, XMMRegister src, int mode) { Assembler::pshufd(dst, src, mode); }
1270 
1271   void pshuflw(XMMRegister dst, XMMRegister src, int mode);
1272   void pshuflw(XMMRegister dst, Address src, int mode) { Assembler::pshuflw(dst, src, mode); }
1273 
1274   void vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandpd(dst, nds, src, vector_len); }
1275   void vandpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len)     { Assembler::vandpd(dst, nds, src, vector_len); }
1276   void vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len);
1277 
1278   void vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vandps(dst, nds, src, vector_len); }
1279   void vandps(XMMRegister dst, XMMRegister nds, Address src, int vector_len)     { Assembler::vandps(dst, nds, src, vector_len); }
1280   void vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len);
1281 
1282   void vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivsd(dst, nds, src); }
1283   void vdivsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vdivsd(dst, nds, src); }
1284   void vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1285 
1286   void vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vdivss(dst, nds, src); }
1287   void vdivss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vdivss(dst, nds, src); }
1288   void vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1289 
1290   void vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulsd(dst, nds, src); }
1291   void vmulsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vmulsd(dst, nds, src); }
1292   void vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1293 
1294   void vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vmulss(dst, nds, src); }
1295   void vmulss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vmulss(dst, nds, src); }
1296   void vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1297 
1298   void vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubsd(dst, nds, src); }
1299   void vsubsd(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vsubsd(dst, nds, src); }
1300   void vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1301 
1302   void vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) { Assembler::vsubss(dst, nds, src); }
1303   void vsubss(XMMRegister dst, XMMRegister nds, Address src)     { Assembler::vsubss(dst, nds, src); }
1304   void vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1305 
1306   void vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1307   void vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src);
1308 
1309   // AVX Vector instructions
1310 
1311   void vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1312   void vxorpd(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorpd(dst, nds, src, vector_len); }
1313   void vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len);
1314 
1315   void vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1316   void vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector_len) { Assembler::vxorps(dst, nds, src, vector_len); }
1317   void vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len);
1318 
1319   void vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1320     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1321       Assembler::vpxor(dst, nds, src, vector_len);
1322     else
1323       Assembler::vxorpd(dst, nds, src, vector_len);
1324   }
1325   void vpxor(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
1326     if (UseAVX > 1 || (vector_len < 1)) // vpxor 256 bit is available only in AVX2
1327       Assembler::vpxor(dst, nds, src, vector_len);
1328     else
1329       Assembler::vxorpd(dst, nds, src, vector_len);
1330   }
1331 
1332   // Simple version for AVX2 256bit vectors
1333   void vpxor(XMMRegister dst, XMMRegister src) { Assembler::vpxor(dst, dst, src, true); }
1334   void vpxor(XMMRegister dst, Address src) { Assembler::vpxor(dst, dst, src, true); }
1335 
1336   void vinserti128(XMMRegister dst, XMMRegister nds, XMMRegister src, uint8_t imm8) {
1337     if (UseAVX > 2) {
1338       Assembler::vinserti32x4(dst, dst, src, imm8);
1339     } else if (UseAVX > 1) {
1340       // vinserti128 is available only in AVX2
1341       Assembler::vinserti128(dst, nds, src, imm8);
1342     } else {
1343       Assembler::vinsertf128(dst, nds, src, imm8);
1344     }
1345   }
1346 
1347   void vinserti128(XMMRegister dst, XMMRegister nds, Address src, uint8_t imm8) {
1348     if (UseAVX > 2) {
1349       Assembler::vinserti32x4(dst, dst, src, imm8);
1350     } else if (UseAVX > 1) {
1351       // vinserti128 is available only in AVX2
1352       Assembler::vinserti128(dst, nds, src, imm8);
1353     } else {
1354       Assembler::vinsertf128(dst, nds, src, imm8);
1355     }
1356   }
1357 
1358   void vextracti128(XMMRegister dst, XMMRegister src, uint8_t imm8) {
1359     if (UseAVX > 2) {
1360       Assembler::vextracti32x4(dst, src, imm8);
1361     } else if (UseAVX > 1) {
1362       // vextracti128 is available only in AVX2
1363       Assembler::vextracti128(dst, src, imm8);
1364     } else {
1365       Assembler::vextractf128(dst, src, imm8);
1366     }
1367   }
1368 
1369   void vextracti128(Address dst, XMMRegister src, uint8_t imm8) {
1370     if (UseAVX > 2) {
1371       Assembler::vextracti32x4(dst, src, imm8);
1372     } else if (UseAVX > 1) {
1373       // vextracti128 is available only in AVX2
1374       Assembler::vextracti128(dst, src, imm8);
1375     } else {
1376       Assembler::vextractf128(dst, src, imm8);
1377     }
1378   }
1379 
1380   // 128bit copy to/from high 128 bits of 256bit (YMM) vector registers
1381   void vinserti128_high(XMMRegister dst, XMMRegister src) {
1382     vinserti128(dst, dst, src, 1);
1383   }
1384   void vinserti128_high(XMMRegister dst, Address src) {
1385     vinserti128(dst, dst, src, 1);
1386   }
1387   void vextracti128_high(XMMRegister dst, XMMRegister src) {
1388     vextracti128(dst, src, 1);
1389   }
1390   void vextracti128_high(Address dst, XMMRegister src) {
1391     vextracti128(dst, src, 1);
1392   }
1393 
1394   void vinsertf128_high(XMMRegister dst, XMMRegister src) {
1395     if (UseAVX > 2) {
1396       Assembler::vinsertf32x4(dst, dst, src, 1);
1397     } else {
1398       Assembler::vinsertf128(dst, dst, src, 1);
1399     }
1400   }
1401 
1402   void vinsertf128_high(XMMRegister dst, Address src) {
1403     if (UseAVX > 2) {
1404       Assembler::vinsertf32x4(dst, dst, src, 1);
1405     } else {
1406       Assembler::vinsertf128(dst, dst, src, 1);
1407     }
1408   }
1409 
1410   void vextractf128_high(XMMRegister dst, XMMRegister src) {
1411     if (UseAVX > 2) {
1412       Assembler::vextractf32x4(dst, src, 1);
1413     } else {
1414       Assembler::vextractf128(dst, src, 1);
1415     }
1416   }
1417 
1418   void vextractf128_high(Address dst, XMMRegister src) {
1419     if (UseAVX > 2) {
1420       Assembler::vextractf32x4(dst, src, 1);
1421     } else {
1422       Assembler::vextractf128(dst, src, 1);
1423     }
1424   }
1425 
1426   // 256bit copy to/from high 256 bits of 512bit (ZMM) vector registers
1427   void vinserti64x4_high(XMMRegister dst, XMMRegister src) {
1428     Assembler::vinserti64x4(dst, dst, src, 1);
1429   }
1430   void vinsertf64x4_high(XMMRegister dst, XMMRegister src) {
1431     Assembler::vinsertf64x4(dst, dst, src, 1);
1432   }
1433   void vextracti64x4_high(XMMRegister dst, XMMRegister src) {
1434     Assembler::vextracti64x4(dst, src, 1);
1435   }
1436   void vextractf64x4_high(XMMRegister dst, XMMRegister src) {
1437     Assembler::vextractf64x4(dst, src, 1);
1438   }
1439   void vextractf64x4_high(Address dst, XMMRegister src) {
1440     Assembler::vextractf64x4(dst, src, 1);
1441   }
1442   void vinsertf64x4_high(XMMRegister dst, Address src) {
1443     Assembler::vinsertf64x4(dst, dst, src, 1);
1444   }
1445 
1446   // 128bit copy to/from low 128 bits of 256bit (YMM) vector registers
1447   void vinserti128_low(XMMRegister dst, XMMRegister src) {
1448     vinserti128(dst, dst, src, 0);
1449   }
1450   void vinserti128_low(XMMRegister dst, Address src) {
1451     vinserti128(dst, dst, src, 0);
1452   }
1453   void vextracti128_low(XMMRegister dst, XMMRegister src) {
1454     vextracti128(dst, src, 0);
1455   }
1456   void vextracti128_low(Address dst, XMMRegister src) {
1457     vextracti128(dst, src, 0);
1458   }
1459 
1460   void vinsertf128_low(XMMRegister dst, XMMRegister src) {
1461     if (UseAVX > 2) {
1462       Assembler::vinsertf32x4(dst, dst, src, 0);
1463     } else {
1464       Assembler::vinsertf128(dst, dst, src, 0);
1465     }
1466   }
1467 
1468   void vinsertf128_low(XMMRegister dst, Address src) {
1469     if (UseAVX > 2) {
1470       Assembler::vinsertf32x4(dst, dst, src, 0);
1471     } else {
1472       Assembler::vinsertf128(dst, dst, src, 0);
1473     }
1474   }
1475 
1476   void vextractf128_low(XMMRegister dst, XMMRegister src) {
1477     if (UseAVX > 2) {
1478       Assembler::vextractf32x4(dst, src, 0);
1479     } else {
1480       Assembler::vextractf128(dst, src, 0);
1481     }
1482   }
1483 
1484   void vextractf128_low(Address dst, XMMRegister src) {
1485     if (UseAVX > 2) {
1486       Assembler::vextractf32x4(dst, src, 0);
1487     } else {
1488       Assembler::vextractf128(dst, src, 0);
1489     }
1490   }
1491 
1492   // 256bit copy to/from low 256 bits of 512bit (ZMM) vector registers
1493   void vinserti64x4_low(XMMRegister dst, XMMRegister src) {
1494     Assembler::vinserti64x4(dst, dst, src, 0);
1495   }
1496   void vinsertf64x4_low(XMMRegister dst, XMMRegister src) {
1497     Assembler::vinsertf64x4(dst, dst, src, 0);
1498   }
1499   void vextracti64x4_low(XMMRegister dst, XMMRegister src) {
1500     Assembler::vextracti64x4(dst, src, 0);
1501   }
1502   void vextractf64x4_low(XMMRegister dst, XMMRegister src) {
1503     Assembler::vextractf64x4(dst, src, 0);
1504   }
1505   void vextractf64x4_low(Address dst, XMMRegister src) {
1506     Assembler::vextractf64x4(dst, src, 0);
1507   }
1508   void vinsertf64x4_low(XMMRegister dst, Address src) {
1509     Assembler::vinsertf64x4(dst, dst, src, 0);
1510   }
1511 
1512   // Carry-Less Multiplication Quadword
1513   void vpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1514     // 0x00 - multiply lower 64 bits [0:63]
1515     Assembler::vpclmulqdq(dst, nds, src, 0x00);
1516   }
1517   void vpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1518     // 0x11 - multiply upper 64 bits [64:127]
1519     Assembler::vpclmulqdq(dst, nds, src, 0x11);
1520   }
1521   void vpclmullqhqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1522     // 0x10 - multiply nds[0:63] and src[64:127]
1523     Assembler::vpclmulqdq(dst, nds, src, 0x10);
1524   }
1525   void vpclmulhqlqdq(XMMRegister dst, XMMRegister nds, XMMRegister src) {
1526     //0x01 - multiply nds[64:127] and src[0:63]
1527     Assembler::vpclmulqdq(dst, nds, src, 0x01);
1528   }
1529 
1530   void evpclmulldq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1531     // 0x00 - multiply lower 64 bits [0:63]
1532     Assembler::evpclmulqdq(dst, nds, src, 0x00, vector_len);
1533   }
1534   void evpclmulhdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
1535     // 0x11 - multiply upper 64 bits [64:127]
1536     Assembler::evpclmulqdq(dst, nds, src, 0x11, vector_len);
1537   }
1538 
1539   // Data
1540 
1541   void cmov32( Condition cc, Register dst, Address  src);
1542   void cmov32( Condition cc, Register dst, Register src);
1543 
1544   void cmov(   Condition cc, Register dst, Register src) { cmovptr(cc, dst, src); }
1545 
1546   void cmovptr(Condition cc, Register dst, Address  src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1547   void cmovptr(Condition cc, Register dst, Register src) { LP64_ONLY(cmovq(cc, dst, src)) NOT_LP64(cmov32(cc, dst, src)); }
1548 
1549   void movoop(Register dst, jobject obj);
1550   void movoop(Address dst, jobject obj);
1551 
1552   void mov_metadata(Register dst, Metadata* obj);
1553   void mov_metadata(Address dst, Metadata* obj);
1554 
1555   void movptr(ArrayAddress dst, Register src);
1556   // can this do an lea?
1557   void movptr(Register dst, ArrayAddress src);
1558 
1559   void movptr(Register dst, Address src);
1560 
1561 #ifdef _LP64
1562   void movptr(Register dst, AddressLiteral src, Register scratch=rscratch1);
1563 #else
1564   void movptr(Register dst, AddressLiteral src, Register scratch=noreg); // Scratch reg is ignored in 32-bit
1565 #endif
1566 
1567   void movptr(Register dst, intptr_t src);
1568   void movptr(Register dst, Register src);
1569   void movptr(Address dst, intptr_t src);
1570 
1571   void movptr(Address dst, Register src);
1572 
1573   void movptr(Register dst, RegisterOrConstant src) {
1574     if (src.is_constant()) movptr(dst, src.as_constant());
1575     else                   movptr(dst, src.as_register());
1576   }
1577 
1578 #ifdef _LP64
1579   // Generally the next two are only used for moving NULL
1580   // Although there are situations in initializing the mark word where
1581   // they could be used. They are dangerous.
1582 
1583   // They only exist on LP64 so that int32_t and intptr_t are not the same
1584   // and we have ambiguous declarations.
1585 
1586   void movptr(Address dst, int32_t imm32);
1587   void movptr(Register dst, int32_t imm32);
1588 #endif // _LP64
1589 
1590   // to avoid hiding movl
1591   void mov32(AddressLiteral dst, Register src);
1592   void mov32(Register dst, AddressLiteral src);
1593 
1594   // to avoid hiding movb
1595   void movbyte(ArrayAddress dst, int src);
1596 
1597   // Import other mov() methods from the parent class or else
1598   // they will be hidden by the following overriding declaration.
1599   using Assembler::movdl;
1600   using Assembler::movq;
1601   void movdl(XMMRegister dst, AddressLiteral src);
1602   void movq(XMMRegister dst, AddressLiteral src);
1603 
1604   // Can push value or effective address
1605   void pushptr(AddressLiteral src);
1606 
1607   void pushptr(Address src) { LP64_ONLY(pushq(src)) NOT_LP64(pushl(src)); }
1608   void popptr(Address src) { LP64_ONLY(popq(src)) NOT_LP64(popl(src)); }
1609 
1610   void pushoop(jobject obj);
1611   void pushklass(Metadata* obj);
1612 
1613   // sign extend as need a l to ptr sized element
1614   void movl2ptr(Register dst, Address src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(movl(dst, src)); }
1615   void movl2ptr(Register dst, Register src) { LP64_ONLY(movslq(dst, src)) NOT_LP64(if (dst != src) movl(dst, src)); }
1616 
1617   // C2 compiled method's prolog code.
1618   void verified_entry(Compile* C, int sp_inc = 0);
1619 
1620   enum RegState {
1621     reg_readonly,
1622     reg_writable,
1623     reg_written
1624   };
1625 
1626   // Unpack all value type arguments passed as oops
1627   void unpack_value_args(Compile* C, bool receiver_only);
1628   bool move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[], int ret_off);
1629   bool unpack_value_helper(const GrowableArray<SigEntry>* sig, int& sig_index, VMReg from, VMRegPair* regs_to, int& to_index, RegState reg_state[], int ret_off);
1630   void restore_stack(Compile* C);
1631 
1632   // clear memory of size 'cnt' qwords, starting at 'base';
1633   // if 'is_large' is set, do not try to produce short loop
1634   void clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only);
1635 
1636   // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
1637   void xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp);
1638 
1639 #ifdef COMPILER2
1640   void string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
1641                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp);
1642 
1643   // IndexOf strings.
1644   // Small strings are loaded through stack if they cross page boundary.
1645   void string_indexof(Register str1, Register str2,
1646                       Register cnt1, Register cnt2,
1647                       int int_cnt2,  Register result,
1648                       XMMRegister vec, Register tmp,
1649                       int ae);
1650 
1651   // IndexOf for constant substrings with size >= 8 elements
1652   // which don't need to be loaded through stack.
1653   void string_indexofC8(Register str1, Register str2,
1654                       Register cnt1, Register cnt2,
1655                       int int_cnt2,  Register result,
1656                       XMMRegister vec, Register tmp,
1657                       int ae);
1658 
1659     // Smallest code: we don't need to load through stack,
1660     // check string tail.
1661 
1662   // helper function for string_compare
1663   void load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
1664                           Address::ScaleFactor scale, Address::ScaleFactor scale1,
1665                           Address::ScaleFactor scale2, Register index, int ae);
1666   // Compare strings.
1667   void string_compare(Register str1, Register str2,
1668                       Register cnt1, Register cnt2, Register result,
1669                       XMMRegister vec1, int ae);
1670 
1671   // Search for Non-ASCII character (Negative byte value) in a byte array,
1672   // return true if it has any and false otherwise.
1673   void has_negatives(Register ary1, Register len,
1674                      Register result, Register tmp1,
1675                      XMMRegister vec1, XMMRegister vec2);
1676 
1677   // Compare char[] or byte[] arrays.
1678   void arrays_equals(bool is_array_equ, Register ary1, Register ary2,
1679                      Register limit, Register result, Register chr,
1680                      XMMRegister vec1, XMMRegister vec2, bool is_char);
1681 
1682 #endif
1683 
1684   // Fill primitive arrays
1685   void generate_fill(BasicType t, bool aligned,
1686                      Register to, Register value, Register count,
1687                      Register rtmp, XMMRegister xtmp);
1688 
1689   void encode_iso_array(Register src, Register dst, Register len,
1690                         XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1691                         XMMRegister tmp4, Register tmp5, Register result);
1692 
1693 #ifdef _LP64
1694   void add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2);
1695   void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
1696                              Register y, Register y_idx, Register z,
1697                              Register carry, Register product,
1698                              Register idx, Register kdx);
1699   void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
1700                               Register yz_idx, Register idx,
1701                               Register carry, Register product, int offset);
1702   void multiply_128_x_128_bmi2_loop(Register y, Register z,
1703                                     Register carry, Register carry2,
1704                                     Register idx, Register jdx,
1705                                     Register yz_idx1, Register yz_idx2,
1706                                     Register tmp, Register tmp3, Register tmp4);
1707   void multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
1708                                Register yz_idx, Register idx, Register jdx,
1709                                Register carry, Register product,
1710                                Register carry2);
1711   void multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
1712                        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5);
1713   void square_rshift(Register x, Register len, Register z, Register tmp1, Register tmp3,
1714                      Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1715   void multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry,
1716                             Register tmp2);
1717   void multiply_add_64(Register sum, Register op1, Register op2, Register carry,
1718                        Register rdxReg, Register raxReg);
1719   void add_one_64(Register z, Register zlen, Register carry, Register tmp1);
1720   void lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1721                        Register tmp3, Register tmp4);
1722   void square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2,
1723                      Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg);
1724 
1725   void mul_add_128_x_32_loop(Register out, Register in, Register offset, Register len, Register tmp1,
1726                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1727                Register raxReg);
1728   void mul_add(Register out, Register in, Register offset, Register len, Register k, Register tmp1,
1729                Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg,
1730                Register raxReg);
1731   void vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
1732                            Register result, Register tmp1, Register tmp2,
1733                            XMMRegister vec1, XMMRegister vec2, XMMRegister vec3);
1734 #endif
1735 
1736   // CRC32 code for java.util.zip.CRC32::updateBytes() intrinsic.
1737   void update_byte_crc32(Register crc, Register val, Register table);
1738   void kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp);
1739   // CRC32C code for java.util.zip.CRC32C::updateBytes() intrinsic
1740   // Note on a naming convention:
1741   // Prefix w = register only used on a Westmere+ architecture
1742   // Prefix n = register only used on a Nehalem architecture
1743 #ifdef _LP64
1744   void crc32c_ipl_alg4(Register in_out, uint32_t n,
1745                        Register tmp1, Register tmp2, Register tmp3);
1746 #else
1747   void crc32c_ipl_alg4(Register in_out, uint32_t n,
1748                        Register tmp1, Register tmp2, Register tmp3,
1749                        XMMRegister xtmp1, XMMRegister xtmp2);
1750 #endif
1751   void crc32c_pclmulqdq(XMMRegister w_xtmp1,
1752                         Register in_out,
1753                         uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
1754                         XMMRegister w_xtmp2,
1755                         Register tmp1,
1756                         Register n_tmp2, Register n_tmp3);
1757   void crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
1758                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1759                        Register tmp1, Register tmp2,
1760                        Register n_tmp3);
1761   void crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
1762                          Register in_out1, Register in_out2, Register in_out3,
1763                          Register tmp1, Register tmp2, Register tmp3,
1764                          XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1765                          Register tmp4, Register tmp5,
1766                          Register n_tmp6);
1767   void crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
1768                             Register tmp1, Register tmp2, Register tmp3,
1769                             Register tmp4, Register tmp5, Register tmp6,
1770                             XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
1771                             bool is_pclmulqdq_supported);
1772   // Fold 128-bit data chunk
1773   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
1774   void fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf);
1775   // Fold 8-bit data
1776   void fold_8bit_crc32(Register crc, Register table, Register tmp);
1777   void fold_8bit_crc32(XMMRegister crc, Register table, XMMRegister xtmp, Register tmp);
1778   void fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset);
1779 
1780   // Compress char[] array to byte[].
1781   void char_array_compress(Register src, Register dst, Register len,
1782                            XMMRegister tmp1, XMMRegister tmp2, XMMRegister tmp3,
1783                            XMMRegister tmp4, Register tmp5, Register result);
1784 
1785   // Inflate byte[] array to char[].
1786   void byte_array_inflate(Register src, Register dst, Register len,
1787                           XMMRegister tmp1, Register tmp2);
1788 
1789 };
1790 
1791 /**
1792  * class SkipIfEqual:
1793  *
1794  * Instantiating this class will result in assembly code being output that will
1795  * jump around any code emitted between the creation of the instance and it's
1796  * automatic destruction at the end of a scope block, depending on the value of
1797  * the flag passed to the constructor, which will be checked at run-time.
1798  */
1799 class SkipIfEqual {
1800  private:
1801   MacroAssembler* _masm;
1802   Label _label;
1803 
1804  public:
1805    SkipIfEqual(MacroAssembler*, const bool* flag_addr, bool value);
1806    ~SkipIfEqual();
1807 };
1808 
1809 #endif // CPU_X86_MACROASSEMBLER_X86_HPP