1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "asm/assembler.hpp"
  30 #include "asm/assembler.inline.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 
  33 #include "compiler/disassembler.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "nativeInst_aarch64.hpp"
  36 #include "oops/klass.inline.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "opto/compile.hpp"
  39 #include "opto/node.hpp"
  40 #include "runtime/biasedLocking.hpp"
  41 #include "runtime/icache.hpp"
  42 #include "runtime/interfaceSupport.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "runtime/thread.hpp"
  45 
  46 #if INCLUDE_ALL_GCS
  47 #include "gc/g1/g1CollectedHeap.inline.hpp"
  48 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  49 #include "gc/g1/heapRegion.hpp"
  50 #endif
  51 
  52 #ifdef PRODUCT
  53 #define BLOCK_COMMENT(str) /* nothing */
  54 #define STOP(error) stop(error)
  55 #else
  56 #define BLOCK_COMMENT(str) block_comment(str)
  57 #define STOP(error) block_comment(error); stop(error)
  58 #endif
  59 
  60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  61 
  62 // Patch any kind of instruction; there may be several instructions.
  63 // Return the total length (in bytes) of the instructions.
  64 int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
  65   int instructions = 1;
  66   assert((uint64_t)target < (1ul << 48), "48-bit overflow in address constant");
  67   long offset = (target - branch) >> 2;
  68   unsigned insn = *(unsigned*)branch;
  69   if ((Instruction_aarch64::extract(insn, 29, 24) & 0b111011) == 0b011000) {
  70     // Load register (literal)
  71     Instruction_aarch64::spatch(branch, 23, 5, offset);
  72   } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
  73     // Unconditional branch (immediate)
  74     Instruction_aarch64::spatch(branch, 25, 0, offset);
  75   } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
  76     // Conditional branch (immediate)
  77     Instruction_aarch64::spatch(branch, 23, 5, offset);
  78   } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
  79     // Compare & branch (immediate)
  80     Instruction_aarch64::spatch(branch, 23, 5, offset);
  81   } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
  82     // Test & branch (immediate)
  83     Instruction_aarch64::spatch(branch, 18, 5, offset);
  84   } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
  85     // PC-rel. addressing
  86     offset = target-branch;
  87     int shift = Instruction_aarch64::extract(insn, 31, 31);
  88     if (shift) {
  89       u_int64_t dest = (u_int64_t)target;
  90       uint64_t pc_page = (uint64_t)branch >> 12;
  91       uint64_t adr_page = (uint64_t)target >> 12;
  92       unsigned offset_lo = dest & 0xfff;
  93       offset = adr_page - pc_page;
  94 
  95       // We handle 4 types of PC relative addressing
  96       //   1 - adrp    Rx, target_page
  97       //       ldr/str Ry, [Rx, #offset_in_page]
  98       //   2 - adrp    Rx, target_page
  99       //       add     Ry, Rx, #offset_in_page
 100       //   3 - adrp    Rx, target_page (page aligned reloc, offset == 0)
 101       //       movk    Rx, #imm16<<32
 102       //   4 - adrp    Rx, target_page (page aligned reloc, offset == 0)
 103       // In the first 3 cases we must check that Rx is the same in the adrp and the
 104       // subsequent ldr/str, add or movk instruction. Otherwise we could accidentally end
 105       // up treating a type 4 relocation as a type 1, 2 or 3 just because it happened
 106       // to be followed by a random unrelated ldr/str, add or movk instruction.
 107       //
 108       unsigned insn2 = ((unsigned*)branch)[1];
 109       if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
 110                 Instruction_aarch64::extract(insn, 4, 0) ==
 111                         Instruction_aarch64::extract(insn2, 9, 5)) {
 112         // Load/store register (unsigned immediate)
 113         unsigned size = Instruction_aarch64::extract(insn2, 31, 30);
 114         Instruction_aarch64::patch(branch + sizeof (unsigned),
 115                                     21, 10, offset_lo >> size);
 116         guarantee(((dest >> size) << size) == dest, "misaligned target");
 117         instructions = 2;
 118       } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
 119                 Instruction_aarch64::extract(insn, 4, 0) ==
 120                         Instruction_aarch64::extract(insn2, 4, 0)) {
 121         // add (immediate)
 122         Instruction_aarch64::patch(branch + sizeof (unsigned),
 123                                    21, 10, offset_lo);
 124         instructions = 2;
 125       } else if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110 &&
 126                    Instruction_aarch64::extract(insn, 4, 0) ==
 127                      Instruction_aarch64::extract(insn2, 4, 0)) {
 128         // movk #imm16<<32
 129         Instruction_aarch64::patch(branch + 4, 20, 5, (uint64_t)target >> 32);
 130         long dest = ((long)target & 0xffffffffL) | ((long)branch & 0xffff00000000L);
 131         long pc_page = (long)branch >> 12;
 132         long adr_page = (long)dest >> 12;
 133         offset = adr_page - pc_page;
 134         instructions = 2;
 135       }
 136     }
 137     int offset_lo = offset & 3;
 138     offset >>= 2;
 139     Instruction_aarch64::spatch(branch, 23, 5, offset);
 140     Instruction_aarch64::patch(branch, 30, 29, offset_lo);
 141   } else if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010100) {
 142     u_int64_t dest = (u_int64_t)target;
 143     // Move wide constant
 144     assert(nativeInstruction_at(branch+4)->is_movk(), "wrong insns in patch");
 145     assert(nativeInstruction_at(branch+8)->is_movk(), "wrong insns in patch");
 146     Instruction_aarch64::patch(branch, 20, 5, dest & 0xffff);
 147     Instruction_aarch64::patch(branch+4, 20, 5, (dest >>= 16) & 0xffff);
 148     Instruction_aarch64::patch(branch+8, 20, 5, (dest >>= 16) & 0xffff);
 149     assert(target_addr_for_insn(branch) == target, "should be");
 150     instructions = 3;
 151   } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
 152              Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
 153     // nothing to do
 154     assert(target == 0, "did not expect to relocate target for polling page load");
 155   } else {
 156     ShouldNotReachHere();
 157   }
 158   return instructions * NativeInstruction::instruction_size;
 159 }
 160 
 161 int MacroAssembler::patch_oop(address insn_addr, address o) {
 162   int instructions;
 163   unsigned insn = *(unsigned*)insn_addr;
 164   assert(nativeInstruction_at(insn_addr+4)->is_movk(), "wrong insns in patch");
 165 
 166   // OOPs are either narrow (32 bits) or wide (48 bits).  We encode
 167   // narrow OOPs by setting the upper 16 bits in the first
 168   // instruction.
 169   if (Instruction_aarch64::extract(insn, 31, 21) == 0b11010010101) {
 170     // Move narrow OOP
 171     narrowOop n = oopDesc::encode_heap_oop((oop)o);
 172     Instruction_aarch64::patch(insn_addr, 20, 5, n >> 16);
 173     Instruction_aarch64::patch(insn_addr+4, 20, 5, n & 0xffff);
 174     instructions = 2;
 175   } else {
 176     // Move wide OOP
 177     assert(nativeInstruction_at(insn_addr+8)->is_movk(), "wrong insns in patch");
 178     uintptr_t dest = (uintptr_t)o;
 179     Instruction_aarch64::patch(insn_addr, 20, 5, dest & 0xffff);
 180     Instruction_aarch64::patch(insn_addr+4, 20, 5, (dest >>= 16) & 0xffff);
 181     Instruction_aarch64::patch(insn_addr+8, 20, 5, (dest >>= 16) & 0xffff);
 182     instructions = 3;
 183   }
 184   return instructions * NativeInstruction::instruction_size;
 185 }
 186 
 187 address MacroAssembler::target_addr_for_insn(address insn_addr, unsigned insn) {
 188   long offset = 0;
 189   if ((Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000) {
 190     // Load register (literal)
 191     offset = Instruction_aarch64::sextract(insn, 23, 5);
 192     return address(((uint64_t)insn_addr + (offset << 2)));
 193   } else if (Instruction_aarch64::extract(insn, 30, 26) == 0b00101) {
 194     // Unconditional branch (immediate)
 195     offset = Instruction_aarch64::sextract(insn, 25, 0);
 196   } else if (Instruction_aarch64::extract(insn, 31, 25) == 0b0101010) {
 197     // Conditional branch (immediate)
 198     offset = Instruction_aarch64::sextract(insn, 23, 5);
 199   } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011010) {
 200     // Compare & branch (immediate)
 201     offset = Instruction_aarch64::sextract(insn, 23, 5);
 202    } else if (Instruction_aarch64::extract(insn, 30, 25) == 0b011011) {
 203     // Test & branch (immediate)
 204     offset = Instruction_aarch64::sextract(insn, 18, 5);
 205   } else if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
 206     // PC-rel. addressing
 207     offset = Instruction_aarch64::extract(insn, 30, 29);
 208     offset |= Instruction_aarch64::sextract(insn, 23, 5) << 2;
 209     int shift = Instruction_aarch64::extract(insn, 31, 31) ? 12 : 0;
 210     if (shift) {
 211       offset <<= shift;
 212       uint64_t target_page = ((uint64_t)insn_addr) + offset;
 213       target_page &= ((uint64_t)-1) << shift;
 214       // Return the target address for the following sequences
 215       //   1 - adrp    Rx, target_page
 216       //       ldr/str Ry, [Rx, #offset_in_page]
 217       //   2 - adrp    Rx, target_page
 218       //       add     Ry, Rx, #offset_in_page
 219       //   3 - adrp    Rx, target_page (page aligned reloc, offset == 0)
 220       //       movk    Rx, #imm12<<32
 221       //   4 - adrp    Rx, target_page (page aligned reloc, offset == 0)
 222       //
 223       // In the first two cases  we check that the register is the same and
 224       // return the target_page + the offset within the page.
 225       // Otherwise we assume it is a page aligned relocation and return
 226       // the target page only.
 227       //
 228       unsigned insn2 = ((unsigned*)insn_addr)[1];
 229       if (Instruction_aarch64::extract(insn2, 29, 24) == 0b111001 &&
 230                 Instruction_aarch64::extract(insn, 4, 0) ==
 231                         Instruction_aarch64::extract(insn2, 9, 5)) {
 232         // Load/store register (unsigned immediate)
 233         unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
 234         unsigned int size = Instruction_aarch64::extract(insn2, 31, 30);
 235         return address(target_page + (byte_offset << size));
 236       } else if (Instruction_aarch64::extract(insn2, 31, 22) == 0b1001000100 &&
 237                 Instruction_aarch64::extract(insn, 4, 0) ==
 238                         Instruction_aarch64::extract(insn2, 4, 0)) {
 239         // add (immediate)
 240         unsigned int byte_offset = Instruction_aarch64::extract(insn2, 21, 10);
 241         return address(target_page + byte_offset);
 242       } else {
 243         if (Instruction_aarch64::extract(insn2, 31, 21) == 0b11110010110  &&
 244                Instruction_aarch64::extract(insn, 4, 0) ==
 245                  Instruction_aarch64::extract(insn2, 4, 0)) {
 246           target_page = (target_page & 0xffffffff) |
 247                          ((uint64_t)Instruction_aarch64::extract(insn2, 20, 5) << 32);
 248         }
 249         return (address)target_page;
 250       }
 251     } else {
 252       ShouldNotReachHere();
 253     }
 254   } else if (Instruction_aarch64::extract(insn, 31, 23) == 0b110100101) {
 255     u_int32_t *insns = (u_int32_t *)insn_addr;
 256     // Move wide constant: movz, movk, movk.  See movptr().
 257     assert(nativeInstruction_at(insns+1)->is_movk(), "wrong insns in patch");
 258     assert(nativeInstruction_at(insns+2)->is_movk(), "wrong insns in patch");
 259     return address(u_int64_t(Instruction_aarch64::extract(insns[0], 20, 5))
 260                    + (u_int64_t(Instruction_aarch64::extract(insns[1], 20, 5)) << 16)
 261                    + (u_int64_t(Instruction_aarch64::extract(insns[2], 20, 5)) << 32));
 262   } else if (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
 263              Instruction_aarch64::extract(insn, 4, 0) == 0b11111) {
 264     return 0;
 265   } else {
 266     ShouldNotReachHere();
 267   }
 268   return address(((uint64_t)insn_addr + (offset << 2)));
 269 }
 270 
 271 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
 272   dsb(Assembler::SY);
 273 }
 274 
 275 
 276 void MacroAssembler::reset_last_Java_frame(bool clear_fp,
 277                                            bool clear_pc) {
 278   // we must set sp to zero to clear frame
 279   str(zr, Address(rthread, JavaThread::last_Java_sp_offset()));
 280   // must clear fp, so that compiled frames are not confused; it is
 281   // possible that we need it only for debugging
 282   if (clear_fp) {
 283     str(zr, Address(rthread, JavaThread::last_Java_fp_offset()));
 284   }
 285 
 286   if (clear_pc) {
 287     str(zr, Address(rthread, JavaThread::last_Java_pc_offset()));
 288   }
 289 }
 290 
 291 // Calls to C land
 292 //
 293 // When entering C land, the rfp, & resp of the last Java frame have to be recorded
 294 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 295 // has to be reset to 0. This is required to allow proper stack traversal.
 296 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 297                                          Register last_java_fp,
 298                                          Register last_java_pc,
 299                                          Register scratch) {
 300 
 301   if (last_java_pc->is_valid()) {
 302       str(last_java_pc, Address(rthread,
 303                                 JavaThread::frame_anchor_offset()
 304                                 + JavaFrameAnchor::last_Java_pc_offset()));
 305     }
 306 
 307   // determine last_java_sp register
 308   if (last_java_sp == sp) {
 309     mov(scratch, sp);
 310     last_java_sp = scratch;
 311   } else if (!last_java_sp->is_valid()) {
 312     last_java_sp = esp;
 313   }
 314 
 315   str(last_java_sp, Address(rthread, JavaThread::last_Java_sp_offset()));
 316 
 317   // last_java_fp is optional
 318   if (last_java_fp->is_valid()) {
 319     str(last_java_fp, Address(rthread, JavaThread::last_Java_fp_offset()));
 320   }
 321 }
 322 
 323 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 324                                          Register last_java_fp,
 325                                          address  last_java_pc,
 326                                          Register scratch) {
 327   if (last_java_pc != NULL) {
 328     adr(scratch, last_java_pc);
 329   } else {
 330     // FIXME: This is almost never correct.  We should delete all
 331     // cases of set_last_Java_frame with last_java_pc=NULL and use the
 332     // correct return address instead.
 333     adr(scratch, pc());
 334   }
 335 
 336   str(scratch, Address(rthread,
 337                        JavaThread::frame_anchor_offset()
 338                        + JavaFrameAnchor::last_Java_pc_offset()));
 339 
 340   set_last_Java_frame(last_java_sp, last_java_fp, noreg, scratch);
 341 }
 342 
 343 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 344                                          Register last_java_fp,
 345                                          Label &L,
 346                                          Register scratch) {
 347   if (L.is_bound()) {
 348     set_last_Java_frame(last_java_sp, last_java_fp, target(L), scratch);
 349   } else {
 350     InstructionMark im(this);
 351     L.add_patch_at(code(), locator());
 352     set_last_Java_frame(last_java_sp, last_java_fp, (address)NULL, scratch);
 353   }
 354 }
 355 
 356 void MacroAssembler::far_call(Address entry, CodeBuffer *cbuf, Register tmp) {
 357   assert(ReservedCodeCacheSize < 4*G, "branch out of range");
 358   assert(CodeCache::find_blob(entry.target()) != NULL,
 359          "destination of far call not found in code cache");
 360   if (far_branches()) {
 361     unsigned long offset;
 362     // We can use ADRP here because we know that the total size of
 363     // the code cache cannot exceed 2Gb.
 364     adrp(tmp, entry, offset);
 365     add(tmp, tmp, offset);
 366     if (cbuf) cbuf->set_insts_mark();
 367     blr(tmp);
 368   } else {
 369     if (cbuf) cbuf->set_insts_mark();
 370     bl(entry);
 371   }
 372 }
 373 
 374 void MacroAssembler::far_jump(Address entry, CodeBuffer *cbuf, Register tmp) {
 375   assert(ReservedCodeCacheSize < 4*G, "branch out of range");
 376   assert(CodeCache::find_blob(entry.target()) != NULL,
 377          "destination of far call not found in code cache");
 378   if (far_branches()) {
 379     unsigned long offset;
 380     // We can use ADRP here because we know that the total size of
 381     // the code cache cannot exceed 2Gb.
 382     adrp(tmp, entry, offset);
 383     add(tmp, tmp, offset);
 384     if (cbuf) cbuf->set_insts_mark();
 385     br(tmp);
 386   } else {
 387     if (cbuf) cbuf->set_insts_mark();
 388     b(entry);
 389   }
 390 }
 391 
 392 int MacroAssembler::biased_locking_enter(Register lock_reg,
 393                                          Register obj_reg,
 394                                          Register swap_reg,
 395                                          Register tmp_reg,
 396                                          bool swap_reg_contains_mark,
 397                                          Label& done,
 398                                          Label* slow_case,
 399                                          BiasedLockingCounters* counters) {
 400   assert(UseBiasedLocking, "why call this otherwise?");
 401   assert_different_registers(lock_reg, obj_reg, swap_reg);
 402 
 403   if (PrintBiasedLockingStatistics && counters == NULL)
 404     counters = BiasedLocking::counters();
 405 
 406   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg, rscratch1, rscratch2, noreg);
 407   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
 408   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
 409   Address klass_addr     (obj_reg, oopDesc::klass_offset_in_bytes());
 410   Address saved_mark_addr(lock_reg, 0);
 411 
 412   // Biased locking
 413   // See whether the lock is currently biased toward our thread and
 414   // whether the epoch is still valid
 415   // Note that the runtime guarantees sufficient alignment of JavaThread
 416   // pointers to allow age to be placed into low bits
 417   // First check to see whether biasing is even enabled for this object
 418   Label cas_label;
 419   int null_check_offset = -1;
 420   if (!swap_reg_contains_mark) {
 421     null_check_offset = offset();
 422     ldr(swap_reg, mark_addr);
 423   }
 424   andr(tmp_reg, swap_reg, markOopDesc::biased_lock_mask_in_place);
 425   cmp(tmp_reg, markOopDesc::biased_lock_pattern);
 426   br(Assembler::NE, cas_label);
 427   // The bias pattern is present in the object's header. Need to check
 428   // whether the bias owner and the epoch are both still current.
 429   load_prototype_header(tmp_reg, obj_reg);
 430   orr(tmp_reg, tmp_reg, rthread);
 431   eor(tmp_reg, swap_reg, tmp_reg);
 432   andr(tmp_reg, tmp_reg, ~((int) markOopDesc::age_mask_in_place));
 433   if (counters != NULL) {
 434     Label around;
 435     cbnz(tmp_reg, around);
 436     atomic_incw(Address((address)counters->biased_lock_entry_count_addr()), tmp_reg, rscratch1, rscratch2);
 437     b(done);
 438     bind(around);
 439   } else {
 440     cbz(tmp_reg, done);
 441   }
 442 
 443   Label try_revoke_bias;
 444   Label try_rebias;
 445 
 446   // At this point we know that the header has the bias pattern and
 447   // that we are not the bias owner in the current epoch. We need to
 448   // figure out more details about the state of the header in order to
 449   // know what operations can be legally performed on the object's
 450   // header.
 451 
 452   // If the low three bits in the xor result aren't clear, that means
 453   // the prototype header is no longer biased and we have to revoke
 454   // the bias on this object.
 455   andr(rscratch1, tmp_reg, markOopDesc::biased_lock_mask_in_place);
 456   cbnz(rscratch1, try_revoke_bias);
 457 
 458   // Biasing is still enabled for this data type. See whether the
 459   // epoch of the current bias is still valid, meaning that the epoch
 460   // bits of the mark word are equal to the epoch bits of the
 461   // prototype header. (Note that the prototype header's epoch bits
 462   // only change at a safepoint.) If not, attempt to rebias the object
 463   // toward the current thread. Note that we must be absolutely sure
 464   // that the current epoch is invalid in order to do this because
 465   // otherwise the manipulations it performs on the mark word are
 466   // illegal.
 467   andr(rscratch1, tmp_reg, markOopDesc::epoch_mask_in_place);
 468   cbnz(rscratch1, try_rebias);
 469 
 470   // The epoch of the current bias is still valid but we know nothing
 471   // about the owner; it might be set or it might be clear. Try to
 472   // acquire the bias of the object using an atomic operation. If this
 473   // fails we will go in to the runtime to revoke the object's bias.
 474   // Note that we first construct the presumed unbiased header so we
 475   // don't accidentally blow away another thread's valid bias.
 476   {
 477     Label here;
 478     mov(rscratch1, markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
 479     andr(swap_reg, swap_reg, rscratch1);
 480     orr(tmp_reg, swap_reg, rthread);
 481     cmpxchgptr(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
 482     // If the biasing toward our thread failed, this means that
 483     // another thread succeeded in biasing it toward itself and we
 484     // need to revoke that bias. The revocation will occur in the
 485     // interpreter runtime in the slow case.
 486     bind(here);
 487     if (counters != NULL) {
 488       atomic_incw(Address((address)counters->anonymously_biased_lock_entry_count_addr()),
 489                   tmp_reg, rscratch1, rscratch2);
 490     }
 491   }
 492   b(done);
 493 
 494   bind(try_rebias);
 495   // At this point we know the epoch has expired, meaning that the
 496   // current "bias owner", if any, is actually invalid. Under these
 497   // circumstances _only_, we are allowed to use the current header's
 498   // value as the comparison value when doing the cas to acquire the
 499   // bias in the current epoch. In other words, we allow transfer of
 500   // the bias from one thread to another directly in this situation.
 501   //
 502   // FIXME: due to a lack of registers we currently blow away the age
 503   // bits in this situation. Should attempt to preserve them.
 504   {
 505     Label here;
 506     load_prototype_header(tmp_reg, obj_reg);
 507     orr(tmp_reg, rthread, tmp_reg);
 508     cmpxchgptr(swap_reg, tmp_reg, obj_reg, rscratch1, here, slow_case);
 509     // If the biasing toward our thread failed, then another thread
 510     // succeeded in biasing it toward itself and we need to revoke that
 511     // bias. The revocation will occur in the runtime in the slow case.
 512     bind(here);
 513     if (counters != NULL) {
 514       atomic_incw(Address((address)counters->rebiased_lock_entry_count_addr()),
 515                   tmp_reg, rscratch1, rscratch2);
 516     }
 517   }
 518   b(done);
 519 
 520   bind(try_revoke_bias);
 521   // The prototype mark in the klass doesn't have the bias bit set any
 522   // more, indicating that objects of this data type are not supposed
 523   // to be biased any more. We are going to try to reset the mark of
 524   // this object to the prototype value and fall through to the
 525   // CAS-based locking scheme. Note that if our CAS fails, it means
 526   // that another thread raced us for the privilege of revoking the
 527   // bias of this particular object, so it's okay to continue in the
 528   // normal locking code.
 529   //
 530   // FIXME: due to a lack of registers we currently blow away the age
 531   // bits in this situation. Should attempt to preserve them.
 532   {
 533     Label here, nope;
 534     load_prototype_header(tmp_reg, obj_reg);
 535     cmpxchgptr(swap_reg, tmp_reg, obj_reg, rscratch1, here, &nope);
 536     bind(here);
 537 
 538     // Fall through to the normal CAS-based lock, because no matter what
 539     // the result of the above CAS, some thread must have succeeded in
 540     // removing the bias bit from the object's header.
 541     if (counters != NULL) {
 542       atomic_incw(Address((address)counters->revoked_lock_entry_count_addr()), tmp_reg,
 543                   rscratch1, rscratch2);
 544     }
 545     bind(nope);
 546   }
 547 
 548   bind(cas_label);
 549 
 550   return null_check_offset;
 551 }
 552 
 553 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
 554   assert(UseBiasedLocking, "why call this otherwise?");
 555 
 556   // Check for biased locking unlock case, which is a no-op
 557   // Note: we do not have to check the thread ID for two reasons.
 558   // First, the interpreter checks for IllegalMonitorStateException at
 559   // a higher level. Second, if the bias was revoked while we held the
 560   // lock, the object could not be rebiased toward another thread, so
 561   // the bias bit would be clear.
 562   ldr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
 563   andr(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);
 564   cmp(temp_reg, markOopDesc::biased_lock_pattern);
 565   br(Assembler::EQ, done);
 566 }
 567 
 568 
 569 // added to make this compile
 570 
 571 REGISTER_DEFINITION(Register, noreg);
 572 
 573 static void pass_arg0(MacroAssembler* masm, Register arg) {
 574   if (c_rarg0 != arg ) {
 575     masm->mov(c_rarg0, arg);
 576   }
 577 }
 578 
 579 static void pass_arg1(MacroAssembler* masm, Register arg) {
 580   if (c_rarg1 != arg ) {
 581     masm->mov(c_rarg1, arg);
 582   }
 583 }
 584 
 585 static void pass_arg2(MacroAssembler* masm, Register arg) {
 586   if (c_rarg2 != arg ) {
 587     masm->mov(c_rarg2, arg);
 588   }
 589 }
 590 
 591 static void pass_arg3(MacroAssembler* masm, Register arg) {
 592   if (c_rarg3 != arg ) {
 593     masm->mov(c_rarg3, arg);
 594   }
 595 }
 596 
 597 void MacroAssembler::call_VM_base(Register oop_result,
 598                                   Register java_thread,
 599                                   Register last_java_sp,
 600                                   address  entry_point,
 601                                   int      number_of_arguments,
 602                                   bool     check_exceptions) {
 603    // determine java_thread register
 604   if (!java_thread->is_valid()) {
 605     java_thread = rthread;
 606   }
 607 
 608   // determine last_java_sp register
 609   if (!last_java_sp->is_valid()) {
 610     last_java_sp = esp;
 611   }
 612 
 613   // debugging support
 614   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 615   assert(java_thread == rthread, "unexpected register");
 616 #ifdef ASSERT
 617   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 618   // if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");
 619 #endif // ASSERT
 620 
 621   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 622   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 623 
 624   // push java thread (becomes first argument of C function)
 625 
 626   mov(c_rarg0, java_thread);
 627 
 628   // set last Java frame before call
 629   assert(last_java_sp != rfp, "can't use rfp");
 630 
 631   Label l;
 632   set_last_Java_frame(last_java_sp, rfp, l, rscratch1);
 633 
 634   // do the call, remove parameters
 635   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l);
 636 
 637   // reset last Java frame
 638   // Only interpreter should have to clear fp
 639   reset_last_Java_frame(true, true);
 640 
 641    // C++ interp handles this in the interpreter
 642   check_and_handle_popframe(java_thread);
 643   check_and_handle_earlyret(java_thread);
 644 
 645   if (check_exceptions) {
 646     // check for pending exceptions (java_thread is set upon return)
 647     ldr(rscratch1, Address(java_thread, in_bytes(Thread::pending_exception_offset())));
 648     Label ok;
 649     cbz(rscratch1, ok);
 650     lea(rscratch1, RuntimeAddress(StubRoutines::forward_exception_entry()));
 651     br(rscratch1);
 652     bind(ok);
 653   }
 654 
 655   // get oop result if there is one and reset the value in the thread
 656   if (oop_result->is_valid()) {
 657     get_vm_result(oop_result, java_thread);
 658   }
 659 }
 660 
 661 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 662   call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions);
 663 }
 664 
 665 // Maybe emit a call via a trampoline.  If the code cache is small
 666 // trampolines won't be emitted.
 667 
 668 address MacroAssembler::trampoline_call(Address entry, CodeBuffer *cbuf) {
 669   assert(entry.rspec().type() == relocInfo::runtime_call_type
 670          || entry.rspec().type() == relocInfo::opt_virtual_call_type
 671          || entry.rspec().type() == relocInfo::static_call_type
 672          || entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type");
 673 
 674   unsigned int start_offset = offset();
 675   if (far_branches() && !Compile::current()->in_scratch_emit_size()) {
 676     address stub = emit_trampoline_stub(start_offset, entry.target());
 677     if (stub == NULL) {
 678       return NULL; // CodeCache is full
 679     }
 680   }
 681 
 682   if (cbuf) cbuf->set_insts_mark();
 683   relocate(entry.rspec());
 684   if (!far_branches()) {
 685     bl(entry.target());
 686   } else {
 687     bl(pc());
 688   }
 689   // just need to return a non-null address
 690   return pc();
 691 }
 692 
 693 
 694 // Emit a trampoline stub for a call to a target which is too far away.
 695 //
 696 // code sequences:
 697 //
 698 // call-site:
 699 //   branch-and-link to <destination> or <trampoline stub>
 700 //
 701 // Related trampoline stub for this call site in the stub section:
 702 //   load the call target from the constant pool
 703 //   branch (LR still points to the call site above)
 704 
 705 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset,
 706                                              address dest) {
 707   address stub = start_a_stub(Compile::MAX_stubs_size/2);
 708   if (stub == NULL) {
 709     return NULL;  // CodeBuffer::expand failed
 710   }
 711 
 712   // Create a trampoline stub relocation which relates this trampoline stub
 713   // with the call instruction at insts_call_instruction_offset in the
 714   // instructions code-section.
 715   align(wordSize);
 716   relocate(trampoline_stub_Relocation::spec(code()->insts()->start()
 717                                             + insts_call_instruction_offset));
 718   const int stub_start_offset = offset();
 719 
 720   // Now, create the trampoline stub's code:
 721   // - load the call
 722   // - call
 723   Label target;
 724   ldr(rscratch1, target);
 725   br(rscratch1);
 726   bind(target);
 727   assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset,
 728          "should be");
 729   emit_int64((int64_t)dest);
 730 
 731   const address stub_start_addr = addr_at(stub_start_offset);
 732 
 733   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
 734 
 735   end_a_stub();
 736   return stub;
 737 }
 738 
 739 address MacroAssembler::ic_call(address entry, jint method_index) {
 740   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
 741   // address const_ptr = long_constant((jlong)Universe::non_oop_word());
 742   // unsigned long offset;
 743   // ldr_constant(rscratch2, const_ptr);
 744   movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
 745   return trampoline_call(Address(entry, rh));
 746 }
 747 
 748 // Implementation of call_VM versions
 749 
 750 void MacroAssembler::call_VM(Register oop_result,
 751                              address entry_point,
 752                              bool check_exceptions) {
 753   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 754 }
 755 
 756 void MacroAssembler::call_VM(Register oop_result,
 757                              address entry_point,
 758                              Register arg_1,
 759                              bool check_exceptions) {
 760   pass_arg1(this, arg_1);
 761   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 762 }
 763 
 764 void MacroAssembler::call_VM(Register oop_result,
 765                              address entry_point,
 766                              Register arg_1,
 767                              Register arg_2,
 768                              bool check_exceptions) {
 769   assert(arg_1 != c_rarg2, "smashed arg");
 770   pass_arg2(this, arg_2);
 771   pass_arg1(this, arg_1);
 772   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 773 }
 774 
 775 void MacroAssembler::call_VM(Register oop_result,
 776                              address entry_point,
 777                              Register arg_1,
 778                              Register arg_2,
 779                              Register arg_3,
 780                              bool check_exceptions) {
 781   assert(arg_1 != c_rarg3, "smashed arg");
 782   assert(arg_2 != c_rarg3, "smashed arg");
 783   pass_arg3(this, arg_3);
 784 
 785   assert(arg_1 != c_rarg2, "smashed arg");
 786   pass_arg2(this, arg_2);
 787 
 788   pass_arg1(this, arg_1);
 789   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 790 }
 791 
 792 void MacroAssembler::call_VM(Register oop_result,
 793                              Register last_java_sp,
 794                              address entry_point,
 795                              int number_of_arguments,
 796                              bool check_exceptions) {
 797   call_VM_base(oop_result, rthread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 798 }
 799 
 800 void MacroAssembler::call_VM(Register oop_result,
 801                              Register last_java_sp,
 802                              address entry_point,
 803                              Register arg_1,
 804                              bool check_exceptions) {
 805   pass_arg1(this, arg_1);
 806   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 807 }
 808 
 809 void MacroAssembler::call_VM(Register oop_result,
 810                              Register last_java_sp,
 811                              address entry_point,
 812                              Register arg_1,
 813                              Register arg_2,
 814                              bool check_exceptions) {
 815 
 816   assert(arg_1 != c_rarg2, "smashed arg");
 817   pass_arg2(this, arg_2);
 818   pass_arg1(this, arg_1);
 819   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 820 }
 821 
 822 void MacroAssembler::call_VM(Register oop_result,
 823                              Register last_java_sp,
 824                              address entry_point,
 825                              Register arg_1,
 826                              Register arg_2,
 827                              Register arg_3,
 828                              bool check_exceptions) {
 829   assert(arg_1 != c_rarg3, "smashed arg");
 830   assert(arg_2 != c_rarg3, "smashed arg");
 831   pass_arg3(this, arg_3);
 832   assert(arg_1 != c_rarg2, "smashed arg");
 833   pass_arg2(this, arg_2);
 834   pass_arg1(this, arg_1);
 835   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 836 }
 837 
 838 
 839 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 840   ldr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 841   str(zr, Address(java_thread, JavaThread::vm_result_offset()));
 842   verify_oop(oop_result, "broken oop in call_VM_base");
 843 }
 844 
 845 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 846   ldr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 847   str(zr, Address(java_thread, JavaThread::vm_result_2_offset()));
 848 }
 849 
 850 void MacroAssembler::align(int modulus) {
 851   while (offset() % modulus != 0) nop();
 852 }
 853 
 854 // these are no-ops overridden by InterpreterMacroAssembler
 855 
 856 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
 857 
 858 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
 859 
 860 
 861 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
 862                                                       Register tmp,
 863                                                       int offset) {
 864   intptr_t value = *delayed_value_addr;
 865   if (value != 0)
 866     return RegisterOrConstant(value + offset);
 867 
 868   // load indirectly to solve generation ordering problem
 869   ldr(tmp, ExternalAddress((address) delayed_value_addr));
 870 
 871   if (offset != 0)
 872     add(tmp, tmp, offset);
 873 
 874   return RegisterOrConstant(tmp);
 875 }
 876 
 877 
 878 void MacroAssembler:: notify(int type) {
 879   if (type == bytecode_start) {
 880     // set_last_Java_frame(esp, rfp, (address)NULL);
 881     Assembler:: notify(type);
 882     // reset_last_Java_frame(true, false);
 883   }
 884   else
 885     Assembler:: notify(type);
 886 }
 887 
 888 // Look up the method for a megamorphic invokeinterface call.
 889 // The target method is determined by <intf_klass, itable_index>.
 890 // The receiver klass is in recv_klass.
 891 // On success, the result will be in method_result, and execution falls through.
 892 // On failure, execution transfers to the given label.
 893 void MacroAssembler::lookup_interface_method(Register recv_klass,
 894                                              Register intf_klass,
 895                                              RegisterOrConstant itable_index,
 896                                              Register method_result,
 897                                              Register scan_temp,
 898                                              Label& L_no_such_interface) {
 899   assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
 900   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 901          "caller must use same register for non-constant itable index as for method");
 902 
 903   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
 904   int vtable_base = in_bytes(Klass::vtable_start_offset());
 905   int itentry_off = itableMethodEntry::method_offset_in_bytes();
 906   int scan_step   = itableOffsetEntry::size() * wordSize;
 907   int vte_size    = vtableEntry::size_in_bytes();
 908   assert(vte_size == wordSize, "else adjust times_vte_scale");
 909 
 910   ldrw(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
 911 
 912   // %%% Could store the aligned, prescaled offset in the klassoop.
 913   // lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 914   lea(scan_temp, Address(recv_klass, scan_temp, Address::lsl(3)));
 915   add(scan_temp, scan_temp, vtable_base);
 916 
 917   // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 918   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 919   // lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
 920   lea(recv_klass, Address(recv_klass, itable_index, Address::lsl(3)));
 921   if (itentry_off)
 922     add(recv_klass, recv_klass, itentry_off);
 923 
 924   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
 925   //   if (scan->interface() == intf) {
 926   //     result = (klass + scan->offset() + itable_index);
 927   //   }
 928   // }
 929   Label search, found_method;
 930 
 931   for (int peel = 1; peel >= 0; peel--) {
 932     ldr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
 933     cmp(intf_klass, method_result);
 934 
 935     if (peel) {
 936       br(Assembler::EQ, found_method);
 937     } else {
 938       br(Assembler::NE, search);
 939       // (invert the test to fall through to found_method...)
 940     }
 941 
 942     if (!peel)  break;
 943 
 944     bind(search);
 945 
 946     // Check that the previous entry is non-null.  A null entry means that
 947     // the receiver class doesn't implement the interface, and wasn't the
 948     // same as when the caller was compiled.
 949     cbz(method_result, L_no_such_interface);
 950     add(scan_temp, scan_temp, scan_step);
 951   }
 952 
 953   bind(found_method);
 954 
 955   // Got a hit.
 956   ldr(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
 957   ldr(method_result, Address(recv_klass, scan_temp));
 958 }
 959 
 960 // virtual method calling
 961 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 962                                            RegisterOrConstant vtable_index,
 963                                            Register method_result) {
 964   const int base = in_bytes(Klass::vtable_start_offset());
 965   assert(vtableEntry::size() * wordSize == 8,
 966          "adjust the scaling in the code below");
 967   int vtable_offset_in_bytes = base + vtableEntry::method_offset_in_bytes();
 968 
 969   if (vtable_index.is_register()) {
 970     lea(method_result, Address(recv_klass,
 971                                vtable_index.as_register(),
 972                                Address::lsl(LogBytesPerWord)));
 973     ldr(method_result, Address(method_result, vtable_offset_in_bytes));
 974   } else {
 975     vtable_offset_in_bytes += vtable_index.as_constant() * wordSize;
 976     ldr(method_result, Address(recv_klass, vtable_offset_in_bytes));
 977   }
 978 }
 979 
 980 void MacroAssembler::check_klass_subtype(Register sub_klass,
 981                            Register super_klass,
 982                            Register temp_reg,
 983                            Label& L_success) {
 984   Label L_failure;
 985   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
 986   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
 987   bind(L_failure);
 988 }
 989 
 990 
 991 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
 992                                                    Register super_klass,
 993                                                    Register temp_reg,
 994                                                    Label* L_success,
 995                                                    Label* L_failure,
 996                                                    Label* L_slow_path,
 997                                         RegisterOrConstant super_check_offset) {
 998   assert_different_registers(sub_klass, super_klass, temp_reg);
 999   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1000   if (super_check_offset.is_register()) {
1001     assert_different_registers(sub_klass, super_klass,
1002                                super_check_offset.as_register());
1003   } else if (must_load_sco) {
1004     assert(temp_reg != noreg, "supply either a temp or a register offset");
1005   }
1006 
1007   Label L_fallthrough;
1008   int label_nulls = 0;
1009   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
1010   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
1011   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
1012   assert(label_nulls <= 1, "at most one NULL in the batch");
1013 
1014   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1015   int sco_offset = in_bytes(Klass::super_check_offset_offset());
1016   Address super_check_offset_addr(super_klass, sco_offset);
1017 
1018   // Hacked jmp, which may only be used just before L_fallthrough.
1019 #define final_jmp(label)                                                \
1020   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
1021   else                            b(label)                /*omit semi*/
1022 
1023   // If the pointers are equal, we are done (e.g., String[] elements).
1024   // This self-check enables sharing of secondary supertype arrays among
1025   // non-primary types such as array-of-interface.  Otherwise, each such
1026   // type would need its own customized SSA.
1027   // We move this check to the front of the fast path because many
1028   // type checks are in fact trivially successful in this manner,
1029   // so we get a nicely predicted branch right at the start of the check.
1030   cmp(sub_klass, super_klass);
1031   br(Assembler::EQ, *L_success);
1032 
1033   // Check the supertype display:
1034   if (must_load_sco) {
1035     ldrw(temp_reg, super_check_offset_addr);
1036     super_check_offset = RegisterOrConstant(temp_reg);
1037   }
1038   Address super_check_addr(sub_klass, super_check_offset);
1039   ldr(rscratch1, super_check_addr);
1040   cmp(super_klass, rscratch1); // load displayed supertype
1041 
1042   // This check has worked decisively for primary supers.
1043   // Secondary supers are sought in the super_cache ('super_cache_addr').
1044   // (Secondary supers are interfaces and very deeply nested subtypes.)
1045   // This works in the same check above because of a tricky aliasing
1046   // between the super_cache and the primary super display elements.
1047   // (The 'super_check_addr' can address either, as the case requires.)
1048   // Note that the cache is updated below if it does not help us find
1049   // what we need immediately.
1050   // So if it was a primary super, we can just fail immediately.
1051   // Otherwise, it's the slow path for us (no success at this point).
1052 
1053   if (super_check_offset.is_register()) {
1054     br(Assembler::EQ, *L_success);
1055     cmp(super_check_offset.as_register(), sc_offset);
1056     if (L_failure == &L_fallthrough) {
1057       br(Assembler::EQ, *L_slow_path);
1058     } else {
1059       br(Assembler::NE, *L_failure);
1060       final_jmp(*L_slow_path);
1061     }
1062   } else if (super_check_offset.as_constant() == sc_offset) {
1063     // Need a slow path; fast failure is impossible.
1064     if (L_slow_path == &L_fallthrough) {
1065       br(Assembler::EQ, *L_success);
1066     } else {
1067       br(Assembler::NE, *L_slow_path);
1068       final_jmp(*L_success);
1069     }
1070   } else {
1071     // No slow path; it's a fast decision.
1072     if (L_failure == &L_fallthrough) {
1073       br(Assembler::EQ, *L_success);
1074     } else {
1075       br(Assembler::NE, *L_failure);
1076       final_jmp(*L_success);
1077     }
1078   }
1079 
1080   bind(L_fallthrough);
1081 
1082 #undef final_jmp
1083 }
1084 
1085 // These two are taken from x86, but they look generally useful
1086 
1087 // scans count pointer sized words at [addr] for occurence of value,
1088 // generic
1089 void MacroAssembler::repne_scan(Register addr, Register value, Register count,
1090                                 Register scratch) {
1091   Label Lloop, Lexit;
1092   cbz(count, Lexit);
1093   bind(Lloop);
1094   ldr(scratch, post(addr, wordSize));
1095   cmp(value, scratch);
1096   br(EQ, Lexit);
1097   sub(count, count, 1);
1098   cbnz(count, Lloop);
1099   bind(Lexit);
1100 }
1101 
1102 // scans count 4 byte words at [addr] for occurence of value,
1103 // generic
1104 void MacroAssembler::repne_scanw(Register addr, Register value, Register count,
1105                                 Register scratch) {
1106   Label Lloop, Lexit;
1107   cbz(count, Lexit);
1108   bind(Lloop);
1109   ldrw(scratch, post(addr, wordSize));
1110   cmpw(value, scratch);
1111   br(EQ, Lexit);
1112   sub(count, count, 1);
1113   cbnz(count, Lloop);
1114   bind(Lexit);
1115 }
1116 
1117 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
1118                                                    Register super_klass,
1119                                                    Register temp_reg,
1120                                                    Register temp2_reg,
1121                                                    Label* L_success,
1122                                                    Label* L_failure,
1123                                                    bool set_cond_codes) {
1124   assert_different_registers(sub_klass, super_klass, temp_reg);
1125   if (temp2_reg != noreg)
1126     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg, rscratch1);
1127 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
1128 
1129   Label L_fallthrough;
1130   int label_nulls = 0;
1131   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
1132   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
1133   assert(label_nulls <= 1, "at most one NULL in the batch");
1134 
1135   // a couple of useful fields in sub_klass:
1136   int ss_offset = in_bytes(Klass::secondary_supers_offset());
1137   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1138   Address secondary_supers_addr(sub_klass, ss_offset);
1139   Address super_cache_addr(     sub_klass, sc_offset);
1140 
1141   BLOCK_COMMENT("check_klass_subtype_slow_path");
1142 
1143   // Do a linear scan of the secondary super-klass chain.
1144   // This code is rarely used, so simplicity is a virtue here.
1145   // The repne_scan instruction uses fixed registers, which we must spill.
1146   // Don't worry too much about pre-existing connections with the input regs.
1147 
1148   assert(sub_klass != r0, "killed reg"); // killed by mov(r0, super)
1149   assert(sub_klass != r2, "killed reg"); // killed by lea(r2, &pst_counter)
1150 
1151   // Get super_klass value into r0 (even if it was in r5 or r2).
1152   RegSet pushed_registers;
1153   if (!IS_A_TEMP(r2))    pushed_registers += r2;
1154   if (!IS_A_TEMP(r5))    pushed_registers += r5;
1155 
1156   if (super_klass != r0 || UseCompressedOops) {
1157     if (!IS_A_TEMP(r0))   pushed_registers += r0;
1158   }
1159 
1160   push(pushed_registers, sp);
1161 
1162 #ifndef PRODUCT
1163   mov(rscratch2, (address)&SharedRuntime::_partial_subtype_ctr);
1164   Address pst_counter_addr(rscratch2);
1165   ldr(rscratch1, pst_counter_addr);
1166   add(rscratch1, rscratch1, 1);
1167   str(rscratch1, pst_counter_addr);
1168 #endif //PRODUCT
1169 
1170   // We will consult the secondary-super array.
1171   ldr(r5, secondary_supers_addr);
1172   // Load the array length.
1173   ldrw(r2, Address(r5, Array<Klass*>::length_offset_in_bytes()));
1174   // Skip to start of data.
1175   add(r5, r5, Array<Klass*>::base_offset_in_bytes());
1176 
1177   cmp(sp, zr); // Clear Z flag; SP is never zero
1178   // Scan R2 words at [R5] for an occurrence of R0.
1179   // Set NZ/Z based on last compare.
1180   repne_scan(r5, r0, r2, rscratch1);
1181 
1182   // Unspill the temp. registers:
1183   pop(pushed_registers, sp);
1184 
1185   br(Assembler::NE, *L_failure);
1186 
1187   // Success.  Cache the super we found and proceed in triumph.
1188   str(super_klass, super_cache_addr);
1189 
1190   if (L_success != &L_fallthrough) {
1191     b(*L_success);
1192   }
1193 
1194 #undef IS_A_TEMP
1195 
1196   bind(L_fallthrough);
1197 }
1198 
1199 
1200 void MacroAssembler::verify_oop(Register reg, const char* s) {
1201   if (!VerifyOops) return;
1202 
1203   // Pass register number to verify_oop_subroutine
1204   const char* b = NULL;
1205   {
1206     ResourceMark rm;
1207     stringStream ss;
1208     ss.print("verify_oop: %s: %s", reg->name(), s);
1209     b = code_string(ss.as_string());
1210   }
1211   BLOCK_COMMENT("verify_oop {");
1212 
1213   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1214   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1215 
1216   mov(r0, reg);
1217   mov(rscratch1, (address)b);
1218 
1219   // call indirectly to solve generation ordering problem
1220   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1221   ldr(rscratch2, Address(rscratch2));
1222   blr(rscratch2);
1223 
1224   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1225   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1226 
1227   BLOCK_COMMENT("} verify_oop");
1228 }
1229 
1230 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
1231   if (!VerifyOops) return;
1232 
1233   const char* b = NULL;
1234   {
1235     ResourceMark rm;
1236     stringStream ss;
1237     ss.print("verify_oop_addr: %s", s);
1238     b = code_string(ss.as_string());
1239   }
1240   BLOCK_COMMENT("verify_oop_addr {");
1241 
1242   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1243   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1244 
1245   // addr may contain sp so we will have to adjust it based on the
1246   // pushes that we just did.
1247   if (addr.uses(sp)) {
1248     lea(r0, addr);
1249     ldr(r0, Address(r0, 4 * wordSize));
1250   } else {
1251     ldr(r0, addr);
1252   }
1253   mov(rscratch1, (address)b);
1254 
1255   // call indirectly to solve generation ordering problem
1256   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1257   ldr(rscratch2, Address(rscratch2));
1258   blr(rscratch2);
1259 
1260   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1261   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1262 
1263   BLOCK_COMMENT("} verify_oop_addr");
1264 }
1265 
1266 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
1267                                          int extra_slot_offset) {
1268   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
1269   int stackElementSize = Interpreter::stackElementSize;
1270   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
1271 #ifdef ASSERT
1272   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
1273   assert(offset1 - offset == stackElementSize, "correct arithmetic");
1274 #endif
1275   if (arg_slot.is_constant()) {
1276     return Address(esp, arg_slot.as_constant() * stackElementSize
1277                    + offset);
1278   } else {
1279     add(rscratch1, esp, arg_slot.as_register(),
1280         ext::uxtx, exact_log2(stackElementSize));
1281     return Address(rscratch1, offset);
1282   }
1283 }
1284 
1285 void MacroAssembler::call_VM_leaf_base(address entry_point,
1286                                        int number_of_arguments,
1287                                        Label *retaddr) {
1288   call_VM_leaf_base1(entry_point, number_of_arguments, 0, ret_type_integral, retaddr);
1289 }
1290 
1291 void MacroAssembler::call_VM_leaf_base1(address entry_point,
1292                                         int number_of_gp_arguments,
1293                                         int number_of_fp_arguments,
1294                                         ret_type type,
1295                                         Label *retaddr) {
1296   Label E, L;
1297 
1298   stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1299 
1300   // We add 1 to number_of_arguments because the thread in arg0 is
1301   // not counted
1302   mov(rscratch1, entry_point);
1303   blrt(rscratch1, number_of_gp_arguments + 1, number_of_fp_arguments, type);
1304   if (retaddr)
1305     bind(*retaddr);
1306 
1307   ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1308   maybe_isb();
1309 }
1310 
1311 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1312   call_VM_leaf_base(entry_point, number_of_arguments);
1313 }
1314 
1315 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1316   pass_arg0(this, arg_0);
1317   call_VM_leaf_base(entry_point, 1);
1318 }
1319 
1320 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1321   pass_arg0(this, arg_0);
1322   pass_arg1(this, arg_1);
1323   call_VM_leaf_base(entry_point, 2);
1324 }
1325 
1326 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1327                                   Register arg_1, Register arg_2) {
1328   pass_arg0(this, arg_0);
1329   pass_arg1(this, arg_1);
1330   pass_arg2(this, arg_2);
1331   call_VM_leaf_base(entry_point, 3);
1332 }
1333 
1334 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1335   pass_arg0(this, arg_0);
1336   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1337 }
1338 
1339 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1340 
1341   assert(arg_0 != c_rarg1, "smashed arg");
1342   pass_arg1(this, arg_1);
1343   pass_arg0(this, arg_0);
1344   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1345 }
1346 
1347 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1348   assert(arg_0 != c_rarg2, "smashed arg");
1349   assert(arg_1 != c_rarg2, "smashed arg");
1350   pass_arg2(this, arg_2);
1351   assert(arg_0 != c_rarg1, "smashed arg");
1352   pass_arg1(this, arg_1);
1353   pass_arg0(this, arg_0);
1354   MacroAssembler::call_VM_leaf_base(entry_point, 3);
1355 }
1356 
1357 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
1358   assert(arg_0 != c_rarg3, "smashed arg");
1359   assert(arg_1 != c_rarg3, "smashed arg");
1360   assert(arg_2 != c_rarg3, "smashed arg");
1361   pass_arg3(this, arg_3);
1362   assert(arg_0 != c_rarg2, "smashed arg");
1363   assert(arg_1 != c_rarg2, "smashed arg");
1364   pass_arg2(this, arg_2);
1365   assert(arg_0 != c_rarg1, "smashed arg");
1366   pass_arg1(this, arg_1);
1367   pass_arg0(this, arg_0);
1368   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1369 }
1370 
1371 void MacroAssembler::null_check(Register reg, int offset) {
1372   if (needs_explicit_null_check(offset)) {
1373     // provoke OS NULL exception if reg = NULL by
1374     // accessing M[reg] w/o changing any registers
1375     // NOTE: this is plenty to provoke a segv
1376     ldr(zr, Address(reg));
1377   } else {
1378     // nothing to do, (later) access of M[reg + offset]
1379     // will provoke OS NULL exception if reg = NULL
1380   }
1381 }
1382 
1383 // MacroAssembler protected routines needed to implement
1384 // public methods
1385 
1386 void MacroAssembler::mov(Register r, Address dest) {
1387   code_section()->relocate(pc(), dest.rspec());
1388   u_int64_t imm64 = (u_int64_t)dest.target();
1389   movptr(r, imm64);
1390 }
1391 
1392 // Move a constant pointer into r.  In AArch64 mode the virtual
1393 // address space is 48 bits in size, so we only need three
1394 // instructions to create a patchable instruction sequence that can
1395 // reach anywhere.
1396 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1397 #ifndef PRODUCT
1398   {
1399     char buffer[64];
1400     snprintf(buffer, sizeof(buffer), "0x%"PRIX64, imm64);
1401     block_comment(buffer);
1402   }
1403 #endif
1404   assert(imm64 < (1ul << 48), "48-bit overflow in address constant");
1405   movz(r, imm64 & 0xffff);
1406   imm64 >>= 16;
1407   movk(r, imm64 & 0xffff, 16);
1408   imm64 >>= 16;
1409   movk(r, imm64 & 0xffff, 32);
1410 }
1411 
1412 // Macro to mov replicated immediate to vector register.
1413 //  Vd will get the following values for different arrangements in T
1414 //   imm32 == hex 000000gh  T8B:  Vd = ghghghghghghghgh
1415 //   imm32 == hex 000000gh  T16B: Vd = ghghghghghghghghghghghghghghghgh
1416 //   imm32 == hex 0000efgh  T4H:  Vd = efghefghefghefgh
1417 //   imm32 == hex 0000efgh  T8H:  Vd = efghefghefghefghefghefghefghefgh
1418 //   imm32 == hex abcdefgh  T2S:  Vd = abcdefghabcdefgh
1419 //   imm32 == hex abcdefgh  T4S:  Vd = abcdefghabcdefghabcdefghabcdefgh
1420 //   T1D/T2D: invalid
1421 void MacroAssembler::mov(FloatRegister Vd, SIMD_Arrangement T, u_int32_t imm32) {
1422   assert(T != T1D && T != T2D, "invalid arrangement");
1423   if (T == T8B || T == T16B) {
1424     assert((imm32 & ~0xff) == 0, "extraneous bits in unsigned imm32 (T8B/T16B)");
1425     movi(Vd, T, imm32 & 0xff, 0);
1426     return;
1427   }
1428   u_int32_t nimm32 = ~imm32;
1429   if (T == T4H || T == T8H) {
1430     assert((imm32  & ~0xffff) == 0, "extraneous bits in unsigned imm32 (T4H/T8H)");
1431     imm32 &= 0xffff;
1432     nimm32 &= 0xffff;
1433   }
1434   u_int32_t x = imm32;
1435   int movi_cnt = 0;
1436   int movn_cnt = 0;
1437   while (x) { if (x & 0xff) movi_cnt++; x >>= 8; }
1438   x = nimm32;
1439   while (x) { if (x & 0xff) movn_cnt++; x >>= 8; }
1440   if (movn_cnt < movi_cnt) imm32 = nimm32;
1441   unsigned lsl = 0;
1442   while (imm32 && (imm32 & 0xff) == 0) { lsl += 8; imm32 >>= 8; }
1443   if (movn_cnt < movi_cnt)
1444     mvni(Vd, T, imm32 & 0xff, lsl);
1445   else
1446     movi(Vd, T, imm32 & 0xff, lsl);
1447   imm32 >>= 8; lsl += 8;
1448   while (imm32) {
1449     while ((imm32 & 0xff) == 0) { lsl += 8; imm32 >>= 8; }
1450     if (movn_cnt < movi_cnt)
1451       bici(Vd, T, imm32 & 0xff, lsl);
1452     else
1453       orri(Vd, T, imm32 & 0xff, lsl);
1454     lsl += 8; imm32 >>= 8;
1455   }
1456 }
1457 
1458 void MacroAssembler::mov_immediate64(Register dst, u_int64_t imm64)
1459 {
1460 #ifndef PRODUCT
1461   {
1462     char buffer[64];
1463     snprintf(buffer, sizeof(buffer), "0x%"PRIX64, imm64);
1464     block_comment(buffer);
1465   }
1466 #endif
1467   if (operand_valid_for_logical_immediate(false, imm64)) {
1468     orr(dst, zr, imm64);
1469   } else {
1470     // we can use a combination of MOVZ or MOVN with
1471     // MOVK to build up the constant
1472     u_int64_t imm_h[4];
1473     int zero_count = 0;
1474     int neg_count = 0;
1475     int i;
1476     for (i = 0; i < 4; i++) {
1477       imm_h[i] = ((imm64 >> (i * 16)) & 0xffffL);
1478       if (imm_h[i] == 0) {
1479         zero_count++;
1480       } else if (imm_h[i] == 0xffffL) {
1481         neg_count++;
1482       }
1483     }
1484     if (zero_count == 4) {
1485       // one MOVZ will do
1486       movz(dst, 0);
1487     } else if (neg_count == 4) {
1488       // one MOVN will do
1489       movn(dst, 0);
1490     } else if (zero_count == 3) {
1491       for (i = 0; i < 4; i++) {
1492         if (imm_h[i] != 0L) {
1493           movz(dst, (u_int32_t)imm_h[i], (i << 4));
1494           break;
1495         }
1496       }
1497     } else if (neg_count == 3) {
1498       // one MOVN will do
1499       for (int i = 0; i < 4; i++) {
1500         if (imm_h[i] != 0xffffL) {
1501           movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1502           break;
1503         }
1504       }
1505     } else if (zero_count == 2) {
1506       // one MOVZ and one MOVK will do
1507       for (i = 0; i < 3; i++) {
1508         if (imm_h[i] != 0L) {
1509           movz(dst, (u_int32_t)imm_h[i], (i << 4));
1510           i++;
1511           break;
1512         }
1513       }
1514       for (;i < 4; i++) {
1515         if (imm_h[i] != 0L) {
1516           movk(dst, (u_int32_t)imm_h[i], (i << 4));
1517         }
1518       }
1519     } else if (neg_count == 2) {
1520       // one MOVN and one MOVK will do
1521       for (i = 0; i < 4; i++) {
1522         if (imm_h[i] != 0xffffL) {
1523           movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1524           i++;
1525           break;
1526         }
1527       }
1528       for (;i < 4; i++) {
1529         if (imm_h[i] != 0xffffL) {
1530           movk(dst, (u_int32_t)imm_h[i], (i << 4));
1531         }
1532       }
1533     } else if (zero_count == 1) {
1534       // one MOVZ and two MOVKs will do
1535       for (i = 0; i < 4; i++) {
1536         if (imm_h[i] != 0L) {
1537           movz(dst, (u_int32_t)imm_h[i], (i << 4));
1538           i++;
1539           break;
1540         }
1541       }
1542       for (;i < 4; i++) {
1543         if (imm_h[i] != 0x0L) {
1544           movk(dst, (u_int32_t)imm_h[i], (i << 4));
1545         }
1546       }
1547     } else if (neg_count == 1) {
1548       // one MOVN and two MOVKs will do
1549       for (i = 0; i < 4; i++) {
1550         if (imm_h[i] != 0xffffL) {
1551           movn(dst, (u_int32_t)imm_h[i] ^ 0xffffL, (i << 4));
1552           i++;
1553           break;
1554         }
1555       }
1556       for (;i < 4; i++) {
1557         if (imm_h[i] != 0xffffL) {
1558           movk(dst, (u_int32_t)imm_h[i], (i << 4));
1559         }
1560       }
1561     } else {
1562       // use a MOVZ and 3 MOVKs (makes it easier to debug)
1563       movz(dst, (u_int32_t)imm_h[0], 0);
1564       for (i = 1; i < 4; i++) {
1565         movk(dst, (u_int32_t)imm_h[i], (i << 4));
1566       }
1567     }
1568   }
1569 }
1570 
1571 void MacroAssembler::mov_immediate32(Register dst, u_int32_t imm32)
1572 {
1573 #ifndef PRODUCT
1574     {
1575       char buffer[64];
1576       snprintf(buffer, sizeof(buffer), "0x%"PRIX32, imm32);
1577       block_comment(buffer);
1578     }
1579 #endif
1580   if (operand_valid_for_logical_immediate(true, imm32)) {
1581     orrw(dst, zr, imm32);
1582   } else {
1583     // we can use MOVZ, MOVN or two calls to MOVK to build up the
1584     // constant
1585     u_int32_t imm_h[2];
1586     imm_h[0] = imm32 & 0xffff;
1587     imm_h[1] = ((imm32 >> 16) & 0xffff);
1588     if (imm_h[0] == 0) {
1589       movzw(dst, imm_h[1], 16);
1590     } else if (imm_h[0] == 0xffff) {
1591       movnw(dst, imm_h[1] ^ 0xffff, 16);
1592     } else if (imm_h[1] == 0) {
1593       movzw(dst, imm_h[0], 0);
1594     } else if (imm_h[1] == 0xffff) {
1595       movnw(dst, imm_h[0] ^ 0xffff, 0);
1596     } else {
1597       // use a MOVZ and MOVK (makes it easier to debug)
1598       movzw(dst, imm_h[0], 0);
1599       movkw(dst, imm_h[1], 16);
1600     }
1601   }
1602 }
1603 
1604 // Form an address from base + offset in Rd.  Rd may or may
1605 // not actually be used: you must use the Address that is returned.
1606 // It is up to you to ensure that the shift provided matches the size
1607 // of your data.
1608 Address MacroAssembler::form_address(Register Rd, Register base, long byte_offset, int shift) {
1609   if (Address::offset_ok_for_immed(byte_offset, shift))
1610     // It fits; no need for any heroics
1611     return Address(base, byte_offset);
1612 
1613   // Don't do anything clever with negative or misaligned offsets
1614   unsigned mask = (1 << shift) - 1;
1615   if (byte_offset < 0 || byte_offset & mask) {
1616     mov(Rd, byte_offset);
1617     add(Rd, base, Rd);
1618     return Address(Rd);
1619   }
1620 
1621   // See if we can do this with two 12-bit offsets
1622   {
1623     unsigned long word_offset = byte_offset >> shift;
1624     unsigned long masked_offset = word_offset & 0xfff000;
1625     if (Address::offset_ok_for_immed(word_offset - masked_offset)
1626         && Assembler::operand_valid_for_add_sub_immediate(masked_offset << shift)) {
1627       add(Rd, base, masked_offset << shift);
1628       word_offset -= masked_offset;
1629       return Address(Rd, word_offset << shift);
1630     }
1631   }
1632 
1633   // Do it the hard way
1634   mov(Rd, byte_offset);
1635   add(Rd, base, Rd);
1636   return Address(Rd);
1637 }
1638 
1639 void MacroAssembler::atomic_incw(Register counter_addr, Register tmp, Register tmp2) {
1640   Label retry_load;
1641   bind(retry_load);
1642   // flush and load exclusive from the memory location
1643   ldxrw(tmp, counter_addr);
1644   addw(tmp, tmp, 1);
1645   // if we store+flush with no intervening write tmp wil be zero
1646   stxrw(tmp2, tmp, counter_addr);
1647   cbnzw(tmp2, retry_load);
1648 }
1649 
1650 
1651 int MacroAssembler::corrected_idivl(Register result, Register ra, Register rb,
1652                                     bool want_remainder, Register scratch)
1653 {
1654   // Full implementation of Java idiv and irem.  The function
1655   // returns the (pc) offset of the div instruction - may be needed
1656   // for implicit exceptions.
1657   //
1658   // constraint : ra/rb =/= scratch
1659   //         normal case
1660   //
1661   // input : ra: dividend
1662   //         rb: divisor
1663   //
1664   // result: either
1665   //         quotient  (= ra idiv rb)
1666   //         remainder (= ra irem rb)
1667 
1668   assert(ra != scratch && rb != scratch, "reg cannot be scratch");
1669 
1670   int idivl_offset = offset();
1671   if (! want_remainder) {
1672     sdivw(result, ra, rb);
1673   } else {
1674     sdivw(scratch, ra, rb);
1675     Assembler::msubw(result, scratch, rb, ra);
1676   }
1677 
1678   return idivl_offset;
1679 }
1680 
1681 int MacroAssembler::corrected_idivq(Register result, Register ra, Register rb,
1682                                     bool want_remainder, Register scratch)
1683 {
1684   // Full implementation of Java ldiv and lrem.  The function
1685   // returns the (pc) offset of the div instruction - may be needed
1686   // for implicit exceptions.
1687   //
1688   // constraint : ra/rb =/= scratch
1689   //         normal case
1690   //
1691   // input : ra: dividend
1692   //         rb: divisor
1693   //
1694   // result: either
1695   //         quotient  (= ra idiv rb)
1696   //         remainder (= ra irem rb)
1697 
1698   assert(ra != scratch && rb != scratch, "reg cannot be scratch");
1699 
1700   int idivq_offset = offset();
1701   if (! want_remainder) {
1702     sdiv(result, ra, rb);
1703   } else {
1704     sdiv(scratch, ra, rb);
1705     Assembler::msub(result, scratch, rb, ra);
1706   }
1707 
1708   return idivq_offset;
1709 }
1710 
1711 void MacroAssembler::membar(Membar_mask_bits order_constraint) {
1712   address prev = pc() - NativeMembar::instruction_size;
1713   if (prev == code()->last_membar()) {
1714     NativeMembar *bar = NativeMembar_at(prev);
1715     // We are merging two memory barrier instructions.  On AArch64 we
1716     // can do this simply by ORing them together.
1717     bar->set_kind(bar->get_kind() | order_constraint);
1718     BLOCK_COMMENT("merged membar");
1719   } else {
1720     code()->set_last_membar(pc());
1721     dmb(Assembler::barrier(order_constraint));
1722   }
1723 }
1724 
1725 // MacroAssembler routines found actually to be needed
1726 
1727 void MacroAssembler::push(Register src)
1728 {
1729   str(src, Address(pre(esp, -1 * wordSize)));
1730 }
1731 
1732 void MacroAssembler::pop(Register dst)
1733 {
1734   ldr(dst, Address(post(esp, 1 * wordSize)));
1735 }
1736 
1737 // Note: load_unsigned_short used to be called load_unsigned_word.
1738 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
1739   int off = offset();
1740   ldrh(dst, src);
1741   return off;
1742 }
1743 
1744 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
1745   int off = offset();
1746   ldrb(dst, src);
1747   return off;
1748 }
1749 
1750 int MacroAssembler::load_signed_short(Register dst, Address src) {
1751   int off = offset();
1752   ldrsh(dst, src);
1753   return off;
1754 }
1755 
1756 int MacroAssembler::load_signed_byte(Register dst, Address src) {
1757   int off = offset();
1758   ldrsb(dst, src);
1759   return off;
1760 }
1761 
1762 int MacroAssembler::load_signed_short32(Register dst, Address src) {
1763   int off = offset();
1764   ldrshw(dst, src);
1765   return off;
1766 }
1767 
1768 int MacroAssembler::load_signed_byte32(Register dst, Address src) {
1769   int off = offset();
1770   ldrsbw(dst, src);
1771   return off;
1772 }
1773 
1774 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
1775   switch (size_in_bytes) {
1776   case  8:  ldr(dst, src); break;
1777   case  4:  ldrw(dst, src); break;
1778   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
1779   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
1780   default:  ShouldNotReachHere();
1781   }
1782 }
1783 
1784 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
1785   switch (size_in_bytes) {
1786   case  8:  str(src, dst); break;
1787   case  4:  strw(src, dst); break;
1788   case  2:  strh(src, dst); break;
1789   case  1:  strb(src, dst); break;
1790   default:  ShouldNotReachHere();
1791   }
1792 }
1793 
1794 void MacroAssembler::decrementw(Register reg, int value)
1795 {
1796   if (value < 0)  { incrementw(reg, -value);      return; }
1797   if (value == 0) {                               return; }
1798   if (value < (1 << 12)) { subw(reg, reg, value); return; }
1799   /* else */ {
1800     guarantee(reg != rscratch2, "invalid dst for register decrement");
1801     movw(rscratch2, (unsigned)value);
1802     subw(reg, reg, rscratch2);
1803   }
1804 }
1805 
1806 void MacroAssembler::decrement(Register reg, int value)
1807 {
1808   if (value < 0)  { increment(reg, -value);      return; }
1809   if (value == 0) {                              return; }
1810   if (value < (1 << 12)) { sub(reg, reg, value); return; }
1811   /* else */ {
1812     assert(reg != rscratch2, "invalid dst for register decrement");
1813     mov(rscratch2, (unsigned long)value);
1814     sub(reg, reg, rscratch2);
1815   }
1816 }
1817 
1818 void MacroAssembler::decrementw(Address dst, int value)
1819 {
1820   assert(!dst.uses(rscratch1), "invalid dst for address decrement");
1821   ldrw(rscratch1, dst);
1822   decrementw(rscratch1, value);
1823   strw(rscratch1, dst);
1824 }
1825 
1826 void MacroAssembler::decrement(Address dst, int value)
1827 {
1828   assert(!dst.uses(rscratch1), "invalid address for decrement");
1829   ldr(rscratch1, dst);
1830   decrement(rscratch1, value);
1831   str(rscratch1, dst);
1832 }
1833 
1834 void MacroAssembler::incrementw(Register reg, int value)
1835 {
1836   if (value < 0)  { decrementw(reg, -value);      return; }
1837   if (value == 0) {                               return; }
1838   if (value < (1 << 12)) { addw(reg, reg, value); return; }
1839   /* else */ {
1840     assert(reg != rscratch2, "invalid dst for register increment");
1841     movw(rscratch2, (unsigned)value);
1842     addw(reg, reg, rscratch2);
1843   }
1844 }
1845 
1846 void MacroAssembler::increment(Register reg, int value)
1847 {
1848   if (value < 0)  { decrement(reg, -value);      return; }
1849   if (value == 0) {                              return; }
1850   if (value < (1 << 12)) { add(reg, reg, value); return; }
1851   /* else */ {
1852     assert(reg != rscratch2, "invalid dst for register increment");
1853     movw(rscratch2, (unsigned)value);
1854     add(reg, reg, rscratch2);
1855   }
1856 }
1857 
1858 void MacroAssembler::incrementw(Address dst, int value)
1859 {
1860   assert(!dst.uses(rscratch1), "invalid dst for address increment");
1861   ldrw(rscratch1, dst);
1862   incrementw(rscratch1, value);
1863   strw(rscratch1, dst);
1864 }
1865 
1866 void MacroAssembler::increment(Address dst, int value)
1867 {
1868   assert(!dst.uses(rscratch1), "invalid dst for address increment");
1869   ldr(rscratch1, dst);
1870   increment(rscratch1, value);
1871   str(rscratch1, dst);
1872 }
1873 
1874 
1875 void MacroAssembler::pusha() {
1876   push(0x7fffffff, sp);
1877 }
1878 
1879 void MacroAssembler::popa() {
1880   pop(0x7fffffff, sp);
1881 }
1882 
1883 // Push lots of registers in the bit set supplied.  Don't push sp.
1884 // Return the number of words pushed
1885 int MacroAssembler::push(unsigned int bitset, Register stack) {
1886   int words_pushed = 0;
1887 
1888   // Scan bitset to accumulate register pairs
1889   unsigned char regs[32];
1890   int count = 0;
1891   for (int reg = 0; reg <= 30; reg++) {
1892     if (1 & bitset)
1893       regs[count++] = reg;
1894     bitset >>= 1;
1895   }
1896   regs[count++] = zr->encoding_nocheck();
1897   count &= ~1;  // Only push an even nuber of regs
1898 
1899   if (count) {
1900     stp(as_Register(regs[0]), as_Register(regs[1]),
1901        Address(pre(stack, -count * wordSize)));
1902     words_pushed += 2;
1903   }
1904   for (int i = 2; i < count; i += 2) {
1905     stp(as_Register(regs[i]), as_Register(regs[i+1]),
1906        Address(stack, i * wordSize));
1907     words_pushed += 2;
1908   }
1909 
1910   assert(words_pushed == count, "oops, pushed != count");
1911 
1912   return count;
1913 }
1914 
1915 int MacroAssembler::pop(unsigned int bitset, Register stack) {
1916   int words_pushed = 0;
1917 
1918   // Scan bitset to accumulate register pairs
1919   unsigned char regs[32];
1920   int count = 0;
1921   for (int reg = 0; reg <= 30; reg++) {
1922     if (1 & bitset)
1923       regs[count++] = reg;
1924     bitset >>= 1;
1925   }
1926   regs[count++] = zr->encoding_nocheck();
1927   count &= ~1;
1928 
1929   for (int i = 2; i < count; i += 2) {
1930     ldp(as_Register(regs[i]), as_Register(regs[i+1]),
1931        Address(stack, i * wordSize));
1932     words_pushed += 2;
1933   }
1934   if (count) {
1935     ldp(as_Register(regs[0]), as_Register(regs[1]),
1936        Address(post(stack, count * wordSize)));
1937     words_pushed += 2;
1938   }
1939 
1940   assert(words_pushed == count, "oops, pushed != count");
1941 
1942   return count;
1943 }
1944 #ifdef ASSERT
1945 void MacroAssembler::verify_heapbase(const char* msg) {
1946 #if 0
1947   assert (UseCompressedOops || UseCompressedClassPointers, "should be compressed");
1948   assert (Universe::heap() != NULL, "java heap should be initialized");
1949   if (CheckCompressedOops) {
1950     Label ok;
1951     push(1 << rscratch1->encoding(), sp); // cmpptr trashes rscratch1
1952     cmpptr(rheapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
1953     br(Assembler::EQ, ok);
1954     stop(msg);
1955     bind(ok);
1956     pop(1 << rscratch1->encoding(), sp);
1957   }
1958 #endif
1959 }
1960 #endif
1961 
1962 void MacroAssembler::stop(const char* msg) {
1963   address ip = pc();
1964   pusha();
1965   mov(c_rarg0, (address)msg);
1966   mov(c_rarg1, (address)ip);
1967   mov(c_rarg2, sp);
1968   mov(c_rarg3, CAST_FROM_FN_PTR(address, MacroAssembler::debug64));
1969   // call(c_rarg3);
1970   blrt(c_rarg3, 3, 0, 1);
1971   hlt(0);
1972 }
1973 
1974 // If a constant does not fit in an immediate field, generate some
1975 // number of MOV instructions and then perform the operation.
1976 void MacroAssembler::wrap_add_sub_imm_insn(Register Rd, Register Rn, unsigned imm,
1977                                            add_sub_imm_insn insn1,
1978                                            add_sub_reg_insn insn2) {
1979   assert(Rd != zr, "Rd = zr and not setting flags?");
1980   if (operand_valid_for_add_sub_immediate((int)imm)) {
1981     (this->*insn1)(Rd, Rn, imm);
1982   } else {
1983     if (uabs(imm) < (1 << 24)) {
1984        (this->*insn1)(Rd, Rn, imm & -(1 << 12));
1985        (this->*insn1)(Rd, Rd, imm & ((1 << 12)-1));
1986     } else {
1987        assert_different_registers(Rd, Rn);
1988        mov(Rd, (uint64_t)imm);
1989        (this->*insn2)(Rd, Rn, Rd, LSL, 0);
1990     }
1991   }
1992 }
1993 
1994 // Seperate vsn which sets the flags. Optimisations are more restricted
1995 // because we must set the flags correctly.
1996 void MacroAssembler::wrap_adds_subs_imm_insn(Register Rd, Register Rn, unsigned imm,
1997                                            add_sub_imm_insn insn1,
1998                                            add_sub_reg_insn insn2) {
1999   if (operand_valid_for_add_sub_immediate((int)imm)) {
2000     (this->*insn1)(Rd, Rn, imm);
2001   } else {
2002     assert_different_registers(Rd, Rn);
2003     assert(Rd != zr, "overflow in immediate operand");
2004     mov(Rd, (uint64_t)imm);
2005     (this->*insn2)(Rd, Rn, Rd, LSL, 0);
2006   }
2007 }
2008 
2009 
2010 void MacroAssembler::add(Register Rd, Register Rn, RegisterOrConstant increment) {
2011   if (increment.is_register()) {
2012     add(Rd, Rn, increment.as_register());
2013   } else {
2014     add(Rd, Rn, increment.as_constant());
2015   }
2016 }
2017 
2018 void MacroAssembler::addw(Register Rd, Register Rn, RegisterOrConstant increment) {
2019   if (increment.is_register()) {
2020     addw(Rd, Rn, increment.as_register());
2021   } else {
2022     addw(Rd, Rn, increment.as_constant());
2023   }
2024 }
2025 
2026 void MacroAssembler::sub(Register Rd, Register Rn, RegisterOrConstant decrement) {
2027   if (decrement.is_register()) {
2028     sub(Rd, Rn, decrement.as_register());
2029   } else {
2030     sub(Rd, Rn, decrement.as_constant());
2031   }
2032 }
2033 
2034 void MacroAssembler::subw(Register Rd, Register Rn, RegisterOrConstant decrement) {
2035   if (decrement.is_register()) {
2036     subw(Rd, Rn, decrement.as_register());
2037   } else {
2038     subw(Rd, Rn, decrement.as_constant());
2039   }
2040 }
2041 
2042 void MacroAssembler::reinit_heapbase()
2043 {
2044   if (UseCompressedOops) {
2045     if (Universe::is_fully_initialized()) {
2046       mov(rheapbase, Universe::narrow_ptrs_base());
2047     } else {
2048       lea(rheapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
2049       ldr(rheapbase, Address(rheapbase));
2050     }
2051   }
2052 }
2053 
2054 // this simulates the behaviour of the x86 cmpxchg instruction using a
2055 // load linked/store conditional pair. we use the acquire/release
2056 // versions of these instructions so that we flush pending writes as
2057 // per Java semantics.
2058 
2059 // n.b the x86 version assumes the old value to be compared against is
2060 // in rax and updates rax with the value located in memory if the
2061 // cmpxchg fails. we supply a register for the old value explicitly
2062 
2063 // the aarch64 load linked/store conditional instructions do not
2064 // accept an offset. so, unlike x86, we must provide a plain register
2065 // to identify the memory word to be compared/exchanged rather than a
2066 // register+offset Address.
2067 
2068 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
2069                                 Label &succeed, Label *fail) {
2070   // oldv holds comparison value
2071   // newv holds value to write in exchange
2072   // addr identifies memory word to compare against/update
2073   // tmp returns 0/1 for success/failure
2074   if (UseLSE) {
2075     mov(tmp, oldv);
2076     casal(oldv, newv, addr);
2077     cmp(tmp, oldv);
2078     br(Assembler::EQ, succeed);
2079     membar(AnyAny);
2080   } else {
2081     Label retry_load, nope;
2082 
2083     bind(retry_load);
2084     // flush and load exclusive from the memory location
2085     // and fail if it is not what we expect
2086     ldaxr(tmp, addr);
2087     cmp(tmp, oldv);
2088     br(Assembler::NE, nope);
2089     // if we store+flush with no intervening write tmp wil be zero
2090     stlxr(tmp, newv, addr);
2091     cbzw(tmp, succeed);
2092     // retry so we only ever return after a load fails to compare
2093     // ensures we don't return a stale value after a failed write.
2094     b(retry_load);
2095     // if the memory word differs we return it in oldv and signal a fail
2096     bind(nope);
2097     membar(AnyAny);
2098     mov(oldv, tmp);
2099   }
2100   if (fail)
2101     b(*fail);
2102 }
2103 
2104 void MacroAssembler::cmpxchgw(Register oldv, Register newv, Register addr, Register tmp,
2105                                 Label &succeed, Label *fail) {
2106   // oldv holds comparison value
2107   // newv holds value to write in exchange
2108   // addr identifies memory word to compare against/update
2109   // tmp returns 0/1 for success/failure
2110   if (UseLSE) {
2111     mov(tmp, oldv);
2112     casalw(oldv, newv, addr);
2113     cmp(tmp, oldv);
2114     br(Assembler::EQ, succeed);
2115     membar(AnyAny);
2116   } else {
2117     Label retry_load, nope;
2118 
2119     bind(retry_load);
2120     // flush and load exclusive from the memory location
2121     // and fail if it is not what we expect
2122     ldaxrw(tmp, addr);
2123     cmp(tmp, oldv);
2124     br(Assembler::NE, nope);
2125     // if we store+flush with no intervening write tmp wil be zero
2126     stlxrw(tmp, newv, addr);
2127     cbzw(tmp, succeed);
2128     // retry so we only ever return after a load fails to compare
2129     // ensures we don't return a stale value after a failed write.
2130     b(retry_load);
2131     // if the memory word differs we return it in oldv and signal a fail
2132     bind(nope);
2133     membar(AnyAny);
2134     mov(oldv, tmp);
2135   }
2136   if (fail)
2137     b(*fail);
2138 }
2139 
2140 static bool different(Register a, RegisterOrConstant b, Register c) {
2141   if (b.is_constant())
2142     return a != c;
2143   else
2144     return a != b.as_register() && a != c && b.as_register() != c;
2145 }
2146 
2147 #define ATOMIC_OP(LDXR, OP, IOP, STXR)                                       \
2148 void MacroAssembler::atomic_##OP(Register prev, RegisterOrConstant incr, Register addr) { \
2149   Register result = rscratch2;                                          \
2150   if (prev->is_valid())                                                 \
2151     result = different(prev, incr, addr) ? prev : rscratch2;            \
2152                                                                         \
2153   Label retry_load;                                                     \
2154   bind(retry_load);                                                     \
2155   LDXR(result, addr);                                                   \
2156   OP(rscratch1, result, incr);                                          \
2157   STXR(rscratch2, rscratch1, addr);                                     \
2158   cbnzw(rscratch2, retry_load);                                         \
2159   if (prev->is_valid() && prev != result) {                             \
2160     IOP(prev, rscratch1, incr);                                         \
2161   }                                                                     \
2162 }
2163 
2164 ATOMIC_OP(ldxr, add, sub, stxr)
2165 ATOMIC_OP(ldxrw, addw, subw, stxrw)
2166 
2167 #undef ATOMIC_OP
2168 
2169 #define ATOMIC_XCHG(OP, LDXR, STXR)                                     \
2170 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
2171   Register result = rscratch2;                                          \
2172   if (prev->is_valid())                                                 \
2173     result = different(prev, newv, addr) ? prev : rscratch2;            \
2174                                                                         \
2175   Label retry_load;                                                     \
2176   bind(retry_load);                                                     \
2177   LDXR(result, addr);                                                   \
2178   STXR(rscratch1, newv, addr);                                          \
2179   cbnzw(rscratch1, retry_load);                                         \
2180   if (prev->is_valid() && prev != result)                               \
2181     mov(prev, result);                                                  \
2182 }
2183 
2184 ATOMIC_XCHG(xchg, ldxr, stxr)
2185 ATOMIC_XCHG(xchgw, ldxrw, stxrw)
2186 
2187 #undef ATOMIC_XCHG
2188 
2189 void MacroAssembler::incr_allocated_bytes(Register thread,
2190                                           Register var_size_in_bytes,
2191                                           int con_size_in_bytes,
2192                                           Register t1) {
2193   if (!thread->is_valid()) {
2194     thread = rthread;
2195   }
2196   assert(t1->is_valid(), "need temp reg");
2197 
2198   ldr(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2199   if (var_size_in_bytes->is_valid()) {
2200     add(t1, t1, var_size_in_bytes);
2201   } else {
2202     add(t1, t1, con_size_in_bytes);
2203   }
2204   str(t1, Address(thread, in_bytes(JavaThread::allocated_bytes_offset())));
2205 }
2206 
2207 #ifndef PRODUCT
2208 extern "C" void findpc(intptr_t x);
2209 #endif
2210 
2211 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
2212 {
2213   // In order to get locks to work, we need to fake a in_VM state
2214   if (ShowMessageBoxOnError ) {
2215     JavaThread* thread = JavaThread::current();
2216     JavaThreadState saved_state = thread->thread_state();
2217     thread->set_thread_state(_thread_in_vm);
2218 #ifndef PRODUCT
2219     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
2220       ttyLocker ttyl;
2221       BytecodeCounter::print();
2222     }
2223 #endif
2224     if (os::message_box(msg, "Execution stopped, print registers?")) {
2225       ttyLocker ttyl;
2226       tty->print_cr(" pc = 0x%016lx", pc);
2227 #ifndef PRODUCT
2228       tty->cr();
2229       findpc(pc);
2230       tty->cr();
2231 #endif
2232       tty->print_cr(" r0 = 0x%016lx", regs[0]);
2233       tty->print_cr(" r1 = 0x%016lx", regs[1]);
2234       tty->print_cr(" r2 = 0x%016lx", regs[2]);
2235       tty->print_cr(" r3 = 0x%016lx", regs[3]);
2236       tty->print_cr(" r4 = 0x%016lx", regs[4]);
2237       tty->print_cr(" r5 = 0x%016lx", regs[5]);
2238       tty->print_cr(" r6 = 0x%016lx", regs[6]);
2239       tty->print_cr(" r7 = 0x%016lx", regs[7]);
2240       tty->print_cr(" r8 = 0x%016lx", regs[8]);
2241       tty->print_cr(" r9 = 0x%016lx", regs[9]);
2242       tty->print_cr("r10 = 0x%016lx", regs[10]);
2243       tty->print_cr("r11 = 0x%016lx", regs[11]);
2244       tty->print_cr("r12 = 0x%016lx", regs[12]);
2245       tty->print_cr("r13 = 0x%016lx", regs[13]);
2246       tty->print_cr("r14 = 0x%016lx", regs[14]);
2247       tty->print_cr("r15 = 0x%016lx", regs[15]);
2248       tty->print_cr("r16 = 0x%016lx", regs[16]);
2249       tty->print_cr("r17 = 0x%016lx", regs[17]);
2250       tty->print_cr("r18 = 0x%016lx", regs[18]);
2251       tty->print_cr("r19 = 0x%016lx", regs[19]);
2252       tty->print_cr("r20 = 0x%016lx", regs[20]);
2253       tty->print_cr("r21 = 0x%016lx", regs[21]);
2254       tty->print_cr("r22 = 0x%016lx", regs[22]);
2255       tty->print_cr("r23 = 0x%016lx", regs[23]);
2256       tty->print_cr("r24 = 0x%016lx", regs[24]);
2257       tty->print_cr("r25 = 0x%016lx", regs[25]);
2258       tty->print_cr("r26 = 0x%016lx", regs[26]);
2259       tty->print_cr("r27 = 0x%016lx", regs[27]);
2260       tty->print_cr("r28 = 0x%016lx", regs[28]);
2261       tty->print_cr("r30 = 0x%016lx", regs[30]);
2262       tty->print_cr("r31 = 0x%016lx", regs[31]);
2263       BREAKPOINT;
2264     }
2265     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
2266   } else {
2267     ttyLocker ttyl;
2268     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
2269                     msg);
2270     assert(false, "DEBUG MESSAGE: %s", msg);
2271   }
2272 }
2273 
2274 #ifdef BUILTIN_SIM
2275 // routine to generate an x86 prolog for a stub function which
2276 // bootstraps into the generated ARM code which directly follows the
2277 // stub
2278 //
2279 // the argument encodes the number of general and fp registers
2280 // passed by the caller and the callng convention (currently just
2281 // the number of general registers and assumes C argument passing)
2282 
2283 extern "C" {
2284 int aarch64_stub_prolog_size();
2285 void aarch64_stub_prolog();
2286 void aarch64_prolog();
2287 }
2288 
2289 void MacroAssembler::c_stub_prolog(int gp_arg_count, int fp_arg_count, int ret_type,
2290                                    address *prolog_ptr)
2291 {
2292   int calltype = (((ret_type & 0x3) << 8) |
2293                   ((fp_arg_count & 0xf) << 4) |
2294                   (gp_arg_count & 0xf));
2295 
2296   // the addresses for the x86 to ARM entry code we need to use
2297   address start = pc();
2298   // printf("start = %lx\n", start);
2299   int byteCount =  aarch64_stub_prolog_size();
2300   // printf("byteCount = %x\n", byteCount);
2301   int instructionCount = (byteCount + 3)/ 4;
2302   // printf("instructionCount = %x\n", instructionCount);
2303   for (int i = 0; i < instructionCount; i++) {
2304     nop();
2305   }
2306 
2307   memcpy(start, (void*)aarch64_stub_prolog, byteCount);
2308 
2309   // write the address of the setup routine and the call format at the
2310   // end of into the copied code
2311   u_int64_t *patch_end = (u_int64_t *)(start + byteCount);
2312   if (prolog_ptr)
2313     patch_end[-2] = (u_int64_t)prolog_ptr;
2314   patch_end[-1] = calltype;
2315 }
2316 #endif
2317 
2318 void MacroAssembler::push_call_clobbered_registers() {
2319   push(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
2320 
2321   // Push v0-v7, v16-v31.
2322   for (int i = 30; i >= 0; i -= 2) {
2323     if (i <= v7->encoding() || i >= v16->encoding()) {
2324         stpd(as_FloatRegister(i), as_FloatRegister(i+1),
2325              Address(pre(sp, -2 * wordSize)));
2326     }
2327   }
2328 }
2329 
2330 void MacroAssembler::pop_call_clobbered_registers() {
2331 
2332   for (int i = 0; i < 32; i += 2) {
2333     if (i <= v7->encoding() || i >= v16->encoding()) {
2334       ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
2335            Address(post(sp, 2 * wordSize)));
2336     }
2337   }
2338 
2339   pop(RegSet::range(r0, r18) - RegSet::of(rscratch1, rscratch2), sp);
2340 }
2341 
2342 void MacroAssembler::push_CPU_state(bool save_vectors) {
2343   push(0x3fffffff, sp);         // integer registers except lr & sp
2344 
2345   if (!save_vectors) {
2346     for (int i = 30; i >= 0; i -= 2)
2347       stpd(as_FloatRegister(i), as_FloatRegister(i+1),
2348            Address(pre(sp, -2 * wordSize)));
2349   } else {
2350     for (int i = 30; i >= 0; i -= 2)
2351       stpq(as_FloatRegister(i), as_FloatRegister(i+1),
2352            Address(pre(sp, -4 * wordSize)));
2353   }
2354 }
2355 
2356 void MacroAssembler::pop_CPU_state(bool restore_vectors) {
2357   if (!restore_vectors) {
2358     for (int i = 0; i < 32; i += 2)
2359       ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
2360            Address(post(sp, 2 * wordSize)));
2361   } else {
2362     for (int i = 0; i < 32; i += 2)
2363       ldpq(as_FloatRegister(i), as_FloatRegister(i+1),
2364            Address(post(sp, 4 * wordSize)));
2365   }
2366 
2367   pop(0x3fffffff, sp);         // integer registers except lr & sp
2368 }
2369 
2370 /**
2371  * Helpers for multiply_to_len().
2372  */
2373 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
2374                                      Register src1, Register src2) {
2375   adds(dest_lo, dest_lo, src1);
2376   adc(dest_hi, dest_hi, zr);
2377   adds(dest_lo, dest_lo, src2);
2378   adc(final_dest_hi, dest_hi, zr);
2379 }
2380 
2381 // Generate an address from (r + r1 extend offset).  "size" is the
2382 // size of the operand.  The result may be in rscratch2.
2383 Address MacroAssembler::offsetted_address(Register r, Register r1,
2384                                           Address::extend ext, int offset, int size) {
2385   if (offset || (ext.shift() % size != 0)) {
2386     lea(rscratch2, Address(r, r1, ext));
2387     return Address(rscratch2, offset);
2388   } else {
2389     return Address(r, r1, ext);
2390   }
2391 }
2392 
2393 Address MacroAssembler::spill_address(int size, int offset, Register tmp)
2394 {
2395   assert(offset >= 0, "spill to negative address?");
2396   // Offset reachable ?
2397   //   Not aligned - 9 bits signed offset
2398   //   Aligned - 12 bits unsigned offset shifted
2399   Register base = sp;
2400   if ((offset & (size-1)) && offset >= (1<<8)) {
2401     add(tmp, base, offset & ((1<<12)-1));
2402     base = tmp;
2403     offset &= -1<<12;
2404   }
2405 
2406   if (offset >= (1<<12) * size) {
2407     add(tmp, base, offset & (((1<<12)-1)<<12));
2408     base = tmp;
2409     offset &= ~(((1<<12)-1)<<12);
2410   }
2411 
2412   return Address(base, offset);
2413 }
2414 
2415 /**
2416  * Multiply 64 bit by 64 bit first loop.
2417  */
2418 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
2419                                            Register y, Register y_idx, Register z,
2420                                            Register carry, Register product,
2421                                            Register idx, Register kdx) {
2422   //
2423   //  jlong carry, x[], y[], z[];
2424   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
2425   //    huge_128 product = y[idx] * x[xstart] + carry;
2426   //    z[kdx] = (jlong)product;
2427   //    carry  = (jlong)(product >>> 64);
2428   //  }
2429   //  z[xstart] = carry;
2430   //
2431 
2432   Label L_first_loop, L_first_loop_exit;
2433   Label L_one_x, L_one_y, L_multiply;
2434 
2435   subsw(xstart, xstart, 1);
2436   br(Assembler::MI, L_one_x);
2437 
2438   lea(rscratch1, Address(x, xstart, Address::lsl(LogBytesPerInt)));
2439   ldr(x_xstart, Address(rscratch1));
2440   ror(x_xstart, x_xstart, 32); // convert big-endian to little-endian
2441 
2442   bind(L_first_loop);
2443   subsw(idx, idx, 1);
2444   br(Assembler::MI, L_first_loop_exit);
2445   subsw(idx, idx, 1);
2446   br(Assembler::MI, L_one_y);
2447   lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2448   ldr(y_idx, Address(rscratch1));
2449   ror(y_idx, y_idx, 32); // convert big-endian to little-endian
2450   bind(L_multiply);
2451 
2452   // AArch64 has a multiply-accumulate instruction that we can't use
2453   // here because it has no way to process carries, so we have to use
2454   // separate add and adc instructions.  Bah.
2455   umulh(rscratch1, x_xstart, y_idx); // x_xstart * y_idx -> rscratch1:product
2456   mul(product, x_xstart, y_idx);
2457   adds(product, product, carry);
2458   adc(carry, rscratch1, zr);   // x_xstart * y_idx + carry -> carry:product
2459 
2460   subw(kdx, kdx, 2);
2461   ror(product, product, 32); // back to big-endian
2462   str(product, offsetted_address(z, kdx, Address::uxtw(LogBytesPerInt), 0, BytesPerLong));
2463 
2464   b(L_first_loop);
2465 
2466   bind(L_one_y);
2467   ldrw(y_idx, Address(y,  0));
2468   b(L_multiply);
2469 
2470   bind(L_one_x);
2471   ldrw(x_xstart, Address(x,  0));
2472   b(L_first_loop);
2473 
2474   bind(L_first_loop_exit);
2475 }
2476 
2477 /**
2478  * Multiply 128 bit by 128. Unrolled inner loop.
2479  *
2480  */
2481 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z,
2482                                              Register carry, Register carry2,
2483                                              Register idx, Register jdx,
2484                                              Register yz_idx1, Register yz_idx2,
2485                                              Register tmp, Register tmp3, Register tmp4,
2486                                              Register tmp6, Register product_hi) {
2487 
2488   //   jlong carry, x[], y[], z[];
2489   //   int kdx = ystart+1;
2490   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
2491   //     huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry;
2492   //     jlong carry2  = (jlong)(tmp3 >>> 64);
2493   //     huge_128 tmp4 = (y[idx]   * product_hi) + z[kdx+idx] + carry2;
2494   //     carry  = (jlong)(tmp4 >>> 64);
2495   //     z[kdx+idx+1] = (jlong)tmp3;
2496   //     z[kdx+idx] = (jlong)tmp4;
2497   //   }
2498   //   idx += 2;
2499   //   if (idx > 0) {
2500   //     yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry;
2501   //     z[kdx+idx] = (jlong)yz_idx1;
2502   //     carry  = (jlong)(yz_idx1 >>> 64);
2503   //   }
2504   //
2505 
2506   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
2507 
2508   lsrw(jdx, idx, 2);
2509 
2510   bind(L_third_loop);
2511 
2512   subsw(jdx, jdx, 1);
2513   br(Assembler::MI, L_third_loop_exit);
2514   subw(idx, idx, 4);
2515 
2516   lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2517 
2518   ldp(yz_idx2, yz_idx1, Address(rscratch1, 0));
2519 
2520   lea(tmp6, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2521 
2522   ror(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
2523   ror(yz_idx2, yz_idx2, 32);
2524 
2525   ldp(rscratch2, rscratch1, Address(tmp6, 0));
2526 
2527   mul(tmp3, product_hi, yz_idx1);  //  yz_idx1 * product_hi -> tmp4:tmp3
2528   umulh(tmp4, product_hi, yz_idx1);
2529 
2530   ror(rscratch1, rscratch1, 32); // convert big-endian to little-endian
2531   ror(rscratch2, rscratch2, 32);
2532 
2533   mul(tmp, product_hi, yz_idx2);   //  yz_idx2 * product_hi -> carry2:tmp
2534   umulh(carry2, product_hi, yz_idx2);
2535 
2536   // propagate sum of both multiplications into carry:tmp4:tmp3
2537   adds(tmp3, tmp3, carry);
2538   adc(tmp4, tmp4, zr);
2539   adds(tmp3, tmp3, rscratch1);
2540   adcs(tmp4, tmp4, tmp);
2541   adc(carry, carry2, zr);
2542   adds(tmp4, tmp4, rscratch2);
2543   adc(carry, carry, zr);
2544 
2545   ror(tmp3, tmp3, 32); // convert little-endian to big-endian
2546   ror(tmp4, tmp4, 32);
2547   stp(tmp4, tmp3, Address(tmp6, 0));
2548 
2549   b(L_third_loop);
2550   bind (L_third_loop_exit);
2551 
2552   andw (idx, idx, 0x3);
2553   cbz(idx, L_post_third_loop_done);
2554 
2555   Label L_check_1;
2556   subsw(idx, idx, 2);
2557   br(Assembler::MI, L_check_1);
2558 
2559   lea(rscratch1, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2560   ldr(yz_idx1, Address(rscratch1, 0));
2561   ror(yz_idx1, yz_idx1, 32);
2562   mul(tmp3, product_hi, yz_idx1);  //  yz_idx1 * product_hi -> tmp4:tmp3
2563   umulh(tmp4, product_hi, yz_idx1);
2564   lea(rscratch1, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2565   ldr(yz_idx2, Address(rscratch1, 0));
2566   ror(yz_idx2, yz_idx2, 32);
2567 
2568   add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2);
2569 
2570   ror(tmp3, tmp3, 32);
2571   str(tmp3, Address(rscratch1, 0));
2572 
2573   bind (L_check_1);
2574 
2575   andw (idx, idx, 0x1);
2576   subsw(idx, idx, 1);
2577   br(Assembler::MI, L_post_third_loop_done);
2578   ldrw(tmp4, Address(y, idx, Address::uxtw(LogBytesPerInt)));
2579   mul(tmp3, tmp4, product_hi);  //  tmp4 * product_hi -> carry2:tmp3
2580   umulh(carry2, tmp4, product_hi);
2581   ldrw(tmp4, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2582 
2583   add2_with_carry(carry2, tmp3, tmp4, carry);
2584 
2585   strw(tmp3, Address(z, idx, Address::uxtw(LogBytesPerInt)));
2586   extr(carry, carry2, tmp3, 32);
2587 
2588   bind(L_post_third_loop_done);
2589 }
2590 
2591 /**
2592  * Code for BigInteger::multiplyToLen() instrinsic.
2593  *
2594  * r0: x
2595  * r1: xlen
2596  * r2: y
2597  * r3: ylen
2598  * r4:  z
2599  * r5: zlen
2600  * r10: tmp1
2601  * r11: tmp2
2602  * r12: tmp3
2603  * r13: tmp4
2604  * r14: tmp5
2605  * r15: tmp6
2606  * r16: tmp7
2607  *
2608  */
2609 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen,
2610                                      Register z, Register zlen,
2611                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4,
2612                                      Register tmp5, Register tmp6, Register product_hi) {
2613 
2614   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
2615 
2616   const Register idx = tmp1;
2617   const Register kdx = tmp2;
2618   const Register xstart = tmp3;
2619 
2620   const Register y_idx = tmp4;
2621   const Register carry = tmp5;
2622   const Register product  = xlen;
2623   const Register x_xstart = zlen;  // reuse register
2624 
2625   // First Loop.
2626   //
2627   //  final static long LONG_MASK = 0xffffffffL;
2628   //  int xstart = xlen - 1;
2629   //  int ystart = ylen - 1;
2630   //  long carry = 0;
2631   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
2632   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
2633   //    z[kdx] = (int)product;
2634   //    carry = product >>> 32;
2635   //  }
2636   //  z[xstart] = (int)carry;
2637   //
2638 
2639   movw(idx, ylen);      // idx = ylen;
2640   movw(kdx, zlen);      // kdx = xlen+ylen;
2641   mov(carry, zr);       // carry = 0;
2642 
2643   Label L_done;
2644 
2645   movw(xstart, xlen);
2646   subsw(xstart, xstart, 1);
2647   br(Assembler::MI, L_done);
2648 
2649   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
2650 
2651   Label L_second_loop;
2652   cbzw(kdx, L_second_loop);
2653 
2654   Label L_carry;
2655   subw(kdx, kdx, 1);
2656   cbzw(kdx, L_carry);
2657 
2658   strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt)));
2659   lsr(carry, carry, 32);
2660   subw(kdx, kdx, 1);
2661 
2662   bind(L_carry);
2663   strw(carry, Address(z, kdx, Address::uxtw(LogBytesPerInt)));
2664 
2665   // Second and third (nested) loops.
2666   //
2667   // for (int i = xstart-1; i >= 0; i--) { // Second loop
2668   //   carry = 0;
2669   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
2670   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
2671   //                    (z[k] & LONG_MASK) + carry;
2672   //     z[k] = (int)product;
2673   //     carry = product >>> 32;
2674   //   }
2675   //   z[i] = (int)carry;
2676   // }
2677   //
2678   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi
2679 
2680   const Register jdx = tmp1;
2681 
2682   bind(L_second_loop);
2683   mov(carry, zr);                // carry = 0;
2684   movw(jdx, ylen);               // j = ystart+1
2685 
2686   subsw(xstart, xstart, 1);      // i = xstart-1;
2687   br(Assembler::MI, L_done);
2688 
2689   str(z, Address(pre(sp, -4 * wordSize)));
2690 
2691   Label L_last_x;
2692   lea(z, offsetted_address(z, xstart, Address::uxtw(LogBytesPerInt), 4, BytesPerInt)); // z = z + k - j
2693   subsw(xstart, xstart, 1);       // i = xstart-1;
2694   br(Assembler::MI, L_last_x);
2695 
2696   lea(rscratch1, Address(x, xstart, Address::uxtw(LogBytesPerInt)));
2697   ldr(product_hi, Address(rscratch1));
2698   ror(product_hi, product_hi, 32);  // convert big-endian to little-endian
2699 
2700   Label L_third_loop_prologue;
2701   bind(L_third_loop_prologue);
2702 
2703   str(ylen, Address(sp, wordSize));
2704   stp(x, xstart, Address(sp, 2 * wordSize));
2705   multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product,
2706                           tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi);
2707   ldp(z, ylen, Address(post(sp, 2 * wordSize)));
2708   ldp(x, xlen, Address(post(sp, 2 * wordSize)));   // copy old xstart -> xlen
2709 
2710   addw(tmp3, xlen, 1);
2711   strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt)));
2712   subsw(tmp3, tmp3, 1);
2713   br(Assembler::MI, L_done);
2714 
2715   lsr(carry, carry, 32);
2716   strw(carry, Address(z, tmp3, Address::uxtw(LogBytesPerInt)));
2717   b(L_second_loop);
2718 
2719   // Next infrequent code is moved outside loops.
2720   bind(L_last_x);
2721   ldrw(product_hi, Address(x,  0));
2722   b(L_third_loop_prologue);
2723 
2724   bind(L_done);
2725 }
2726 
2727 /**
2728  * Emits code to update CRC-32 with a byte value according to constants in table
2729  *
2730  * @param [in,out]crc   Register containing the crc.
2731  * @param [in]val       Register containing the byte to fold into the CRC.
2732  * @param [in]table     Register containing the table of crc constants.
2733  *
2734  * uint32_t crc;
2735  * val = crc_table[(val ^ crc) & 0xFF];
2736  * crc = val ^ (crc >> 8);
2737  *
2738  */
2739 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
2740   eor(val, val, crc);
2741   andr(val, val, 0xff);
2742   ldrw(val, Address(table, val, Address::lsl(2)));
2743   eor(crc, val, crc, Assembler::LSR, 8);
2744 }
2745 
2746 /**
2747  * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3
2748  *
2749  * @param [in,out]crc   Register containing the crc.
2750  * @param [in]v         Register containing the 32-bit to fold into the CRC.
2751  * @param [in]table0    Register containing table 0 of crc constants.
2752  * @param [in]table1    Register containing table 1 of crc constants.
2753  * @param [in]table2    Register containing table 2 of crc constants.
2754  * @param [in]table3    Register containing table 3 of crc constants.
2755  *
2756  * uint32_t crc;
2757  *   v = crc ^ v
2758  *   crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24]
2759  *
2760  */
2761 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp,
2762         Register table0, Register table1, Register table2, Register table3,
2763         bool upper) {
2764   eor(v, crc, v, upper ? LSR:LSL, upper ? 32:0);
2765   uxtb(tmp, v);
2766   ldrw(crc, Address(table3, tmp, Address::lsl(2)));
2767   ubfx(tmp, v, 8, 8);
2768   ldrw(tmp, Address(table2, tmp, Address::lsl(2)));
2769   eor(crc, crc, tmp);
2770   ubfx(tmp, v, 16, 8);
2771   ldrw(tmp, Address(table1, tmp, Address::lsl(2)));
2772   eor(crc, crc, tmp);
2773   ubfx(tmp, v, 24, 8);
2774   ldrw(tmp, Address(table0, tmp, Address::lsl(2)));
2775   eor(crc, crc, tmp);
2776 }
2777 
2778 /**
2779  * @param crc   register containing existing CRC (32-bit)
2780  * @param buf   register pointing to input byte buffer (byte*)
2781  * @param len   register containing number of bytes
2782  * @param table register that will contain address of CRC table
2783  * @param tmp   scratch register
2784  */
2785 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
2786         Register table0, Register table1, Register table2, Register table3,
2787         Register tmp, Register tmp2, Register tmp3) {
2788   Label L_by16, L_by16_loop, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit;
2789   unsigned long offset;
2790 
2791     ornw(crc, zr, crc);
2792 
2793   if (UseCRC32) {
2794     Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop;
2795 
2796       subs(len, len, 64);
2797       br(Assembler::GE, CRC_by64_loop);
2798       adds(len, len, 64-4);
2799       br(Assembler::GE, CRC_by4_loop);
2800       adds(len, len, 4);
2801       br(Assembler::GT, CRC_by1_loop);
2802       b(L_exit);
2803 
2804     BIND(CRC_by4_loop);
2805       ldrw(tmp, Address(post(buf, 4)));
2806       subs(len, len, 4);
2807       crc32w(crc, crc, tmp);
2808       br(Assembler::GE, CRC_by4_loop);
2809       adds(len, len, 4);
2810       br(Assembler::LE, L_exit);
2811     BIND(CRC_by1_loop);
2812       ldrb(tmp, Address(post(buf, 1)));
2813       subs(len, len, 1);
2814       crc32b(crc, crc, tmp);
2815       br(Assembler::GT, CRC_by1_loop);
2816       b(L_exit);
2817 
2818       align(CodeEntryAlignment);
2819     BIND(CRC_by64_loop);
2820       subs(len, len, 64);
2821       ldp(tmp, tmp3, Address(post(buf, 16)));
2822       crc32x(crc, crc, tmp);
2823       crc32x(crc, crc, tmp3);
2824       ldp(tmp, tmp3, Address(post(buf, 16)));
2825       crc32x(crc, crc, tmp);
2826       crc32x(crc, crc, tmp3);
2827       ldp(tmp, tmp3, Address(post(buf, 16)));
2828       crc32x(crc, crc, tmp);
2829       crc32x(crc, crc, tmp3);
2830       ldp(tmp, tmp3, Address(post(buf, 16)));
2831       crc32x(crc, crc, tmp);
2832       crc32x(crc, crc, tmp3);
2833       br(Assembler::GE, CRC_by64_loop);
2834       adds(len, len, 64-4);
2835       br(Assembler::GE, CRC_by4_loop);
2836       adds(len, len, 4);
2837       br(Assembler::GT, CRC_by1_loop);
2838     BIND(L_exit);
2839       ornw(crc, zr, crc);
2840       return;
2841   }
2842 
2843     adrp(table0, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2844     if (offset) add(table0, table0, offset);
2845     add(table1, table0, 1*256*sizeof(juint));
2846     add(table2, table0, 2*256*sizeof(juint));
2847     add(table3, table0, 3*256*sizeof(juint));
2848 
2849   if (UseNeon) {
2850       cmp(len, 64);
2851       br(Assembler::LT, L_by16);
2852       eor(v16, T16B, v16, v16);
2853 
2854     Label L_fold;
2855 
2856       add(tmp, table0, 4*256*sizeof(juint)); // Point at the Neon constants
2857 
2858       ld1(v0, v1, T2D, post(buf, 32));
2859       ld1r(v4, T2D, post(tmp, 8));
2860       ld1r(v5, T2D, post(tmp, 8));
2861       ld1r(v6, T2D, post(tmp, 8));
2862       ld1r(v7, T2D, post(tmp, 8));
2863       mov(v16, T4S, 0, crc);
2864 
2865       eor(v0, T16B, v0, v16);
2866       sub(len, len, 64);
2867 
2868     BIND(L_fold);
2869       pmull(v22, T8H, v0, v5, T8B);
2870       pmull(v20, T8H, v0, v7, T8B);
2871       pmull(v23, T8H, v0, v4, T8B);
2872       pmull(v21, T8H, v0, v6, T8B);
2873 
2874       pmull2(v18, T8H, v0, v5, T16B);
2875       pmull2(v16, T8H, v0, v7, T16B);
2876       pmull2(v19, T8H, v0, v4, T16B);
2877       pmull2(v17, T8H, v0, v6, T16B);
2878 
2879       uzp1(v24, v20, v22, T8H);
2880       uzp2(v25, v20, v22, T8H);
2881       eor(v20, T16B, v24, v25);
2882 
2883       uzp1(v26, v16, v18, T8H);
2884       uzp2(v27, v16, v18, T8H);
2885       eor(v16, T16B, v26, v27);
2886 
2887       ushll2(v22, T4S, v20, T8H, 8);
2888       ushll(v20, T4S, v20, T4H, 8);
2889 
2890       ushll2(v18, T4S, v16, T8H, 8);
2891       ushll(v16, T4S, v16, T4H, 8);
2892 
2893       eor(v22, T16B, v23, v22);
2894       eor(v18, T16B, v19, v18);
2895       eor(v20, T16B, v21, v20);
2896       eor(v16, T16B, v17, v16);
2897 
2898       uzp1(v17, v16, v20, T2D);
2899       uzp2(v21, v16, v20, T2D);
2900       eor(v17, T16B, v17, v21);
2901 
2902       ushll2(v20, T2D, v17, T4S, 16);
2903       ushll(v16, T2D, v17, T2S, 16);
2904 
2905       eor(v20, T16B, v20, v22);
2906       eor(v16, T16B, v16, v18);
2907 
2908       uzp1(v17, v20, v16, T2D);
2909       uzp2(v21, v20, v16, T2D);
2910       eor(v28, T16B, v17, v21);
2911 
2912       pmull(v22, T8H, v1, v5, T8B);
2913       pmull(v20, T8H, v1, v7, T8B);
2914       pmull(v23, T8H, v1, v4, T8B);
2915       pmull(v21, T8H, v1, v6, T8B);
2916 
2917       pmull2(v18, T8H, v1, v5, T16B);
2918       pmull2(v16, T8H, v1, v7, T16B);
2919       pmull2(v19, T8H, v1, v4, T16B);
2920       pmull2(v17, T8H, v1, v6, T16B);
2921 
2922       ld1(v0, v1, T2D, post(buf, 32));
2923 
2924       uzp1(v24, v20, v22, T8H);
2925       uzp2(v25, v20, v22, T8H);
2926       eor(v20, T16B, v24, v25);
2927 
2928       uzp1(v26, v16, v18, T8H);
2929       uzp2(v27, v16, v18, T8H);
2930       eor(v16, T16B, v26, v27);
2931 
2932       ushll2(v22, T4S, v20, T8H, 8);
2933       ushll(v20, T4S, v20, T4H, 8);
2934 
2935       ushll2(v18, T4S, v16, T8H, 8);
2936       ushll(v16, T4S, v16, T4H, 8);
2937 
2938       eor(v22, T16B, v23, v22);
2939       eor(v18, T16B, v19, v18);
2940       eor(v20, T16B, v21, v20);
2941       eor(v16, T16B, v17, v16);
2942 
2943       uzp1(v17, v16, v20, T2D);
2944       uzp2(v21, v16, v20, T2D);
2945       eor(v16, T16B, v17, v21);
2946 
2947       ushll2(v20, T2D, v16, T4S, 16);
2948       ushll(v16, T2D, v16, T2S, 16);
2949 
2950       eor(v20, T16B, v22, v20);
2951       eor(v16, T16B, v16, v18);
2952 
2953       uzp1(v17, v20, v16, T2D);
2954       uzp2(v21, v20, v16, T2D);
2955       eor(v20, T16B, v17, v21);
2956 
2957       shl(v16, T2D, v28, 1);
2958       shl(v17, T2D, v20, 1);
2959 
2960       eor(v0, T16B, v0, v16);
2961       eor(v1, T16B, v1, v17);
2962 
2963       subs(len, len, 32);
2964       br(Assembler::GE, L_fold);
2965 
2966       mov(crc, 0);
2967       mov(tmp, v0, T1D, 0);
2968       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
2969       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
2970       mov(tmp, v0, T1D, 1);
2971       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
2972       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
2973       mov(tmp, v1, T1D, 0);
2974       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
2975       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
2976       mov(tmp, v1, T1D, 1);
2977       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
2978       update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
2979 
2980       add(len, len, 32);
2981   }
2982 
2983   BIND(L_by16);
2984     subs(len, len, 16);
2985     br(Assembler::GE, L_by16_loop);
2986     adds(len, len, 16-4);
2987     br(Assembler::GE, L_by4_loop);
2988     adds(len, len, 4);
2989     br(Assembler::GT, L_by1_loop);
2990     b(L_exit);
2991 
2992   BIND(L_by4_loop);
2993     ldrw(tmp, Address(post(buf, 4)));
2994     update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3);
2995     subs(len, len, 4);
2996     br(Assembler::GE, L_by4_loop);
2997     adds(len, len, 4);
2998     br(Assembler::LE, L_exit);
2999   BIND(L_by1_loop);
3000     subs(len, len, 1);
3001     ldrb(tmp, Address(post(buf, 1)));
3002     update_byte_crc32(crc, tmp, table0);
3003     br(Assembler::GT, L_by1_loop);
3004     b(L_exit);
3005 
3006     align(CodeEntryAlignment);
3007   BIND(L_by16_loop);
3008     subs(len, len, 16);
3009     ldp(tmp, tmp3, Address(post(buf, 16)));
3010     update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false);
3011     update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true);
3012     update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false);
3013     update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true);
3014     br(Assembler::GE, L_by16_loop);
3015     adds(len, len, 16-4);
3016     br(Assembler::GE, L_by4_loop);
3017     adds(len, len, 4);
3018     br(Assembler::GT, L_by1_loop);
3019   BIND(L_exit);
3020     ornw(crc, zr, crc);
3021 }
3022 
3023 /**
3024  * @param crc   register containing existing CRC (32-bit)
3025  * @param buf   register pointing to input byte buffer (byte*)
3026  * @param len   register containing number of bytes
3027  * @param table register that will contain address of CRC table
3028  * @param tmp   scratch register
3029  */
3030 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len,
3031         Register table0, Register table1, Register table2, Register table3,
3032         Register tmp, Register tmp2, Register tmp3) {
3033   Label L_exit;
3034   Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop;
3035 
3036     subs(len, len, 64);
3037     br(Assembler::GE, CRC_by64_loop);
3038     adds(len, len, 64-4);
3039     br(Assembler::GE, CRC_by4_loop);
3040     adds(len, len, 4);
3041     br(Assembler::GT, CRC_by1_loop);
3042     b(L_exit);
3043 
3044   BIND(CRC_by4_loop);
3045     ldrw(tmp, Address(post(buf, 4)));
3046     subs(len, len, 4);
3047     crc32cw(crc, crc, tmp);
3048     br(Assembler::GE, CRC_by4_loop);
3049     adds(len, len, 4);
3050     br(Assembler::LE, L_exit);
3051   BIND(CRC_by1_loop);
3052     ldrb(tmp, Address(post(buf, 1)));
3053     subs(len, len, 1);
3054     crc32cb(crc, crc, tmp);
3055     br(Assembler::GT, CRC_by1_loop);
3056     b(L_exit);
3057 
3058     align(CodeEntryAlignment);
3059   BIND(CRC_by64_loop);
3060     subs(len, len, 64);
3061     ldp(tmp, tmp3, Address(post(buf, 16)));
3062     crc32cx(crc, crc, tmp);
3063     crc32cx(crc, crc, tmp3);
3064     ldp(tmp, tmp3, Address(post(buf, 16)));
3065     crc32cx(crc, crc, tmp);
3066     crc32cx(crc, crc, tmp3);
3067     ldp(tmp, tmp3, Address(post(buf, 16)));
3068     crc32cx(crc, crc, tmp);
3069     crc32cx(crc, crc, tmp3);
3070     ldp(tmp, tmp3, Address(post(buf, 16)));
3071     crc32cx(crc, crc, tmp);
3072     crc32cx(crc, crc, tmp3);
3073     br(Assembler::GE, CRC_by64_loop);
3074     adds(len, len, 64-4);
3075     br(Assembler::GE, CRC_by4_loop);
3076     adds(len, len, 4);
3077     br(Assembler::GT, CRC_by1_loop);
3078   BIND(L_exit);
3079     return;
3080 }
3081 
3082 SkipIfEqual::SkipIfEqual(
3083     MacroAssembler* masm, const bool* flag_addr, bool value) {
3084   _masm = masm;
3085   unsigned long offset;
3086   _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset);
3087   _masm->ldrb(rscratch1, Address(rscratch1, offset));
3088   _masm->cbzw(rscratch1, _label);
3089 }
3090 
3091 SkipIfEqual::~SkipIfEqual() {
3092   _masm->bind(_label);
3093 }
3094 
3095 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3096   Address adr;
3097   switch(dst.getMode()) {
3098   case Address::base_plus_offset:
3099     // This is the expected mode, although we allow all the other
3100     // forms below.
3101     adr = form_address(rscratch2, dst.base(), dst.offset(), LogBytesPerWord);
3102     break;
3103   default:
3104     lea(rscratch2, dst);
3105     adr = Address(rscratch2);
3106     break;
3107   }
3108   ldr(rscratch1, adr);
3109   add(rscratch1, rscratch1, src);
3110   str(rscratch1, adr);
3111 }
3112 
3113 void MacroAssembler::cmpptr(Register src1, Address src2) {
3114   unsigned long offset;
3115   adrp(rscratch1, src2, offset);
3116   ldr(rscratch1, Address(rscratch1, offset));
3117   cmp(src1, rscratch1);
3118 }
3119 
3120 void MacroAssembler::store_check(Register obj, Address dst) {
3121   store_check(obj);
3122 }
3123 
3124 void MacroAssembler::store_check(Register obj) {
3125   // Does a store check for the oop in register obj. The content of
3126   // register obj is destroyed afterwards.
3127 
3128   BarrierSet* bs = Universe::heap()->barrier_set();
3129   assert(bs->kind() == BarrierSet::CardTableForRS ||
3130          bs->kind() == BarrierSet::CardTableExtension,
3131          "Wrong barrier set kind");
3132 
3133   CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
3134   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3135 
3136   lsr(obj, obj, CardTableModRefBS::card_shift);
3137 
3138   assert(CardTableModRefBS::dirty_card_val() == 0, "must be");
3139 
3140   load_byte_map_base(rscratch1);
3141 
3142   if (UseCondCardMark) {
3143     Label L_already_dirty;
3144     membar(StoreLoad);
3145     ldrb(rscratch2,  Address(obj, rscratch1));
3146     cbz(rscratch2, L_already_dirty);
3147     strb(zr, Address(obj, rscratch1));
3148     bind(L_already_dirty);
3149   } else {
3150     if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {
3151       membar(StoreStore);
3152     }
3153     strb(zr, Address(obj, rscratch1));
3154   }
3155 }
3156 
3157 void MacroAssembler::load_klass(Register dst, Register src) {
3158   if (UseCompressedClassPointers) {
3159     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3160     decode_klass_not_null(dst);
3161   } else {
3162     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3163   }
3164 }
3165 
3166 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3167   if (UseCompressedClassPointers) {
3168     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3169     if (Universe::narrow_klass_base() == NULL) {
3170       cmp(trial_klass, tmp, LSL, Universe::narrow_klass_shift());
3171       return;
3172     } else if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3173                && Universe::narrow_klass_shift() == 0) {
3174       // Only the bottom 32 bits matter
3175       cmpw(trial_klass, tmp);
3176       return;
3177     }
3178     decode_klass_not_null(tmp);
3179   } else {
3180     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3181   }
3182   cmp(trial_klass, tmp);
3183 }
3184 
3185 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3186   load_klass(dst, src);
3187   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3188 }
3189 
3190 void MacroAssembler::store_klass(Register dst, Register src) {
3191   // FIXME: Should this be a store release?  concurrent gcs assumes
3192   // klass length is valid if klass field is not null.
3193   if (UseCompressedClassPointers) {
3194     encode_klass_not_null(src);
3195     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3196   } else {
3197     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3198   }
3199 }
3200 
3201 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3202   if (UseCompressedClassPointers) {
3203     // Store to klass gap in destination
3204     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3205   }
3206 }
3207 
3208 // Algorithm must match oop.inline.hpp encode_heap_oop.
3209 void MacroAssembler::encode_heap_oop(Register d, Register s) {
3210 #ifdef ASSERT
3211   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
3212 #endif
3213   verify_oop(s, "broken oop in encode_heap_oop");
3214   if (Universe::narrow_oop_base() == NULL) {
3215     if (Universe::narrow_oop_shift() != 0) {
3216       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3217       lsr(d, s, LogMinObjAlignmentInBytes);
3218     } else {
3219       mov(d, s);
3220     }
3221   } else {
3222     subs(d, s, rheapbase);
3223     csel(d, d, zr, Assembler::HS);
3224     lsr(d, d, LogMinObjAlignmentInBytes);
3225 
3226     /*  Old algorithm: is this any worse?
3227     Label nonnull;
3228     cbnz(r, nonnull);
3229     sub(r, r, rheapbase);
3230     bind(nonnull);
3231     lsr(r, r, LogMinObjAlignmentInBytes);
3232     */
3233   }
3234 }
3235 
3236 void MacroAssembler::encode_heap_oop_not_null(Register r) {
3237 #ifdef ASSERT
3238   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
3239   if (CheckCompressedOops) {
3240     Label ok;
3241     cbnz(r, ok);
3242     stop("null oop passed to encode_heap_oop_not_null");
3243     bind(ok);
3244   }
3245 #endif
3246   verify_oop(r, "broken oop in encode_heap_oop_not_null");
3247   if (Universe::narrow_oop_base() != NULL) {
3248     sub(r, r, rheapbase);
3249   }
3250   if (Universe::narrow_oop_shift() != 0) {
3251     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3252     lsr(r, r, LogMinObjAlignmentInBytes);
3253   }
3254 }
3255 
3256 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
3257 #ifdef ASSERT
3258   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
3259   if (CheckCompressedOops) {
3260     Label ok;
3261     cbnz(src, ok);
3262     stop("null oop passed to encode_heap_oop_not_null2");
3263     bind(ok);
3264   }
3265 #endif
3266   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
3267 
3268   Register data = src;
3269   if (Universe::narrow_oop_base() != NULL) {
3270     sub(dst, src, rheapbase);
3271     data = dst;
3272   }
3273   if (Universe::narrow_oop_shift() != 0) {
3274     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3275     lsr(dst, data, LogMinObjAlignmentInBytes);
3276     data = dst;
3277   }
3278   if (data == src)
3279     mov(dst, src);
3280 }
3281 
3282 void  MacroAssembler::decode_heap_oop(Register d, Register s) {
3283 #ifdef ASSERT
3284   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
3285 #endif
3286   if (Universe::narrow_oop_base() == NULL) {
3287     if (Universe::narrow_oop_shift() != 0 || d != s) {
3288       lsl(d, s, Universe::narrow_oop_shift());
3289     }
3290   } else {
3291     Label done;
3292     if (d != s)
3293       mov(d, s);
3294     cbz(s, done);
3295     add(d, rheapbase, s, Assembler::LSL, LogMinObjAlignmentInBytes);
3296     bind(done);
3297   }
3298   verify_oop(d, "broken oop in decode_heap_oop");
3299 }
3300 
3301 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
3302   assert (UseCompressedOops, "should only be used for compressed headers");
3303   assert (Universe::heap() != NULL, "java heap should be initialized");
3304   // Cannot assert, unverified entry point counts instructions (see .ad file)
3305   // vtableStubs also counts instructions in pd_code_size_limit.
3306   // Also do not verify_oop as this is called by verify_oop.
3307   if (Universe::narrow_oop_shift() != 0) {
3308     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3309     if (Universe::narrow_oop_base() != NULL) {
3310       add(r, rheapbase, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3311     } else {
3312       add(r, zr, r, Assembler::LSL, LogMinObjAlignmentInBytes);
3313     }
3314   } else {
3315     assert (Universe::narrow_oop_base() == NULL, "sanity");
3316   }
3317 }
3318 
3319 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
3320   assert (UseCompressedOops, "should only be used for compressed headers");
3321   assert (Universe::heap() != NULL, "java heap should be initialized");
3322   // Cannot assert, unverified entry point counts instructions (see .ad file)
3323   // vtableStubs also counts instructions in pd_code_size_limit.
3324   // Also do not verify_oop as this is called by verify_oop.
3325   if (Universe::narrow_oop_shift() != 0) {
3326     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
3327     if (Universe::narrow_oop_base() != NULL) {
3328       add(dst, rheapbase, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3329     } else {
3330       add(dst, zr, src, Assembler::LSL, LogMinObjAlignmentInBytes);
3331     }
3332   } else {
3333     assert (Universe::narrow_oop_base() == NULL, "sanity");
3334     if (dst != src) {
3335       mov(dst, src);
3336     }
3337   }
3338 }
3339 
3340 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3341   if (Universe::narrow_klass_base() == NULL) {
3342     if (Universe::narrow_klass_shift() != 0) {
3343       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3344       lsr(dst, src, LogKlassAlignmentInBytes);
3345     } else {
3346       if (dst != src) mov(dst, src);
3347     }
3348     return;
3349   }
3350 
3351   if (use_XOR_for_compressed_class_base) {
3352     if (Universe::narrow_klass_shift() != 0) {
3353       eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3354       lsr(dst, dst, LogKlassAlignmentInBytes);
3355     } else {
3356       eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3357     }
3358     return;
3359   }
3360 
3361   if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3362       && Universe::narrow_klass_shift() == 0) {
3363     movw(dst, src);
3364     return;
3365   }
3366 
3367 #ifdef ASSERT
3368   verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
3369 #endif
3370 
3371   Register rbase = dst;
3372   if (dst == src) rbase = rheapbase;
3373   mov(rbase, (uint64_t)Universe::narrow_klass_base());
3374   sub(dst, src, rbase);
3375   if (Universe::narrow_klass_shift() != 0) {
3376     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3377     lsr(dst, dst, LogKlassAlignmentInBytes);
3378   }
3379   if (dst == src) reinit_heapbase();
3380 }
3381 
3382 void MacroAssembler::encode_klass_not_null(Register r) {
3383   encode_klass_not_null(r, r);
3384 }
3385 
3386 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3387   Register rbase = dst;
3388   assert (UseCompressedClassPointers, "should only be used for compressed headers");
3389 
3390   if (Universe::narrow_klass_base() == NULL) {
3391     if (Universe::narrow_klass_shift() != 0) {
3392       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3393       lsl(dst, src, LogKlassAlignmentInBytes);
3394     } else {
3395       if (dst != src) mov(dst, src);
3396     }
3397     return;
3398   }
3399 
3400   if (use_XOR_for_compressed_class_base) {
3401     if (Universe::narrow_klass_shift() != 0) {
3402       lsl(dst, src, LogKlassAlignmentInBytes);
3403       eor(dst, dst, (uint64_t)Universe::narrow_klass_base());
3404     } else {
3405       eor(dst, src, (uint64_t)Universe::narrow_klass_base());
3406     }
3407     return;
3408   }
3409 
3410   if (((uint64_t)Universe::narrow_klass_base() & 0xffffffff) == 0
3411       && Universe::narrow_klass_shift() == 0) {
3412     if (dst != src)
3413       movw(dst, src);
3414     movk(dst, (uint64_t)Universe::narrow_klass_base() >> 32, 32);
3415     return;
3416   }
3417 
3418   // Cannot assert, unverified entry point counts instructions (see .ad file)
3419   // vtableStubs also counts instructions in pd_code_size_limit.
3420   // Also do not verify_oop as this is called by verify_oop.
3421   if (dst == src) rbase = rheapbase;
3422   mov(rbase, (uint64_t)Universe::narrow_klass_base());
3423   if (Universe::narrow_klass_shift() != 0) {
3424     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
3425     add(dst, rbase, src, Assembler::LSL, LogKlassAlignmentInBytes);
3426   } else {
3427     add(dst, rbase, src);
3428   }
3429   if (dst == src) reinit_heapbase();
3430 }
3431 
3432 void  MacroAssembler::decode_klass_not_null(Register r) {
3433   decode_klass_not_null(r, r);
3434 }
3435 
3436 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
3437   assert (UseCompressedOops, "should only be used for compressed oops");
3438   assert (Universe::heap() != NULL, "java heap should be initialized");
3439   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3440 
3441   int oop_index = oop_recorder()->find_index(obj);
3442   assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
3443 
3444   InstructionMark im(this);
3445   RelocationHolder rspec = oop_Relocation::spec(oop_index);
3446   code_section()->relocate(inst_mark(), rspec);
3447   movz(dst, 0xDEAD, 16);
3448   movk(dst, 0xBEEF);
3449 }
3450 
3451 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
3452   assert (UseCompressedClassPointers, "should only be used for compressed headers");
3453   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
3454   int index = oop_recorder()->find_index(k);
3455   assert(! Universe::heap()->is_in_reserved(k), "should not be an oop");
3456 
3457   InstructionMark im(this);
3458   RelocationHolder rspec = metadata_Relocation::spec(index);
3459   code_section()->relocate(inst_mark(), rspec);
3460   narrowKlass nk = Klass::encode_klass(k);
3461   movz(dst, (nk >> 16), 16);
3462   movk(dst, nk & 0xffff);
3463 }
3464 
3465 void MacroAssembler::load_heap_oop(Register dst, Address src)
3466 {
3467   if (UseCompressedOops) {
3468     ldrw(dst, src);
3469     decode_heap_oop(dst);
3470   } else {
3471     ldr(dst, src);
3472   }
3473 }
3474 
3475 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src)
3476 {
3477   if (UseCompressedOops) {
3478     ldrw(dst, src);
3479     decode_heap_oop_not_null(dst);
3480   } else {
3481     ldr(dst, src);
3482   }
3483 }
3484 
3485 void MacroAssembler::store_heap_oop(Address dst, Register src) {
3486   if (UseCompressedOops) {
3487     assert(!dst.uses(src), "not enough registers");
3488     encode_heap_oop(src);
3489     strw(src, dst);
3490   } else
3491     str(src, dst);
3492 }
3493 
3494 // Used for storing NULLs.
3495 void MacroAssembler::store_heap_oop_null(Address dst) {
3496   if (UseCompressedOops) {
3497     strw(zr, dst);
3498   } else
3499     str(zr, dst);
3500 }
3501 
3502 #if INCLUDE_ALL_GCS
3503 void MacroAssembler::g1_write_barrier_pre(Register obj,
3504                                           Register pre_val,
3505                                           Register thread,
3506                                           Register tmp,
3507                                           bool tosca_live,
3508                                           bool expand_call) {
3509   // If expand_call is true then we expand the call_VM_leaf macro
3510   // directly to skip generating the check by
3511   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
3512 
3513   assert(thread == rthread, "must be");
3514 
3515   Label done;
3516   Label runtime;
3517 
3518   assert(pre_val != noreg, "check this code");
3519 
3520   if (obj != noreg)
3521     assert_different_registers(obj, pre_val, tmp);
3522 
3523   Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
3524                                        SATBMarkQueue::byte_offset_of_active()));
3525   Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
3526                                        SATBMarkQueue::byte_offset_of_index()));
3527   Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
3528                                        SATBMarkQueue::byte_offset_of_buf()));
3529 
3530 
3531   // Is marking active?
3532   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
3533     ldrw(tmp, in_progress);
3534   } else {
3535     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
3536     ldrb(tmp, in_progress);
3537   }
3538   cbzw(tmp, done);
3539 
3540   // Do we need to load the previous value?
3541   if (obj != noreg) {
3542     load_heap_oop(pre_val, Address(obj, 0));
3543   }
3544 
3545   // Is the previous value null?
3546   cbz(pre_val, done);
3547 
3548   // Can we store original value in the thread's buffer?
3549   // Is index == 0?
3550   // (The index field is typed as size_t.)
3551 
3552   ldr(tmp, index);                      // tmp := *index_adr
3553   cbz(tmp, runtime);                    // tmp == 0?
3554                                         // If yes, goto runtime
3555 
3556   sub(tmp, tmp, wordSize);              // tmp := tmp - wordSize
3557   str(tmp, index);                      // *index_adr := tmp
3558   ldr(rscratch1, buffer);
3559   add(tmp, tmp, rscratch1);             // tmp := tmp + *buffer_adr
3560 
3561   // Record the previous value
3562   str(pre_val, Address(tmp, 0));
3563   b(done);
3564 
3565   bind(runtime);
3566   // save the live input values
3567   push(r0->bit(tosca_live) | obj->bit(obj != noreg) | pre_val->bit(true), sp);
3568 
3569   // Calling the runtime using the regular call_VM_leaf mechanism generates
3570   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
3571   // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
3572   //
3573   // If we care generating the pre-barrier without a frame (e.g. in the
3574   // intrinsified Reference.get() routine) then ebp might be pointing to
3575   // the caller frame and so this check will most likely fail at runtime.
3576   //
3577   // Expanding the call directly bypasses the generation of the check.
3578   // So when we do not have have a full interpreter frame on the stack
3579   // expand_call should be passed true.
3580 
3581   if (expand_call) {
3582     assert(pre_val != c_rarg1, "smashed arg");
3583     pass_arg1(this, thread);
3584     pass_arg0(this, pre_val);
3585     MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
3586   } else {
3587     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
3588   }
3589 
3590   pop(r0->bit(tosca_live) | obj->bit(obj != noreg) | pre_val->bit(true), sp);
3591 
3592   bind(done);
3593 }
3594 
3595 void MacroAssembler::g1_write_barrier_post(Register store_addr,
3596                                            Register new_val,
3597                                            Register thread,
3598                                            Register tmp,
3599                                            Register tmp2) {
3600   assert(thread == rthread, "must be");
3601 
3602   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
3603                                        DirtyCardQueue::byte_offset_of_index()));
3604   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
3605                                        DirtyCardQueue::byte_offset_of_buf()));
3606 
3607   BarrierSet* bs = Universe::heap()->barrier_set();
3608   CardTableModRefBS* ct = (CardTableModRefBS*)bs;
3609   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3610 
3611   Label done;
3612   Label runtime;
3613 
3614   // Does store cross heap regions?
3615 
3616   eor(tmp, store_addr, new_val);
3617   lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
3618   cbz(tmp, done);
3619 
3620   // crosses regions, storing NULL?
3621 
3622   cbz(new_val, done);
3623 
3624   // storing region crossing non-NULL, is card already dirty?
3625 
3626   ExternalAddress cardtable((address) ct->byte_map_base);
3627   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3628   const Register card_addr = tmp;
3629 
3630   lsr(card_addr, store_addr, CardTableModRefBS::card_shift);
3631 
3632   // get the address of the card
3633   load_byte_map_base(tmp2);
3634   add(card_addr, card_addr, tmp2);
3635   ldrb(tmp2, Address(card_addr));
3636   cmpw(tmp2, (int)G1SATBCardTableModRefBS::g1_young_card_val());
3637   br(Assembler::EQ, done);
3638 
3639   assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
3640 
3641   membar(Assembler::StoreLoad);
3642 
3643   ldrb(tmp2, Address(card_addr));
3644   cbzw(tmp2, done);
3645 
3646   // storing a region crossing, non-NULL oop, card is clean.
3647   // dirty card and log.
3648 
3649   strb(zr, Address(card_addr));
3650 
3651   ldr(rscratch1, queue_index);
3652   cbz(rscratch1, runtime);
3653   sub(rscratch1, rscratch1, wordSize);
3654   str(rscratch1, queue_index);
3655 
3656   ldr(tmp2, buffer);
3657   str(card_addr, Address(tmp2, rscratch1));
3658   b(done);
3659 
3660   bind(runtime);
3661   // save the live input values
3662   push(store_addr->bit(true) | new_val->bit(true), sp);
3663   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
3664   pop(store_addr->bit(true) | new_val->bit(true), sp);
3665 
3666   bind(done);
3667 }
3668 
3669 #endif // INCLUDE_ALL_GCS
3670 
3671 Address MacroAssembler::allocate_metadata_address(Metadata* obj) {
3672   assert(oop_recorder() != NULL, "this assembler needs a Recorder");
3673   int index = oop_recorder()->allocate_metadata_index(obj);
3674   RelocationHolder rspec = metadata_Relocation::spec(index);
3675   return Address((address)obj, rspec);
3676 }
3677 
3678 // Move an oop into a register.  immediate is true if we want
3679 // immediate instrcutions, i.e. we are not going to patch this
3680 // instruction while the code is being executed by another thread.  In
3681 // that case we can use move immediates rather than the constant pool.
3682 void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) {
3683   int oop_index;
3684   if (obj == NULL) {
3685     oop_index = oop_recorder()->allocate_oop_index(obj);
3686   } else {
3687     oop_index = oop_recorder()->find_index(obj);
3688     assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "should be real oop");
3689   }
3690   RelocationHolder rspec = oop_Relocation::spec(oop_index);
3691   if (! immediate) {
3692     address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
3693     ldr_constant(dst, Address(dummy, rspec));
3694   } else
3695     mov(dst, Address((address)obj, rspec));
3696 }
3697 
3698 // Move a metadata address into a register.
3699 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
3700   int oop_index;
3701   if (obj == NULL) {
3702     oop_index = oop_recorder()->allocate_metadata_index(obj);
3703   } else {
3704     oop_index = oop_recorder()->find_index(obj);
3705   }
3706   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
3707   mov(dst, Address((address)obj, rspec));
3708 }
3709 
3710 Address MacroAssembler::constant_oop_address(jobject obj) {
3711   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
3712   assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(obj)), "not an oop");
3713   int oop_index = oop_recorder()->find_index(obj);
3714   return Address((address)obj, oop_Relocation::spec(oop_index));
3715 }
3716 
3717 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3718 void MacroAssembler::tlab_allocate(Register obj,
3719                                    Register var_size_in_bytes,
3720                                    int con_size_in_bytes,
3721                                    Register t1,
3722                                    Register t2,
3723                                    Label& slow_case) {
3724   assert_different_registers(obj, t2);
3725   assert_different_registers(obj, var_size_in_bytes);
3726   Register end = t2;
3727 
3728   // verify_tlab();
3729 
3730   ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
3731   if (var_size_in_bytes == noreg) {
3732     lea(end, Address(obj, con_size_in_bytes));
3733   } else {
3734     lea(end, Address(obj, var_size_in_bytes));
3735   }
3736   ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
3737   cmp(end, rscratch1);
3738   br(Assembler::HI, slow_case);
3739 
3740   // update the tlab top pointer
3741   str(end, Address(rthread, JavaThread::tlab_top_offset()));
3742 
3743   // recover var_size_in_bytes if necessary
3744   if (var_size_in_bytes == end) {
3745     sub(var_size_in_bytes, var_size_in_bytes, obj);
3746   }
3747   // verify_tlab();
3748 }
3749 
3750 // Preserves r19, and r3.
3751 Register MacroAssembler::tlab_refill(Label& retry,
3752                                      Label& try_eden,
3753                                      Label& slow_case) {
3754   Register top = r0;
3755   Register t1  = r2;
3756   Register t2  = r4;
3757   assert_different_registers(top, rthread, t1, t2, /* preserve: */ r19, r3);
3758   Label do_refill, discard_tlab;
3759 
3760   if (!Universe::heap()->supports_inline_contig_alloc()) {
3761     // No allocation in the shared eden.
3762     b(slow_case);
3763   }
3764 
3765   ldr(top, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
3766   ldr(t1,  Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
3767 
3768   // calculate amount of free space
3769   sub(t1, t1, top);
3770   lsr(t1, t1, LogHeapWordSize);
3771 
3772   // Retain tlab and allocate object in shared space if
3773   // the amount free in the tlab is too large to discard.
3774 
3775   ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
3776   cmp(t1, rscratch1);
3777   br(Assembler::LE, discard_tlab);
3778 
3779   // Retain
3780   // ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
3781   mov(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
3782   add(rscratch1, rscratch1, t2);
3783   str(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
3784 
3785   if (TLABStats) {
3786     // increment number of slow_allocations
3787     addmw(Address(rthread, in_bytes(JavaThread::tlab_slow_allocations_offset())),
3788          1, rscratch1);
3789   }
3790   b(try_eden);
3791 
3792   bind(discard_tlab);
3793   if (TLABStats) {
3794     // increment number of refills
3795     addmw(Address(rthread, in_bytes(JavaThread::tlab_number_of_refills_offset())), 1,
3796          rscratch1);
3797     // accumulate wastage -- t1 is amount free in tlab
3798     addmw(Address(rthread, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1,
3799          rscratch1);
3800   }
3801 
3802   // if tlab is currently allocated (top or end != null) then
3803   // fill [top, end + alignment_reserve) with array object
3804   cbz(top, do_refill);
3805 
3806   // set up the mark word
3807   mov(rscratch1, (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2));
3808   str(rscratch1, Address(top, oopDesc::mark_offset_in_bytes()));
3809   // set the length to the remaining space
3810   sub(t1, t1, typeArrayOopDesc::header_size(T_INT));
3811   add(t1, t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve());
3812   lsl(t1, t1, log2_intptr(HeapWordSize/sizeof(jint)));
3813   strw(t1, Address(top, arrayOopDesc::length_offset_in_bytes()));
3814   // set klass to intArrayKlass
3815   {
3816     unsigned long offset;
3817     // dubious reloc why not an oop reloc?
3818     adrp(rscratch1, ExternalAddress((address)Universe::intArrayKlassObj_addr()),
3819          offset);
3820     ldr(t1, Address(rscratch1, offset));
3821   }
3822   // store klass last.  concurrent gcs assumes klass length is valid if
3823   // klass field is not null.
3824   store_klass(top, t1);
3825 
3826   mov(t1, top);
3827   ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
3828   sub(t1, t1, rscratch1);
3829   incr_allocated_bytes(rthread, t1, 0, rscratch1);
3830 
3831   // refill the tlab with an eden allocation
3832   bind(do_refill);
3833   ldr(t1, Address(rthread, in_bytes(JavaThread::tlab_size_offset())));
3834   lsl(t1, t1, LogHeapWordSize);
3835   // allocate new tlab, address returned in top
3836   eden_allocate(top, t1, 0, t2, slow_case);
3837 
3838   // Check that t1 was preserved in eden_allocate.
3839 #ifdef ASSERT
3840   if (UseTLAB) {
3841     Label ok;
3842     Register tsize = r4;
3843     assert_different_registers(tsize, rthread, t1);
3844     str(tsize, Address(pre(sp, -16)));
3845     ldr(tsize, Address(rthread, in_bytes(JavaThread::tlab_size_offset())));
3846     lsl(tsize, tsize, LogHeapWordSize);
3847     cmp(t1, tsize);
3848     br(Assembler::EQ, ok);
3849     STOP("assert(t1 != tlab size)");
3850     should_not_reach_here();
3851 
3852     bind(ok);
3853     ldr(tsize, Address(post(sp, 16)));
3854   }
3855 #endif
3856   str(top, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
3857   str(top, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
3858   add(top, top, t1);
3859   sub(top, top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
3860   str(top, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
3861   verify_tlab();
3862   b(retry);
3863 
3864   return rthread; // for use by caller
3865 }
3866 
3867 // Defines obj, preserves var_size_in_bytes
3868 void MacroAssembler::eden_allocate(Register obj,
3869                                    Register var_size_in_bytes,
3870                                    int con_size_in_bytes,
3871                                    Register t1,
3872                                    Label& slow_case) {
3873   assert_different_registers(obj, var_size_in_bytes, t1);
3874   if (!Universe::heap()->supports_inline_contig_alloc()) {
3875     b(slow_case);
3876   } else {
3877     Register end = t1;
3878     Register heap_end = rscratch2;
3879     Label retry;
3880     bind(retry);
3881     {
3882       unsigned long offset;
3883       adrp(rscratch1, ExternalAddress((address) Universe::heap()->end_addr()), offset);
3884       ldr(heap_end, Address(rscratch1, offset));
3885     }
3886 
3887     ExternalAddress heap_top((address) Universe::heap()->top_addr());
3888 
3889     // Get the current top of the heap
3890     {
3891       unsigned long offset;
3892       adrp(rscratch1, heap_top, offset);
3893       // Use add() here after ARDP, rather than lea().
3894       // lea() does not generate anything if its offset is zero.
3895       // However, relocs expect to find either an ADD or a load/store
3896       // insn after an ADRP.  add() always generates an ADD insn, even
3897       // for add(Rn, Rn, 0).
3898       add(rscratch1, rscratch1, offset);
3899       ldaxr(obj, rscratch1);
3900     }
3901 
3902     // Adjust it my the size of our new object
3903     if (var_size_in_bytes == noreg) {
3904       lea(end, Address(obj, con_size_in_bytes));
3905     } else {
3906       lea(end, Address(obj, var_size_in_bytes));
3907     }
3908 
3909     // if end < obj then we wrapped around high memory
3910     cmp(end, obj);
3911     br(Assembler::LO, slow_case);
3912 
3913     cmp(end, heap_end);
3914     br(Assembler::HI, slow_case);
3915 
3916     // If heap_top hasn't been changed by some other thread, update it.
3917     stlxr(rscratch2, end, rscratch1);
3918     cbnzw(rscratch2, retry);
3919   }
3920 }
3921 
3922 void MacroAssembler::verify_tlab() {
3923 #ifdef ASSERT
3924   if (UseTLAB && VerifyOops) {
3925     Label next, ok;
3926 
3927     stp(rscratch2, rscratch1, Address(pre(sp, -16)));
3928 
3929     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
3930     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
3931     cmp(rscratch2, rscratch1);
3932     br(Assembler::HS, next);
3933     STOP("assert(top >= start)");
3934     should_not_reach_here();
3935 
3936     bind(next);
3937     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
3938     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
3939     cmp(rscratch2, rscratch1);
3940     br(Assembler::HS, ok);
3941     STOP("assert(top <= end)");
3942     should_not_reach_here();
3943 
3944     bind(ok);
3945     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
3946   }
3947 #endif
3948 }
3949 
3950 // Writes to stack successive pages until offset reached to check for
3951 // stack overflow + shadow pages.  This clobbers tmp.
3952 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
3953   assert_different_registers(tmp, size, rscratch1);
3954   mov(tmp, sp);
3955   // Bang stack for total size given plus shadow page size.
3956   // Bang one page at a time because large size can bang beyond yellow and
3957   // red zones.
3958   Label loop;
3959   mov(rscratch1, os::vm_page_size());
3960   bind(loop);
3961   lea(tmp, Address(tmp, -os::vm_page_size()));
3962   subsw(size, size, rscratch1);
3963   str(size, Address(tmp));
3964   br(Assembler::GT, loop);
3965 
3966   // Bang down shadow pages too.
3967   // At this point, (tmp-0) is the last address touched, so don't
3968   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
3969   // was post-decremented.)  Skip this address by starting at i=1, and
3970   // touch a few more pages below.  N.B.  It is important to touch all
3971   // the way down to and including i=StackShadowPages.
3972   for (int i = 0; i < (int)(JavaThread::stack_shadow_zone_size() / os::vm_page_size()) - 1; i++) {
3973     // this could be any sized move but this is can be a debugging crumb
3974     // so the bigger the better.
3975     lea(tmp, Address(tmp, -os::vm_page_size()));
3976     str(size, Address(tmp));
3977   }
3978 }
3979 
3980 
3981 address MacroAssembler::read_polling_page(Register r, address page, relocInfo::relocType rtype) {
3982   unsigned long off;
3983   adrp(r, Address(page, rtype), off);
3984   InstructionMark im(this);
3985   code_section()->relocate(inst_mark(), rtype);
3986   ldrw(zr, Address(r, off));
3987   return inst_mark();
3988 }
3989 
3990 address MacroAssembler::read_polling_page(Register r, relocInfo::relocType rtype) {
3991   InstructionMark im(this);
3992   code_section()->relocate(inst_mark(), rtype);
3993   ldrw(zr, Address(r, 0));
3994   return inst_mark();
3995 }
3996 
3997 void MacroAssembler::adrp(Register reg1, const Address &dest, unsigned long &byte_offset) {
3998   relocInfo::relocType rtype = dest.rspec().reloc()->type();
3999   unsigned long low_page = (unsigned long)CodeCache::low_bound() >> 12;
4000   unsigned long high_page = (unsigned long)(CodeCache::high_bound()-1) >> 12;
4001   unsigned long dest_page = (unsigned long)dest.target() >> 12;
4002   long offset_low = dest_page - low_page;
4003   long offset_high = dest_page - high_page;
4004 
4005   assert(is_valid_AArch64_address(dest.target()), "bad address");
4006   assert(dest.getMode() == Address::literal, "ADRP must be applied to a literal address");
4007 
4008   InstructionMark im(this);
4009   code_section()->relocate(inst_mark(), dest.rspec());
4010   // 8143067: Ensure that the adrp can reach the dest from anywhere within
4011   // the code cache so that if it is relocated we know it will still reach
4012   if (offset_high >= -(1<<20) && offset_low < (1<<20)) {
4013     _adrp(reg1, dest.target());
4014   } else {
4015     unsigned long target = (unsigned long)dest.target();
4016     unsigned long adrp_target
4017       = (target & 0xffffffffUL) | ((unsigned long)pc() & 0xffff00000000UL);
4018 
4019     _adrp(reg1, (address)adrp_target);
4020     movk(reg1, target >> 32, 32);
4021   }
4022   byte_offset = (unsigned long)dest.target() & 0xfff;
4023 }
4024 
4025 void MacroAssembler::load_byte_map_base(Register reg) {
4026   jbyte *byte_map_base =
4027     ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base;
4028 
4029   if (is_valid_AArch64_address((address)byte_map_base)) {
4030     // Strictly speaking the byte_map_base isn't an address at all,
4031     // and it might even be negative.
4032     unsigned long offset;
4033     adrp(reg, ExternalAddress((address)byte_map_base), offset);
4034     assert(offset == 0, "misaligned card table base");
4035   } else {
4036     mov(reg, (uint64_t)byte_map_base);
4037   }
4038 }
4039 
4040 void MacroAssembler::build_frame(int framesize) {
4041   assert(framesize > 0, "framesize must be > 0");
4042   if (framesize < ((1 << 9) + 2 * wordSize)) {
4043     sub(sp, sp, framesize);
4044     stp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4045     if (PreserveFramePointer) add(rfp, sp, framesize - 2 * wordSize);
4046   } else {
4047     stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
4048     if (PreserveFramePointer) mov(rfp, sp);
4049     if (framesize < ((1 << 12) + 2 * wordSize))
4050       sub(sp, sp, framesize - 2 * wordSize);
4051     else {
4052       mov(rscratch1, framesize - 2 * wordSize);
4053       sub(sp, sp, rscratch1);
4054     }
4055   }
4056 }
4057 
4058 void MacroAssembler::remove_frame(int framesize) {
4059   assert(framesize > 0, "framesize must be > 0");
4060   if (framesize < ((1 << 9) + 2 * wordSize)) {
4061     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4062     add(sp, sp, framesize);
4063   } else {
4064     if (framesize < ((1 << 12) + 2 * wordSize))
4065       add(sp, sp, framesize - 2 * wordSize);
4066     else {
4067       mov(rscratch1, framesize - 2 * wordSize);
4068       add(sp, sp, rscratch1);
4069     }
4070     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4071   }
4072 }
4073 
4074 
4075 // Search for str1 in str2 and return index or -1
4076 void MacroAssembler::string_indexof(Register str2, Register str1,
4077                                     Register cnt2, Register cnt1,
4078                                     Register tmp1, Register tmp2,
4079                                     Register tmp3, Register tmp4,
4080                                     int icnt1, Register result) {
4081   Label BM, LINEARSEARCH, DONE, NOMATCH, MATCH;
4082 
4083   Register ch1 = rscratch1;
4084   Register ch2 = rscratch2;
4085   Register cnt1tmp = tmp1;
4086   Register cnt2tmp = tmp2;
4087   Register cnt1_neg = cnt1;
4088   Register cnt2_neg = cnt2;
4089   Register result_tmp = tmp4;
4090 
4091   // Note, inline_string_indexOf() generates checks:
4092   // if (substr.count > string.count) return -1;
4093   // if (substr.count == 0) return 0;
4094 
4095 // We have two strings, a source string in str2, cnt2 and a pattern string
4096 // in str1, cnt1. Find the 1st occurence of pattern in source or return -1.
4097 
4098 // For larger pattern and source we use a simplified Boyer Moore algorithm.
4099 // With a small pattern and source we use linear scan.
4100 
4101   if (icnt1 == -1) {
4102     cmp(cnt1, 256);             // Use Linear Scan if cnt1 < 8 || cnt1 >= 256
4103     ccmp(cnt1, 8, 0b0000, LO);  // Can't handle skip >= 256 because we use
4104     br(LO, LINEARSEARCH);       // a byte array.
4105     cmp(cnt1, cnt2, LSR, 2);    // Source must be 4 * pattern for BM
4106     br(HS, LINEARSEARCH);
4107   }
4108 
4109 // The Boyer Moore alogorithm is based on the description here:-
4110 //
4111 // http://en.wikipedia.org/wiki/Boyer%E2%80%93Moore_string_search_algorithm
4112 //
4113 // This describes and algorithm with 2 shift rules. The 'Bad Character' rule
4114 // and the 'Good Suffix' rule.
4115 //
4116 // These rules are essentially heuristics for how far we can shift the
4117 // pattern along the search string.
4118 //
4119 // The implementation here uses the 'Bad Character' rule only because of the
4120 // complexity of initialisation for the 'Good Suffix' rule.
4121 //
4122 // This is also known as the Boyer-Moore-Horspool algorithm:-
4123 //
4124 // http://en.wikipedia.org/wiki/Boyer-Moore-Horspool_algorithm
4125 //
4126 // #define ASIZE 128
4127 //
4128 //    int bm(unsigned char *x, int m, unsigned char *y, int n) {
4129 //       int i, j;
4130 //       unsigned c;
4131 //       unsigned char bc[ASIZE];
4132 //
4133 //       /* Preprocessing */
4134 //       for (i = 0; i < ASIZE; ++i)
4135 //          bc[i] = 0;
4136 //       for (i = 0; i < m - 1; ) {
4137 //          c = x[i];
4138 //          ++i;
4139 //          if (c < ASIZE) bc[c] = i;
4140 //       }
4141 //
4142 //       /* Searching */
4143 //       j = 0;
4144 //       while (j <= n - m) {
4145 //          c = y[i+j];
4146 //          if (x[m-1] == c)
4147 //            for (i = m - 2; i >= 0 && x[i] == y[i + j]; --i);
4148 //          if (i < 0) return j;
4149 //          if (c < ASIZE)
4150 //            j = j - bc[y[j+m-1]] + m;
4151 //          else
4152 //            j += 1; // Advance by 1 only if char >= ASIZE
4153 //       }
4154 //    }
4155 
4156   if (icnt1 == -1) {
4157     BIND(BM);
4158 
4159     Label ZLOOP, BCLOOP, BCSKIP, BMLOOPSTR2, BMLOOPSTR1, BMSKIP;
4160     Label BMADV, BMMATCH, BMCHECKEND;
4161 
4162     Register cnt1end = tmp2;
4163     Register str2end = cnt2;
4164     Register skipch = tmp2;
4165 
4166     // Restrict ASIZE to 128 to reduce stack space/initialisation.
4167     // The presence of chars >= ASIZE in the target string does not affect
4168     // performance, but we must be careful not to initialise them in the stack
4169     // array.
4170     // The presence of chars >= ASIZE in the source string may adversely affect
4171     // performance since we can only advance by one when we encounter one.
4172 
4173       stp(zr, zr, pre(sp, -128));
4174       for (int i = 1; i < 8; i++)
4175           stp(zr, zr, Address(sp, i*16));
4176 
4177       mov(cnt1tmp, 0);
4178       sub(cnt1end, cnt1, 1);
4179     BIND(BCLOOP);
4180       ldrh(ch1, Address(str1, cnt1tmp, Address::lsl(1)));
4181       cmp(ch1, 128);
4182       add(cnt1tmp, cnt1tmp, 1);
4183       br(HS, BCSKIP);
4184       strb(cnt1tmp, Address(sp, ch1));
4185     BIND(BCSKIP);
4186       cmp(cnt1tmp, cnt1end);
4187       br(LT, BCLOOP);
4188 
4189       mov(result_tmp, str2);
4190 
4191       sub(cnt2, cnt2, cnt1);
4192       add(str2end, str2, cnt2, LSL, 1);
4193     BIND(BMLOOPSTR2);
4194       sub(cnt1tmp, cnt1, 1);
4195       ldrh(ch1, Address(str1, cnt1tmp, Address::lsl(1)));
4196       ldrh(skipch, Address(str2, cnt1tmp, Address::lsl(1)));
4197       cmp(ch1, skipch);
4198       br(NE, BMSKIP);
4199       subs(cnt1tmp, cnt1tmp, 1);
4200       br(LT, BMMATCH);
4201     BIND(BMLOOPSTR1);
4202       ldrh(ch1, Address(str1, cnt1tmp, Address::lsl(1)));
4203       ldrh(ch2, Address(str2, cnt1tmp, Address::lsl(1)));
4204       cmp(ch1, ch2);
4205       br(NE, BMSKIP);
4206       subs(cnt1tmp, cnt1tmp, 1);
4207       br(GE, BMLOOPSTR1);
4208     BIND(BMMATCH);
4209       sub(result_tmp, str2, result_tmp);
4210       lsr(result, result_tmp, 1);
4211       add(sp, sp, 128);
4212       b(DONE);
4213     BIND(BMADV);
4214       add(str2, str2, 2);
4215       b(BMCHECKEND);
4216     BIND(BMSKIP);
4217       cmp(skipch, 128);
4218       br(HS, BMADV);
4219       ldrb(ch2, Address(sp, skipch));
4220       add(str2, str2, cnt1, LSL, 1);
4221       sub(str2, str2, ch2, LSL, 1);
4222     BIND(BMCHECKEND);
4223       cmp(str2, str2end);
4224       br(LE, BMLOOPSTR2);
4225       add(sp, sp, 128);
4226       b(NOMATCH);
4227   }
4228 
4229   BIND(LINEARSEARCH);
4230   {
4231     Label DO1, DO2, DO3;
4232 
4233     Register str2tmp = tmp2;
4234     Register first = tmp3;
4235 
4236     if (icnt1 == -1)
4237     {
4238         Label DOSHORT, FIRST_LOOP, STR2_NEXT, STR1_LOOP, STR1_NEXT, LAST_WORD;
4239 
4240         cmp(cnt1, 4);
4241         br(LT, DOSHORT);
4242 
4243         sub(cnt2, cnt2, cnt1);
4244         sub(cnt1, cnt1, 4);
4245         mov(result_tmp, cnt2);
4246 
4247         lea(str1, Address(str1, cnt1, Address::uxtw(1)));
4248         lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4249         sub(cnt1_neg, zr, cnt1, LSL, 1);
4250         sub(cnt2_neg, zr, cnt2, LSL, 1);
4251         ldr(first, Address(str1, cnt1_neg));
4252 
4253       BIND(FIRST_LOOP);
4254         ldr(ch2, Address(str2, cnt2_neg));
4255         cmp(first, ch2);
4256         br(EQ, STR1_LOOP);
4257       BIND(STR2_NEXT);
4258         adds(cnt2_neg, cnt2_neg, 2);
4259         br(LE, FIRST_LOOP);
4260         b(NOMATCH);
4261 
4262       BIND(STR1_LOOP);
4263         adds(cnt1tmp, cnt1_neg, 8);
4264         add(cnt2tmp, cnt2_neg, 8);
4265         br(GE, LAST_WORD);
4266 
4267       BIND(STR1_NEXT);
4268         ldr(ch1, Address(str1, cnt1tmp));
4269         ldr(ch2, Address(str2, cnt2tmp));
4270         cmp(ch1, ch2);
4271         br(NE, STR2_NEXT);
4272         adds(cnt1tmp, cnt1tmp, 8);
4273         add(cnt2tmp, cnt2tmp, 8);
4274         br(LT, STR1_NEXT);
4275 
4276       BIND(LAST_WORD);
4277         ldr(ch1, Address(str1));
4278         sub(str2tmp, str2, cnt1_neg);         // adjust to corresponding
4279         ldr(ch2, Address(str2tmp, cnt2_neg)); // word in str2
4280         cmp(ch1, ch2);
4281         br(NE, STR2_NEXT);
4282         b(MATCH);
4283 
4284       BIND(DOSHORT);
4285         cmp(cnt1, 2);
4286         br(LT, DO1);
4287         br(GT, DO3);
4288     }
4289 
4290     if (icnt1 == 4) {
4291       Label CH1_LOOP;
4292 
4293         ldr(ch1, str1);
4294         sub(cnt2, cnt2, 4);
4295         mov(result_tmp, cnt2);
4296         lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4297         sub(cnt2_neg, zr, cnt2, LSL, 1);
4298 
4299       BIND(CH1_LOOP);
4300         ldr(ch2, Address(str2, cnt2_neg));
4301         cmp(ch1, ch2);
4302         br(EQ, MATCH);
4303         adds(cnt2_neg, cnt2_neg, 2);
4304         br(LE, CH1_LOOP);
4305         b(NOMATCH);
4306     }
4307 
4308     if (icnt1 == -1 || icnt1 == 2) {
4309       Label CH1_LOOP;
4310 
4311       BIND(DO2);
4312         ldrw(ch1, str1);
4313         sub(cnt2, cnt2, 2);
4314         mov(result_tmp, cnt2);
4315         lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4316         sub(cnt2_neg, zr, cnt2, LSL, 1);
4317 
4318       BIND(CH1_LOOP);
4319         ldrw(ch2, Address(str2, cnt2_neg));
4320         cmp(ch1, ch2);
4321         br(EQ, MATCH);
4322         adds(cnt2_neg, cnt2_neg, 2);
4323         br(LE, CH1_LOOP);
4324         b(NOMATCH);
4325     }
4326 
4327     if (icnt1 == -1 || icnt1 == 3) {
4328       Label FIRST_LOOP, STR2_NEXT, STR1_LOOP;
4329 
4330       BIND(DO3);
4331         ldrw(first, str1);
4332         ldrh(ch1, Address(str1, 4));
4333 
4334         sub(cnt2, cnt2, 3);
4335         mov(result_tmp, cnt2);
4336         lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4337         sub(cnt2_neg, zr, cnt2, LSL, 1);
4338 
4339       BIND(FIRST_LOOP);
4340         ldrw(ch2, Address(str2, cnt2_neg));
4341         cmpw(first, ch2);
4342         br(EQ, STR1_LOOP);
4343       BIND(STR2_NEXT);
4344         adds(cnt2_neg, cnt2_neg, 2);
4345         br(LE, FIRST_LOOP);
4346         b(NOMATCH);
4347 
4348       BIND(STR1_LOOP);
4349         add(cnt2tmp, cnt2_neg, 4);
4350         ldrh(ch2, Address(str2, cnt2tmp));
4351         cmp(ch1, ch2);
4352         br(NE, STR2_NEXT);
4353         b(MATCH);
4354     }
4355 
4356     if (icnt1 == -1 || icnt1 == 1) {
4357       Label CH1_LOOP, HAS_ZERO;
4358       Label DO1_SHORT, DO1_LOOP;
4359 
4360       BIND(DO1);
4361         ldrh(ch1, str1);
4362         cmp(cnt2, 4);
4363         br(LT, DO1_SHORT);
4364 
4365         orr(ch1, ch1, ch1, LSL, 16);
4366         orr(ch1, ch1, ch1, LSL, 32);
4367 
4368         sub(cnt2, cnt2, 4);
4369         mov(result_tmp, cnt2);
4370         lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4371         sub(cnt2_neg, zr, cnt2, LSL, 1);
4372 
4373         mov(tmp3, 0x0001000100010001);
4374       BIND(CH1_LOOP);
4375         ldr(ch2, Address(str2, cnt2_neg));
4376         eor(ch2, ch1, ch2);
4377         sub(tmp1, ch2, tmp3);
4378         orr(tmp2, ch2, 0x7fff7fff7fff7fff);
4379         bics(tmp1, tmp1, tmp2);
4380         br(NE, HAS_ZERO);
4381         adds(cnt2_neg, cnt2_neg, 8);
4382         br(LT, CH1_LOOP);
4383 
4384         cmp(cnt2_neg, 8);
4385         mov(cnt2_neg, 0);
4386         br(LT, CH1_LOOP);
4387         b(NOMATCH);
4388 
4389       BIND(HAS_ZERO);
4390         rev(tmp1, tmp1);
4391         clz(tmp1, tmp1);
4392         add(cnt2_neg, cnt2_neg, tmp1, LSR, 3);
4393         b(MATCH);
4394 
4395       BIND(DO1_SHORT);
4396         mov(result_tmp, cnt2);
4397         lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4398         sub(cnt2_neg, zr, cnt2, LSL, 1);
4399       BIND(DO1_LOOP);
4400         ldrh(ch2, Address(str2, cnt2_neg));
4401         cmpw(ch1, ch2);
4402         br(EQ, MATCH);
4403         adds(cnt2_neg, cnt2_neg, 2);
4404         br(LT, DO1_LOOP);
4405     }
4406   }
4407   BIND(NOMATCH);
4408     mov(result, -1);
4409     b(DONE);
4410   BIND(MATCH);
4411     add(result, result_tmp, cnt2_neg, ASR, 1);
4412   BIND(DONE);
4413 }
4414 
4415 // Compare strings.
4416 void MacroAssembler::string_compare(Register str1, Register str2,
4417                                     Register cnt1, Register cnt2, Register result,
4418                                     Register tmp1) {
4419   Label LENGTH_DIFF, DONE, SHORT_LOOP, SHORT_STRING,
4420     NEXT_WORD, DIFFERENCE;
4421 
4422   BLOCK_COMMENT("string_compare {");
4423 
4424   // Compute the minimum of the string lengths and save the difference.
4425   subsw(tmp1, cnt1, cnt2);
4426   cselw(cnt2, cnt1, cnt2, Assembler::LE); // min
4427 
4428   // A very short string
4429   cmpw(cnt2, 4);
4430   br(Assembler::LT, SHORT_STRING);
4431 
4432   // Check if the strings start at the same location.
4433   cmp(str1, str2);
4434   br(Assembler::EQ, LENGTH_DIFF);
4435 
4436   // Compare longwords
4437   {
4438     subw(cnt2, cnt2, 4); // The last longword is a special case
4439 
4440     // Move both string pointers to the last longword of their
4441     // strings, negate the remaining count, and convert it to bytes.
4442     lea(str1, Address(str1, cnt2, Address::uxtw(1)));
4443     lea(str2, Address(str2, cnt2, Address::uxtw(1)));
4444     sub(cnt2, zr, cnt2, LSL, 1);
4445 
4446     // Loop, loading longwords and comparing them into rscratch2.
4447     bind(NEXT_WORD);
4448     ldr(result, Address(str1, cnt2));
4449     ldr(cnt1, Address(str2, cnt2));
4450     adds(cnt2, cnt2, wordSize);
4451     eor(rscratch2, result, cnt1);
4452     cbnz(rscratch2, DIFFERENCE);
4453     br(Assembler::LT, NEXT_WORD);
4454 
4455     // Last longword.  In the case where length == 4 we compare the
4456     // same longword twice, but that's still faster than another
4457     // conditional branch.
4458 
4459     ldr(result, Address(str1));
4460     ldr(cnt1, Address(str2));
4461     eor(rscratch2, result, cnt1);
4462     cbz(rscratch2, LENGTH_DIFF);
4463 
4464     // Find the first different characters in the longwords and
4465     // compute their difference.
4466     bind(DIFFERENCE);
4467     rev(rscratch2, rscratch2);
4468     clz(rscratch2, rscratch2);
4469     andr(rscratch2, rscratch2, -16);
4470     lsrv(result, result, rscratch2);
4471     uxthw(result, result);
4472     lsrv(cnt1, cnt1, rscratch2);
4473     uxthw(cnt1, cnt1);
4474     subw(result, result, cnt1);
4475     b(DONE);
4476   }
4477 
4478   bind(SHORT_STRING);
4479   // Is the minimum length zero?
4480   cbz(cnt2, LENGTH_DIFF);
4481 
4482   bind(SHORT_LOOP);
4483   load_unsigned_short(result, Address(post(str1, 2)));
4484   load_unsigned_short(cnt1, Address(post(str2, 2)));
4485   subw(result, result, cnt1);
4486   cbnz(result, DONE);
4487   sub(cnt2, cnt2, 1);
4488   cbnz(cnt2, SHORT_LOOP);
4489 
4490   // Strings are equal up to min length.  Return the length difference.
4491   bind(LENGTH_DIFF);
4492   mov(result, tmp1);
4493 
4494   // That's it
4495   bind(DONE);
4496 
4497   BLOCK_COMMENT("} string_compare");
4498 }
4499 
4500 
4501 void MacroAssembler::string_equals(Register str1, Register str2,
4502                                    Register cnt, Register result,
4503                                    Register tmp1) {
4504   Label SAME_CHARS, DONE, SHORT_LOOP, SHORT_STRING,
4505     NEXT_WORD;
4506 
4507   const Register tmp2 = rscratch1;
4508   assert_different_registers(str1, str2, cnt, result, tmp1, tmp2, rscratch2);
4509 
4510   BLOCK_COMMENT("string_equals {");
4511 
4512   // Start by assuming that the strings are not equal.
4513   mov(result, zr);
4514 
4515   // A very short string
4516   cmpw(cnt, 4);
4517   br(Assembler::LT, SHORT_STRING);
4518 
4519   // Check if the strings start at the same location.
4520   cmp(str1, str2);
4521   br(Assembler::EQ, SAME_CHARS);
4522 
4523   // Compare longwords
4524   {
4525     subw(cnt, cnt, 4); // The last longword is a special case
4526 
4527     // Move both string pointers to the last longword of their
4528     // strings, negate the remaining count, and convert it to bytes.
4529     lea(str1, Address(str1, cnt, Address::uxtw(1)));
4530     lea(str2, Address(str2, cnt, Address::uxtw(1)));
4531     sub(cnt, zr, cnt, LSL, 1);
4532 
4533     // Loop, loading longwords and comparing them into rscratch2.
4534     bind(NEXT_WORD);
4535     ldr(tmp1, Address(str1, cnt));
4536     ldr(tmp2, Address(str2, cnt));
4537     adds(cnt, cnt, wordSize);
4538     eor(rscratch2, tmp1, tmp2);
4539     cbnz(rscratch2, DONE);
4540     br(Assembler::LT, NEXT_WORD);
4541 
4542     // Last longword.  In the case where length == 4 we compare the
4543     // same longword twice, but that's still faster than another
4544     // conditional branch.
4545 
4546     ldr(tmp1, Address(str1));
4547     ldr(tmp2, Address(str2));
4548     eor(rscratch2, tmp1, tmp2);
4549     cbz(rscratch2, SAME_CHARS);
4550     b(DONE);
4551   }
4552 
4553   bind(SHORT_STRING);
4554   // Is the length zero?
4555   cbz(cnt, SAME_CHARS);
4556 
4557   bind(SHORT_LOOP);
4558   load_unsigned_short(tmp1, Address(post(str1, 2)));
4559   load_unsigned_short(tmp2, Address(post(str2, 2)));
4560   subw(tmp1, tmp1, tmp2);
4561   cbnz(tmp1, DONE);
4562   sub(cnt, cnt, 1);
4563   cbnz(cnt, SHORT_LOOP);
4564 
4565   // Strings are equal.
4566   bind(SAME_CHARS);
4567   mov(result, true);
4568 
4569   // That's it
4570   bind(DONE);
4571 
4572   BLOCK_COMMENT("} string_equals");
4573 }
4574 
4575 
4576 void MacroAssembler::byte_arrays_equals(Register ary1, Register ary2,
4577                                         Register result, Register tmp1)
4578 {
4579   Register cnt1 = rscratch1;
4580   Register cnt2 = rscratch2;
4581   Register tmp2 = rscratch2;
4582 
4583   Label SAME, DIFFER, NEXT, TAIL07, TAIL03, TAIL01;
4584 
4585   int length_offset  = arrayOopDesc::length_offset_in_bytes();
4586   int base_offset    = arrayOopDesc::base_offset_in_bytes(T_BYTE);
4587 
4588   BLOCK_COMMENT("byte_arrays_equals  {");
4589 
4590     // different until proven equal
4591     mov(result, false);
4592 
4593     // same array?
4594     cmp(ary1, ary2);
4595     br(Assembler::EQ, SAME);
4596 
4597     // ne if either null
4598     cbz(ary1, DIFFER);
4599     cbz(ary2, DIFFER);
4600 
4601     // lengths ne?
4602     ldrw(cnt1, Address(ary1, length_offset));
4603     ldrw(cnt2, Address(ary2, length_offset));
4604     cmp(cnt1, cnt2);
4605     br(Assembler::NE, DIFFER);
4606 
4607     lea(ary1, Address(ary1, base_offset));
4608     lea(ary2, Address(ary2, base_offset));
4609 
4610     subs(cnt1, cnt1, 8);
4611     br(LT, TAIL07);
4612 
4613   BIND(NEXT);
4614     ldr(tmp1, Address(post(ary1, 8)));
4615     ldr(tmp2, Address(post(ary2, 8)));
4616     subs(cnt1, cnt1, 8);
4617     eor(tmp1, tmp1, tmp2);
4618     cbnz(tmp1, DIFFER);
4619     br(GE, NEXT);
4620 
4621   BIND(TAIL07);  // 0-7 bytes left, cnt1 = #bytes left - 4
4622     tst(cnt1, 0b100);
4623     br(EQ, TAIL03);
4624     ldrw(tmp1, Address(post(ary1, 4)));
4625     ldrw(tmp2, Address(post(ary2, 4)));
4626     cmp(tmp1, tmp2);
4627     br(NE, DIFFER);
4628 
4629   BIND(TAIL03);  // 0-3 bytes left, cnt1 = #bytes left - 4
4630     tst(cnt1, 0b10);
4631     br(EQ, TAIL01);
4632     ldrh(tmp1, Address(post(ary1, 2)));
4633     ldrh(tmp2, Address(post(ary2, 2)));
4634     cmp(tmp1, tmp2);
4635     br(NE, DIFFER);
4636   BIND(TAIL01);  // 0-1 byte left
4637     tst(cnt1, 0b01);
4638     br(EQ, SAME);
4639     ldrb(tmp1, ary1);
4640     ldrb(tmp2, ary2);
4641     cmp(tmp1, tmp2);
4642     br(NE, DIFFER);
4643 
4644   BIND(SAME);
4645     mov(result, true);
4646   BIND(DIFFER); // result already set
4647 
4648   BLOCK_COMMENT("} byte_arrays_equals");
4649 }
4650 
4651 // Compare char[] arrays aligned to 4 bytes
4652 void MacroAssembler::char_arrays_equals(Register ary1, Register ary2,
4653                                         Register result, Register tmp1)
4654 {
4655   Register cnt1 = rscratch1;
4656   Register cnt2 = rscratch2;
4657   Register tmp2 = rscratch2;
4658 
4659   Label SAME, DIFFER, NEXT, TAIL03, TAIL01;
4660 
4661   int length_offset  = arrayOopDesc::length_offset_in_bytes();
4662   int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
4663 
4664   BLOCK_COMMENT("char_arrays_equals  {");
4665 
4666     // different until proven equal
4667     mov(result, false);
4668 
4669     // same array?
4670     cmp(ary1, ary2);
4671     br(Assembler::EQ, SAME);
4672 
4673     // ne if either null
4674     cbz(ary1, DIFFER);
4675     cbz(ary2, DIFFER);
4676 
4677     // lengths ne?
4678     ldrw(cnt1, Address(ary1, length_offset));
4679     ldrw(cnt2, Address(ary2, length_offset));
4680     cmp(cnt1, cnt2);
4681     br(Assembler::NE, DIFFER);
4682 
4683     lea(ary1, Address(ary1, base_offset));
4684     lea(ary2, Address(ary2, base_offset));
4685 
4686     subs(cnt1, cnt1, 4);
4687     br(LT, TAIL03);
4688 
4689   BIND(NEXT);
4690     ldr(tmp1, Address(post(ary1, 8)));
4691     ldr(tmp2, Address(post(ary2, 8)));
4692     subs(cnt1, cnt1, 4);
4693     eor(tmp1, tmp1, tmp2);
4694     cbnz(tmp1, DIFFER);
4695     br(GE, NEXT);
4696 
4697   BIND(TAIL03);  // 0-3 chars left, cnt1 = #chars left - 4
4698     tst(cnt1, 0b10);
4699     br(EQ, TAIL01);
4700     ldrw(tmp1, Address(post(ary1, 4)));
4701     ldrw(tmp2, Address(post(ary2, 4)));
4702     cmp(tmp1, tmp2);
4703     br(NE, DIFFER);
4704   BIND(TAIL01);  // 0-1 chars left
4705     tst(cnt1, 0b01);
4706     br(EQ, SAME);
4707     ldrh(tmp1, ary1);
4708     ldrh(tmp2, ary2);
4709     cmp(tmp1, tmp2);
4710     br(NE, DIFFER);
4711 
4712   BIND(SAME);
4713     mov(result, true);
4714   BIND(DIFFER); // result already set
4715 
4716   BLOCK_COMMENT("} char_arrays_equals");
4717 }
4718 
4719 // encode char[] to byte[] in ISO_8859_1
4720 void MacroAssembler::encode_iso_array(Register src, Register dst,
4721                       Register len, Register result,
4722                       FloatRegister Vtmp1, FloatRegister Vtmp2,
4723                       FloatRegister Vtmp3, FloatRegister Vtmp4)
4724 {
4725     Label DONE, NEXT_32, LOOP_8, NEXT_8, LOOP_1, NEXT_1;
4726     Register tmp1 = rscratch1;
4727 
4728       mov(result, len); // Save initial len
4729 
4730 #ifndef BUILTIN_SIM
4731       subs(len, len, 32);
4732       br(LT, LOOP_8);
4733 
4734 // The following code uses the SIMD 'uqxtn' and 'uqxtn2' instructions
4735 // to convert chars to bytes. These set the 'QC' bit in the FPSR if
4736 // any char could not fit in a byte, so clear the FPSR so we can test it.
4737       clear_fpsr();
4738 
4739     BIND(NEXT_32);
4740       ld1(Vtmp1, Vtmp2, Vtmp3, Vtmp4, T8H, src);
4741       uqxtn(Vtmp1, T8B, Vtmp1, T8H);  // uqxtn  - write bottom half
4742       uqxtn(Vtmp1, T16B, Vtmp2, T8H); // uqxtn2 - write top half
4743       uqxtn(Vtmp2, T8B, Vtmp3, T8H);
4744       uqxtn(Vtmp2, T16B, Vtmp4, T8H); // uqxtn2
4745       get_fpsr(tmp1);
4746       cbnzw(tmp1, LOOP_8);
4747       st1(Vtmp1, Vtmp2, T16B, post(dst, 32));
4748       subs(len, len, 32);
4749       add(src, src, 64);
4750       br(GE, NEXT_32);
4751 
4752     BIND(LOOP_8);
4753       adds(len, len, 32-8);
4754       br(LT, LOOP_1);
4755       clear_fpsr(); // QC may be set from loop above, clear again
4756     BIND(NEXT_8);
4757       ld1(Vtmp1, T8H, src);
4758       uqxtn(Vtmp1, T8B, Vtmp1, T8H);
4759       get_fpsr(tmp1);
4760       cbnzw(tmp1, LOOP_1);
4761       st1(Vtmp1, T8B, post(dst, 8));
4762       subs(len, len, 8);
4763       add(src, src, 16);
4764       br(GE, NEXT_8);
4765 
4766     BIND(LOOP_1);
4767       adds(len, len, 8);
4768       br(LE, DONE);
4769 #else
4770       cbz(len, DONE);
4771 #endif
4772     BIND(NEXT_1);
4773       ldrh(tmp1, Address(post(src, 2)));
4774       tst(tmp1, 0xff00);
4775       br(NE, DONE);
4776       strb(tmp1, Address(post(dst, 1)));
4777       subs(len, len, 1);
4778       br(GT, NEXT_1);
4779 
4780     BIND(DONE);
4781       sub(result, result, len); // Return index where we stopped
4782 }
4783 
4784 // get_thread() can be called anywhere inside generated code so we
4785 // need to save whatever non-callee save context might get clobbered
4786 // by the call to JavaThread::aarch64_get_thread_helper() or, indeed,
4787 // the call setup code.
4788 //
4789 // aarch64_get_thread_helper() clobbers only r0, r1, and flags.
4790 //
4791 void MacroAssembler::get_thread(Register dst) {
4792   RegSet saved_regs = RegSet::range(r0, r1) + lr - dst;
4793   push(saved_regs, sp);
4794 
4795   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
4796   blrt(lr, 1, 0, 1);
4797   if (dst != c_rarg0) {
4798     mov(dst, c_rarg0);
4799   }
4800 
4801   pop(saved_regs, sp);
4802 }