1 /* 2 * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2012, 2016 SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/macroAssembler.inline.hpp" 28 #include "compiler/disassembler.hpp" 29 #include "gc/shared/cardTableModRefBS.hpp" 30 #include "gc/shared/collectedHeap.inline.hpp" 31 #include "interpreter/interpreter.hpp" 32 #include "memory/resourceArea.hpp" 33 #include "nativeInst_ppc.hpp" 34 #include "prims/methodHandles.hpp" 35 #include "runtime/biasedLocking.hpp" 36 #include "runtime/icache.hpp" 37 #include "runtime/interfaceSupport.hpp" 38 #include "runtime/objectMonitor.hpp" 39 #include "runtime/os.hpp" 40 #include "runtime/sharedRuntime.hpp" 41 #include "runtime/stubRoutines.hpp" 42 #include "utilities/macros.hpp" 43 #if INCLUDE_ALL_GCS 44 #include "gc/g1/g1CollectedHeap.inline.hpp" 45 #include "gc/g1/g1SATBCardTableModRefBS.hpp" 46 #include "gc/g1/heapRegion.hpp" 47 #endif // INCLUDE_ALL_GCS 48 #ifdef COMPILER2 49 #include "opto/intrinsicnode.hpp" 50 #endif 51 52 #ifdef PRODUCT 53 #define BLOCK_COMMENT(str) // nothing 54 #else 55 #define BLOCK_COMMENT(str) block_comment(str) 56 #endif 57 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 58 59 #ifdef ASSERT 60 // On RISC, there's no benefit to verifying instruction boundaries. 61 bool AbstractAssembler::pd_check_instruction_mark() { return false; } 62 #endif 63 64 void MacroAssembler::ld_largeoffset_unchecked(Register d, int si31, Register a, int emit_filler_nop) { 65 assert(Assembler::is_simm(si31, 31) && si31 >= 0, "si31 out of range"); 66 if (Assembler::is_simm(si31, 16)) { 67 ld(d, si31, a); 68 if (emit_filler_nop) nop(); 69 } else { 70 const int hi = MacroAssembler::largeoffset_si16_si16_hi(si31); 71 const int lo = MacroAssembler::largeoffset_si16_si16_lo(si31); 72 addis(d, a, hi); 73 ld(d, lo, d); 74 } 75 } 76 77 void MacroAssembler::ld_largeoffset(Register d, int si31, Register a, int emit_filler_nop) { 78 assert_different_registers(d, a); 79 ld_largeoffset_unchecked(d, si31, a, emit_filler_nop); 80 } 81 82 void MacroAssembler::load_sized_value(Register dst, RegisterOrConstant offs, Register base, 83 size_t size_in_bytes, bool is_signed) { 84 switch (size_in_bytes) { 85 case 8: ld(dst, offs, base); break; 86 case 4: is_signed ? lwa(dst, offs, base) : lwz(dst, offs, base); break; 87 case 2: is_signed ? lha(dst, offs, base) : lhz(dst, offs, base); break; 88 case 1: lbz(dst, offs, base); if (is_signed) extsb(dst, dst); break; // lba doesn't exist :( 89 default: ShouldNotReachHere(); 90 } 91 } 92 93 void MacroAssembler::store_sized_value(Register dst, RegisterOrConstant offs, Register base, 94 size_t size_in_bytes) { 95 switch (size_in_bytes) { 96 case 8: std(dst, offs, base); break; 97 case 4: stw(dst, offs, base); break; 98 case 2: sth(dst, offs, base); break; 99 case 1: stb(dst, offs, base); break; 100 default: ShouldNotReachHere(); 101 } 102 } 103 104 void MacroAssembler::align(int modulus, int max, int rem) { 105 int padding = (rem + modulus - (offset() % modulus)) % modulus; 106 if (padding > max) return; 107 for (int c = (padding >> 2); c > 0; --c) { nop(); } 108 } 109 110 // Issue instructions that calculate given TOC from global TOC. 111 void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16, 112 bool add_relocation, bool emit_dummy_addr) { 113 int offset = -1; 114 if (emit_dummy_addr) { 115 offset = -128; // dummy address 116 } else if (addr != (address)(intptr_t)-1) { 117 offset = MacroAssembler::offset_to_global_toc(addr); 118 } 119 120 if (hi16) { 121 addis(dst, R29_TOC, MacroAssembler::largeoffset_si16_si16_hi(offset)); 122 } 123 if (lo16) { 124 if (add_relocation) { 125 // Relocate at the addi to avoid confusion with a load from the method's TOC. 126 relocate(internal_word_Relocation::spec(addr)); 127 } 128 addi(dst, dst, MacroAssembler::largeoffset_si16_si16_lo(offset)); 129 } 130 } 131 132 int MacroAssembler::patch_calculate_address_from_global_toc_at(address a, address bound, address addr) { 133 const int offset = MacroAssembler::offset_to_global_toc(addr); 134 135 const address inst2_addr = a; 136 const int inst2 = *(int *)inst2_addr; 137 138 // The relocation points to the second instruction, the addi, 139 // and the addi reads and writes the same register dst. 140 const int dst = inv_rt_field(inst2); 141 assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst"); 142 143 // Now, find the preceding addis which writes to dst. 144 int inst1 = 0; 145 address inst1_addr = inst2_addr - BytesPerInstWord; 146 while (inst1_addr >= bound) { 147 inst1 = *(int *) inst1_addr; 148 if (is_addis(inst1) && inv_rt_field(inst1) == dst) { 149 // Stop, found the addis which writes dst. 150 break; 151 } 152 inst1_addr -= BytesPerInstWord; 153 } 154 155 assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC"); 156 set_imm((int *)inst1_addr, MacroAssembler::largeoffset_si16_si16_hi(offset)); 157 set_imm((int *)inst2_addr, MacroAssembler::largeoffset_si16_si16_lo(offset)); 158 return (int)((intptr_t)addr - (intptr_t)inst1_addr); 159 } 160 161 address MacroAssembler::get_address_of_calculate_address_from_global_toc_at(address a, address bound) { 162 const address inst2_addr = a; 163 const int inst2 = *(int *)inst2_addr; 164 165 // The relocation points to the second instruction, the addi, 166 // and the addi reads and writes the same register dst. 167 const int dst = inv_rt_field(inst2); 168 assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst"); 169 170 // Now, find the preceding addis which writes to dst. 171 int inst1 = 0; 172 address inst1_addr = inst2_addr - BytesPerInstWord; 173 while (inst1_addr >= bound) { 174 inst1 = *(int *) inst1_addr; 175 if (is_addis(inst1) && inv_rt_field(inst1) == dst) { 176 // stop, found the addis which writes dst 177 break; 178 } 179 inst1_addr -= BytesPerInstWord; 180 } 181 182 assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC"); 183 184 int offset = (get_imm(inst1_addr, 0) << 16) + get_imm(inst2_addr, 0); 185 // -1 is a special case 186 if (offset == -1) { 187 return (address)(intptr_t)-1; 188 } else { 189 return global_toc() + offset; 190 } 191 } 192 193 #ifdef _LP64 194 // Patch compressed oops or klass constants. 195 // Assembler sequence is 196 // 1) compressed oops: 197 // lis rx = const.hi 198 // ori rx = rx | const.lo 199 // 2) compressed klass: 200 // lis rx = const.hi 201 // clrldi rx = rx & 0xFFFFffff // clearMS32b, optional 202 // ori rx = rx | const.lo 203 // Clrldi will be passed by. 204 int MacroAssembler::patch_set_narrow_oop(address a, address bound, narrowOop data) { 205 assert(UseCompressedOops, "Should only patch compressed oops"); 206 207 const address inst2_addr = a; 208 const int inst2 = *(int *)inst2_addr; 209 210 // The relocation points to the second instruction, the ori, 211 // and the ori reads and writes the same register dst. 212 const int dst = inv_rta_field(inst2); 213 assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst"); 214 // Now, find the preceding addis which writes to dst. 215 int inst1 = 0; 216 address inst1_addr = inst2_addr - BytesPerInstWord; 217 bool inst1_found = false; 218 while (inst1_addr >= bound) { 219 inst1 = *(int *)inst1_addr; 220 if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break; } 221 inst1_addr -= BytesPerInstWord; 222 } 223 assert(inst1_found, "inst is not lis"); 224 225 int xc = (data >> 16) & 0xffff; 226 int xd = (data >> 0) & 0xffff; 227 228 set_imm((int *)inst1_addr, (short)(xc)); // see enc_load_con_narrow_hi/_lo 229 set_imm((int *)inst2_addr, (xd)); // unsigned int 230 return (int)((intptr_t)inst2_addr - (intptr_t)inst1_addr); 231 } 232 233 // Get compressed oop or klass constant. 234 narrowOop MacroAssembler::get_narrow_oop(address a, address bound) { 235 assert(UseCompressedOops, "Should only patch compressed oops"); 236 237 const address inst2_addr = a; 238 const int inst2 = *(int *)inst2_addr; 239 240 // The relocation points to the second instruction, the ori, 241 // and the ori reads and writes the same register dst. 242 const int dst = inv_rta_field(inst2); 243 assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst"); 244 // Now, find the preceding lis which writes to dst. 245 int inst1 = 0; 246 address inst1_addr = inst2_addr - BytesPerInstWord; 247 bool inst1_found = false; 248 249 while (inst1_addr >= bound) { 250 inst1 = *(int *) inst1_addr; 251 if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break;} 252 inst1_addr -= BytesPerInstWord; 253 } 254 assert(inst1_found, "inst is not lis"); 255 256 uint xl = ((unsigned int) (get_imm(inst2_addr, 0) & 0xffff)); 257 uint xh = (((get_imm(inst1_addr, 0)) & 0xffff) << 16); 258 259 return (int) (xl | xh); 260 } 261 #endif // _LP64 262 263 // Returns true if successful. 264 bool MacroAssembler::load_const_from_method_toc(Register dst, AddressLiteral& a, 265 Register toc, bool fixed_size) { 266 int toc_offset = 0; 267 // Use RelocationHolder::none for the constant pool entry, otherwise 268 // we will end up with a failing NativeCall::verify(x) where x is 269 // the address of the constant pool entry. 270 // FIXME: We should insert relocation information for oops at the constant 271 // pool entries instead of inserting it at the loads; patching of a constant 272 // pool entry should be less expensive. 273 address const_address = address_constant((address)a.value(), RelocationHolder::none); 274 if (const_address == NULL) { return false; } // allocation failure 275 // Relocate at the pc of the load. 276 relocate(a.rspec()); 277 toc_offset = (int)(const_address - code()->consts()->start()); 278 ld_largeoffset_unchecked(dst, toc_offset, toc, fixed_size); 279 return true; 280 } 281 282 bool MacroAssembler::is_load_const_from_method_toc_at(address a) { 283 const address inst1_addr = a; 284 const int inst1 = *(int *)inst1_addr; 285 286 // The relocation points to the ld or the addis. 287 return (is_ld(inst1)) || 288 (is_addis(inst1) && inv_ra_field(inst1) != 0); 289 } 290 291 int MacroAssembler::get_offset_of_load_const_from_method_toc_at(address a) { 292 assert(is_load_const_from_method_toc_at(a), "must be load_const_from_method_toc"); 293 294 const address inst1_addr = a; 295 const int inst1 = *(int *)inst1_addr; 296 297 if (is_ld(inst1)) { 298 return inv_d1_field(inst1); 299 } else if (is_addis(inst1)) { 300 const int dst = inv_rt_field(inst1); 301 302 // Now, find the succeeding ld which reads and writes to dst. 303 address inst2_addr = inst1_addr + BytesPerInstWord; 304 int inst2 = 0; 305 while (true) { 306 inst2 = *(int *) inst2_addr; 307 if (is_ld(inst2) && inv_ra_field(inst2) == dst && inv_rt_field(inst2) == dst) { 308 // Stop, found the ld which reads and writes dst. 309 break; 310 } 311 inst2_addr += BytesPerInstWord; 312 } 313 return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2); 314 } 315 ShouldNotReachHere(); 316 return 0; 317 } 318 319 // Get the constant from a `load_const' sequence. 320 long MacroAssembler::get_const(address a) { 321 assert(is_load_const_at(a), "not a load of a constant"); 322 const int *p = (const int*) a; 323 unsigned long x = (((unsigned long) (get_imm(a,0) & 0xffff)) << 48); 324 if (is_ori(*(p+1))) { 325 x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 32); 326 x |= (((unsigned long) (get_imm(a,3) & 0xffff)) << 16); 327 x |= (((unsigned long) (get_imm(a,4) & 0xffff))); 328 } else if (is_lis(*(p+1))) { 329 x |= (((unsigned long) (get_imm(a,2) & 0xffff)) << 32); 330 x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 16); 331 x |= (((unsigned long) (get_imm(a,3) & 0xffff))); 332 } else { 333 ShouldNotReachHere(); 334 return (long) 0; 335 } 336 return (long) x; 337 } 338 339 // Patch the 64 bit constant of a `load_const' sequence. This is a low 340 // level procedure. It neither flushes the instruction cache nor is it 341 // mt safe. 342 void MacroAssembler::patch_const(address a, long x) { 343 assert(is_load_const_at(a), "not a load of a constant"); 344 int *p = (int*) a; 345 if (is_ori(*(p+1))) { 346 set_imm(0 + p, (x >> 48) & 0xffff); 347 set_imm(1 + p, (x >> 32) & 0xffff); 348 set_imm(3 + p, (x >> 16) & 0xffff); 349 set_imm(4 + p, x & 0xffff); 350 } else if (is_lis(*(p+1))) { 351 set_imm(0 + p, (x >> 48) & 0xffff); 352 set_imm(2 + p, (x >> 32) & 0xffff); 353 set_imm(1 + p, (x >> 16) & 0xffff); 354 set_imm(3 + p, x & 0xffff); 355 } else { 356 ShouldNotReachHere(); 357 } 358 } 359 360 AddressLiteral MacroAssembler::allocate_metadata_address(Metadata* obj) { 361 assert(oop_recorder() != NULL, "this assembler needs a Recorder"); 362 int index = oop_recorder()->allocate_metadata_index(obj); 363 RelocationHolder rspec = metadata_Relocation::spec(index); 364 return AddressLiteral((address)obj, rspec); 365 } 366 367 AddressLiteral MacroAssembler::constant_metadata_address(Metadata* obj) { 368 assert(oop_recorder() != NULL, "this assembler needs a Recorder"); 369 int index = oop_recorder()->find_index(obj); 370 RelocationHolder rspec = metadata_Relocation::spec(index); 371 return AddressLiteral((address)obj, rspec); 372 } 373 374 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) { 375 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder"); 376 int oop_index = oop_recorder()->allocate_oop_index(obj); 377 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index)); 378 } 379 380 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) { 381 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder"); 382 int oop_index = oop_recorder()->find_index(obj); 383 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index)); 384 } 385 386 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr, 387 Register tmp, int offset) { 388 intptr_t value = *delayed_value_addr; 389 if (value != 0) { 390 return RegisterOrConstant(value + offset); 391 } 392 393 // Load indirectly to solve generation ordering problem. 394 // static address, no relocation 395 int simm16_offset = load_const_optimized(tmp, delayed_value_addr, noreg, true); 396 ld(tmp, simm16_offset, tmp); // must be aligned ((xa & 3) == 0) 397 398 if (offset != 0) { 399 addi(tmp, tmp, offset); 400 } 401 402 return RegisterOrConstant(tmp); 403 } 404 405 #ifndef PRODUCT 406 void MacroAssembler::pd_print_patched_instruction(address branch) { 407 Unimplemented(); // TODO: PPC port 408 } 409 #endif // ndef PRODUCT 410 411 // Conditional far branch for destinations encodable in 24+2 bits. 412 void MacroAssembler::bc_far(int boint, int biint, Label& dest, int optimize) { 413 414 // If requested by flag optimize, relocate the bc_far as a 415 // runtime_call and prepare for optimizing it when the code gets 416 // relocated. 417 if (optimize == bc_far_optimize_on_relocate) { 418 relocate(relocInfo::runtime_call_type); 419 } 420 421 // variant 2: 422 // 423 // b!cxx SKIP 424 // bxx DEST 425 // SKIP: 426 // 427 428 const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)), 429 opposite_bcond(inv_boint_bcond(boint))); 430 431 // We emit two branches. 432 // First, a conditional branch which jumps around the far branch. 433 const address not_taken_pc = pc() + 2 * BytesPerInstWord; 434 const address bc_pc = pc(); 435 bc(opposite_boint, biint, not_taken_pc); 436 437 const int bc_instr = *(int*)bc_pc; 438 assert(not_taken_pc == (address)inv_bd_field(bc_instr, (intptr_t)bc_pc), "postcondition"); 439 assert(opposite_boint == inv_bo_field(bc_instr), "postcondition"); 440 assert(boint == add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(bc_instr))), 441 opposite_bcond(inv_boint_bcond(inv_bo_field(bc_instr)))), 442 "postcondition"); 443 assert(biint == inv_bi_field(bc_instr), "postcondition"); 444 445 // Second, an unconditional far branch which jumps to dest. 446 // Note: target(dest) remembers the current pc (see CodeSection::target) 447 // and returns the current pc if the label is not bound yet; when 448 // the label gets bound, the unconditional far branch will be patched. 449 const address target_pc = target(dest); 450 const address b_pc = pc(); 451 b(target_pc); 452 453 assert(not_taken_pc == pc(), "postcondition"); 454 assert(dest.is_bound() || target_pc == b_pc, "postcondition"); 455 } 456 457 // 1 or 2 instructions 458 void MacroAssembler::bc_far_optimized(int boint, int biint, Label& dest) { 459 if (dest.is_bound() && is_within_range_of_bcxx(target(dest), pc())) { 460 bc(boint, biint, dest); 461 } else { 462 bc_far(boint, biint, dest, MacroAssembler::bc_far_optimize_on_relocate); 463 } 464 } 465 466 bool MacroAssembler::is_bc_far_at(address instruction_addr) { 467 return is_bc_far_variant1_at(instruction_addr) || 468 is_bc_far_variant2_at(instruction_addr) || 469 is_bc_far_variant3_at(instruction_addr); 470 } 471 472 address MacroAssembler::get_dest_of_bc_far_at(address instruction_addr) { 473 if (is_bc_far_variant1_at(instruction_addr)) { 474 const address instruction_1_addr = instruction_addr; 475 const int instruction_1 = *(int*)instruction_1_addr; 476 return (address)inv_bd_field(instruction_1, (intptr_t)instruction_1_addr); 477 } else if (is_bc_far_variant2_at(instruction_addr)) { 478 const address instruction_2_addr = instruction_addr + 4; 479 return bxx_destination(instruction_2_addr); 480 } else if (is_bc_far_variant3_at(instruction_addr)) { 481 return instruction_addr + 8; 482 } 483 // variant 4 ??? 484 ShouldNotReachHere(); 485 return NULL; 486 } 487 void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address dest) { 488 489 if (is_bc_far_variant3_at(instruction_addr)) { 490 // variant 3, far cond branch to the next instruction, already patched to nops: 491 // 492 // nop 493 // endgroup 494 // SKIP/DEST: 495 // 496 return; 497 } 498 499 // first, extract boint and biint from the current branch 500 int boint = 0; 501 int biint = 0; 502 503 ResourceMark rm; 504 const int code_size = 2 * BytesPerInstWord; 505 CodeBuffer buf(instruction_addr, code_size); 506 MacroAssembler masm(&buf); 507 if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) { 508 // Far branch to next instruction: Optimize it by patching nops (produce variant 3). 509 masm.nop(); 510 masm.endgroup(); 511 } else { 512 if (is_bc_far_variant1_at(instruction_addr)) { 513 // variant 1, the 1st instruction contains the destination address: 514 // 515 // bcxx DEST 516 // nop 517 // 518 const int instruction_1 = *(int*)(instruction_addr); 519 boint = inv_bo_field(instruction_1); 520 biint = inv_bi_field(instruction_1); 521 } else if (is_bc_far_variant2_at(instruction_addr)) { 522 // variant 2, the 2nd instruction contains the destination address: 523 // 524 // b!cxx SKIP 525 // bxx DEST 526 // SKIP: 527 // 528 const int instruction_1 = *(int*)(instruction_addr); 529 boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(instruction_1))), 530 opposite_bcond(inv_boint_bcond(inv_bo_field(instruction_1)))); 531 biint = inv_bi_field(instruction_1); 532 } else { 533 // variant 4??? 534 ShouldNotReachHere(); 535 } 536 537 // second, set the new branch destination and optimize the code 538 if (dest != instruction_addr + 4 && // the bc_far is still unbound! 539 masm.is_within_range_of_bcxx(dest, instruction_addr)) { 540 // variant 1: 541 // 542 // bcxx DEST 543 // nop 544 // 545 masm.bc(boint, biint, dest); 546 masm.nop(); 547 } else { 548 // variant 2: 549 // 550 // b!cxx SKIP 551 // bxx DEST 552 // SKIP: 553 // 554 const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)), 555 opposite_bcond(inv_boint_bcond(boint))); 556 const address not_taken_pc = masm.pc() + 2 * BytesPerInstWord; 557 masm.bc(opposite_boint, biint, not_taken_pc); 558 masm.b(dest); 559 } 560 } 561 ICache::ppc64_flush_icache_bytes(instruction_addr, code_size); 562 } 563 564 // Emit a NOT mt-safe patchable 64 bit absolute call/jump. 565 void MacroAssembler::bxx64_patchable(address dest, relocInfo::relocType rt, bool link) { 566 // get current pc 567 uint64_t start_pc = (uint64_t) pc(); 568 569 const address pc_of_bl = (address) (start_pc + (6*BytesPerInstWord)); // bl is last 570 const address pc_of_b = (address) (start_pc + (0*BytesPerInstWord)); // b is first 571 572 // relocate here 573 if (rt != relocInfo::none) { 574 relocate(rt); 575 } 576 577 if ( ReoptimizeCallSequences && 578 (( link && is_within_range_of_b(dest, pc_of_bl)) || 579 (!link && is_within_range_of_b(dest, pc_of_b)))) { 580 // variant 2: 581 // Emit an optimized, pc-relative call/jump. 582 583 if (link) { 584 // some padding 585 nop(); 586 nop(); 587 nop(); 588 nop(); 589 nop(); 590 nop(); 591 592 // do the call 593 assert(pc() == pc_of_bl, "just checking"); 594 bl(dest, relocInfo::none); 595 } else { 596 // do the jump 597 assert(pc() == pc_of_b, "just checking"); 598 b(dest, relocInfo::none); 599 600 // some padding 601 nop(); 602 nop(); 603 nop(); 604 nop(); 605 nop(); 606 nop(); 607 } 608 609 // Assert that we can identify the emitted call/jump. 610 assert(is_bxx64_patchable_variant2_at((address)start_pc, link), 611 "can't identify emitted call"); 612 } else { 613 // variant 1: 614 mr(R0, R11); // spill R11 -> R0. 615 616 // Load the destination address into CTR, 617 // calculate destination relative to global toc. 618 calculate_address_from_global_toc(R11, dest, true, true, false); 619 620 mtctr(R11); 621 mr(R11, R0); // spill R11 <- R0. 622 nop(); 623 624 // do the call/jump 625 if (link) { 626 bctrl(); 627 } else{ 628 bctr(); 629 } 630 // Assert that we can identify the emitted call/jump. 631 assert(is_bxx64_patchable_variant1b_at((address)start_pc, link), 632 "can't identify emitted call"); 633 } 634 635 // Assert that we can identify the emitted call/jump. 636 assert(is_bxx64_patchable_at((address)start_pc, link), 637 "can't identify emitted call"); 638 assert(get_dest_of_bxx64_patchable_at((address)start_pc, link) == dest, 639 "wrong encoding of dest address"); 640 } 641 642 // Identify a bxx64_patchable instruction. 643 bool MacroAssembler::is_bxx64_patchable_at(address instruction_addr, bool link) { 644 return is_bxx64_patchable_variant1b_at(instruction_addr, link) 645 //|| is_bxx64_patchable_variant1_at(instruction_addr, link) 646 || is_bxx64_patchable_variant2_at(instruction_addr, link); 647 } 648 649 // Does the call64_patchable instruction use a pc-relative encoding of 650 // the call destination? 651 bool MacroAssembler::is_bxx64_patchable_pcrelative_at(address instruction_addr, bool link) { 652 // variant 2 is pc-relative 653 return is_bxx64_patchable_variant2_at(instruction_addr, link); 654 } 655 656 // Identify variant 1. 657 bool MacroAssembler::is_bxx64_patchable_variant1_at(address instruction_addr, bool link) { 658 unsigned int* instr = (unsigned int*) instruction_addr; 659 return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l] 660 && is_mtctr(instr[5]) // mtctr 661 && is_load_const_at(instruction_addr); 662 } 663 664 // Identify variant 1b: load destination relative to global toc. 665 bool MacroAssembler::is_bxx64_patchable_variant1b_at(address instruction_addr, bool link) { 666 unsigned int* instr = (unsigned int*) instruction_addr; 667 return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l] 668 && is_mtctr(instr[3]) // mtctr 669 && is_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord, instruction_addr); 670 } 671 672 // Identify variant 2. 673 bool MacroAssembler::is_bxx64_patchable_variant2_at(address instruction_addr, bool link) { 674 unsigned int* instr = (unsigned int*) instruction_addr; 675 if (link) { 676 return is_bl (instr[6]) // bl dest is last 677 && is_nop(instr[0]) // nop 678 && is_nop(instr[1]) // nop 679 && is_nop(instr[2]) // nop 680 && is_nop(instr[3]) // nop 681 && is_nop(instr[4]) // nop 682 && is_nop(instr[5]); // nop 683 } else { 684 return is_b (instr[0]) // b dest is first 685 && is_nop(instr[1]) // nop 686 && is_nop(instr[2]) // nop 687 && is_nop(instr[3]) // nop 688 && is_nop(instr[4]) // nop 689 && is_nop(instr[5]) // nop 690 && is_nop(instr[6]); // nop 691 } 692 } 693 694 // Set dest address of a bxx64_patchable instruction. 695 void MacroAssembler::set_dest_of_bxx64_patchable_at(address instruction_addr, address dest, bool link) { 696 ResourceMark rm; 697 int code_size = MacroAssembler::bxx64_patchable_size; 698 CodeBuffer buf(instruction_addr, code_size); 699 MacroAssembler masm(&buf); 700 masm.bxx64_patchable(dest, relocInfo::none, link); 701 ICache::ppc64_flush_icache_bytes(instruction_addr, code_size); 702 } 703 704 // Get dest address of a bxx64_patchable instruction. 705 address MacroAssembler::get_dest_of_bxx64_patchable_at(address instruction_addr, bool link) { 706 if (is_bxx64_patchable_variant1_at(instruction_addr, link)) { 707 return (address) (unsigned long) get_const(instruction_addr); 708 } else if (is_bxx64_patchable_variant2_at(instruction_addr, link)) { 709 unsigned int* instr = (unsigned int*) instruction_addr; 710 if (link) { 711 const int instr_idx = 6; // bl is last 712 int branchoffset = branch_destination(instr[instr_idx], 0); 713 return instruction_addr + branchoffset + instr_idx*BytesPerInstWord; 714 } else { 715 const int instr_idx = 0; // b is first 716 int branchoffset = branch_destination(instr[instr_idx], 0); 717 return instruction_addr + branchoffset + instr_idx*BytesPerInstWord; 718 } 719 // Load dest relative to global toc. 720 } else if (is_bxx64_patchable_variant1b_at(instruction_addr, link)) { 721 return get_address_of_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord, 722 instruction_addr); 723 } else { 724 ShouldNotReachHere(); 725 return NULL; 726 } 727 } 728 729 // Uses ordering which corresponds to ABI: 730 // _savegpr0_14: std r14,-144(r1) 731 // _savegpr0_15: std r15,-136(r1) 732 // _savegpr0_16: std r16,-128(r1) 733 void MacroAssembler::save_nonvolatile_gprs(Register dst, int offset) { 734 std(R14, offset, dst); offset += 8; 735 std(R15, offset, dst); offset += 8; 736 std(R16, offset, dst); offset += 8; 737 std(R17, offset, dst); offset += 8; 738 std(R18, offset, dst); offset += 8; 739 std(R19, offset, dst); offset += 8; 740 std(R20, offset, dst); offset += 8; 741 std(R21, offset, dst); offset += 8; 742 std(R22, offset, dst); offset += 8; 743 std(R23, offset, dst); offset += 8; 744 std(R24, offset, dst); offset += 8; 745 std(R25, offset, dst); offset += 8; 746 std(R26, offset, dst); offset += 8; 747 std(R27, offset, dst); offset += 8; 748 std(R28, offset, dst); offset += 8; 749 std(R29, offset, dst); offset += 8; 750 std(R30, offset, dst); offset += 8; 751 std(R31, offset, dst); offset += 8; 752 753 stfd(F14, offset, dst); offset += 8; 754 stfd(F15, offset, dst); offset += 8; 755 stfd(F16, offset, dst); offset += 8; 756 stfd(F17, offset, dst); offset += 8; 757 stfd(F18, offset, dst); offset += 8; 758 stfd(F19, offset, dst); offset += 8; 759 stfd(F20, offset, dst); offset += 8; 760 stfd(F21, offset, dst); offset += 8; 761 stfd(F22, offset, dst); offset += 8; 762 stfd(F23, offset, dst); offset += 8; 763 stfd(F24, offset, dst); offset += 8; 764 stfd(F25, offset, dst); offset += 8; 765 stfd(F26, offset, dst); offset += 8; 766 stfd(F27, offset, dst); offset += 8; 767 stfd(F28, offset, dst); offset += 8; 768 stfd(F29, offset, dst); offset += 8; 769 stfd(F30, offset, dst); offset += 8; 770 stfd(F31, offset, dst); 771 } 772 773 // Uses ordering which corresponds to ABI: 774 // _restgpr0_14: ld r14,-144(r1) 775 // _restgpr0_15: ld r15,-136(r1) 776 // _restgpr0_16: ld r16,-128(r1) 777 void MacroAssembler::restore_nonvolatile_gprs(Register src, int offset) { 778 ld(R14, offset, src); offset += 8; 779 ld(R15, offset, src); offset += 8; 780 ld(R16, offset, src); offset += 8; 781 ld(R17, offset, src); offset += 8; 782 ld(R18, offset, src); offset += 8; 783 ld(R19, offset, src); offset += 8; 784 ld(R20, offset, src); offset += 8; 785 ld(R21, offset, src); offset += 8; 786 ld(R22, offset, src); offset += 8; 787 ld(R23, offset, src); offset += 8; 788 ld(R24, offset, src); offset += 8; 789 ld(R25, offset, src); offset += 8; 790 ld(R26, offset, src); offset += 8; 791 ld(R27, offset, src); offset += 8; 792 ld(R28, offset, src); offset += 8; 793 ld(R29, offset, src); offset += 8; 794 ld(R30, offset, src); offset += 8; 795 ld(R31, offset, src); offset += 8; 796 797 // FP registers 798 lfd(F14, offset, src); offset += 8; 799 lfd(F15, offset, src); offset += 8; 800 lfd(F16, offset, src); offset += 8; 801 lfd(F17, offset, src); offset += 8; 802 lfd(F18, offset, src); offset += 8; 803 lfd(F19, offset, src); offset += 8; 804 lfd(F20, offset, src); offset += 8; 805 lfd(F21, offset, src); offset += 8; 806 lfd(F22, offset, src); offset += 8; 807 lfd(F23, offset, src); offset += 8; 808 lfd(F24, offset, src); offset += 8; 809 lfd(F25, offset, src); offset += 8; 810 lfd(F26, offset, src); offset += 8; 811 lfd(F27, offset, src); offset += 8; 812 lfd(F28, offset, src); offset += 8; 813 lfd(F29, offset, src); offset += 8; 814 lfd(F30, offset, src); offset += 8; 815 lfd(F31, offset, src); 816 } 817 818 // For verify_oops. 819 void MacroAssembler::save_volatile_gprs(Register dst, int offset) { 820 std(R2, offset, dst); offset += 8; 821 std(R3, offset, dst); offset += 8; 822 std(R4, offset, dst); offset += 8; 823 std(R5, offset, dst); offset += 8; 824 std(R6, offset, dst); offset += 8; 825 std(R7, offset, dst); offset += 8; 826 std(R8, offset, dst); offset += 8; 827 std(R9, offset, dst); offset += 8; 828 std(R10, offset, dst); offset += 8; 829 std(R11, offset, dst); offset += 8; 830 std(R12, offset, dst); offset += 8; 831 832 stfd(F0, offset, dst); offset += 8; 833 stfd(F1, offset, dst); offset += 8; 834 stfd(F2, offset, dst); offset += 8; 835 stfd(F3, offset, dst); offset += 8; 836 stfd(F4, offset, dst); offset += 8; 837 stfd(F5, offset, dst); offset += 8; 838 stfd(F6, offset, dst); offset += 8; 839 stfd(F7, offset, dst); offset += 8; 840 stfd(F8, offset, dst); offset += 8; 841 stfd(F9, offset, dst); offset += 8; 842 stfd(F10, offset, dst); offset += 8; 843 stfd(F11, offset, dst); offset += 8; 844 stfd(F12, offset, dst); offset += 8; 845 stfd(F13, offset, dst); 846 } 847 848 // For verify_oops. 849 void MacroAssembler::restore_volatile_gprs(Register src, int offset) { 850 ld(R2, offset, src); offset += 8; 851 ld(R3, offset, src); offset += 8; 852 ld(R4, offset, src); offset += 8; 853 ld(R5, offset, src); offset += 8; 854 ld(R6, offset, src); offset += 8; 855 ld(R7, offset, src); offset += 8; 856 ld(R8, offset, src); offset += 8; 857 ld(R9, offset, src); offset += 8; 858 ld(R10, offset, src); offset += 8; 859 ld(R11, offset, src); offset += 8; 860 ld(R12, offset, src); offset += 8; 861 862 lfd(F0, offset, src); offset += 8; 863 lfd(F1, offset, src); offset += 8; 864 lfd(F2, offset, src); offset += 8; 865 lfd(F3, offset, src); offset += 8; 866 lfd(F4, offset, src); offset += 8; 867 lfd(F5, offset, src); offset += 8; 868 lfd(F6, offset, src); offset += 8; 869 lfd(F7, offset, src); offset += 8; 870 lfd(F8, offset, src); offset += 8; 871 lfd(F9, offset, src); offset += 8; 872 lfd(F10, offset, src); offset += 8; 873 lfd(F11, offset, src); offset += 8; 874 lfd(F12, offset, src); offset += 8; 875 lfd(F13, offset, src); 876 } 877 878 void MacroAssembler::save_LR_CR(Register tmp) { 879 mfcr(tmp); 880 std(tmp, _abi(cr), R1_SP); 881 mflr(tmp); 882 std(tmp, _abi(lr), R1_SP); 883 // Tmp must contain lr on exit! (see return_addr and prolog in ppc64.ad) 884 } 885 886 void MacroAssembler::restore_LR_CR(Register tmp) { 887 assert(tmp != R1_SP, "must be distinct"); 888 ld(tmp, _abi(lr), R1_SP); 889 mtlr(tmp); 890 ld(tmp, _abi(cr), R1_SP); 891 mtcr(tmp); 892 } 893 894 address MacroAssembler::get_PC_trash_LR(Register result) { 895 Label L; 896 bl(L); 897 bind(L); 898 address lr_pc = pc(); 899 mflr(result); 900 return lr_pc; 901 } 902 903 void MacroAssembler::resize_frame(Register offset, Register tmp) { 904 #ifdef ASSERT 905 assert_different_registers(offset, tmp, R1_SP); 906 andi_(tmp, offset, frame::alignment_in_bytes-1); 907 asm_assert_eq("resize_frame: unaligned", 0x204); 908 #endif 909 910 // tmp <- *(SP) 911 ld(tmp, _abi(callers_sp), R1_SP); 912 // addr <- SP + offset; 913 // *(addr) <- tmp; 914 // SP <- addr 915 stdux(tmp, R1_SP, offset); 916 } 917 918 void MacroAssembler::resize_frame(int offset, Register tmp) { 919 assert(is_simm(offset, 16), "too big an offset"); 920 assert_different_registers(tmp, R1_SP); 921 assert((offset & (frame::alignment_in_bytes-1))==0, "resize_frame: unaligned"); 922 // tmp <- *(SP) 923 ld(tmp, _abi(callers_sp), R1_SP); 924 // addr <- SP + offset; 925 // *(addr) <- tmp; 926 // SP <- addr 927 stdu(tmp, offset, R1_SP); 928 } 929 930 void MacroAssembler::resize_frame_absolute(Register addr, Register tmp1, Register tmp2) { 931 // (addr == tmp1) || (addr == tmp2) is allowed here! 932 assert(tmp1 != tmp2, "must be distinct"); 933 934 // compute offset w.r.t. current stack pointer 935 // tmp_1 <- addr - SP (!) 936 subf(tmp1, R1_SP, addr); 937 938 // atomically update SP keeping back link. 939 resize_frame(tmp1/* offset */, tmp2/* tmp */); 940 } 941 942 void MacroAssembler::push_frame(Register bytes, Register tmp) { 943 #ifdef ASSERT 944 assert(bytes != R0, "r0 not allowed here"); 945 andi_(R0, bytes, frame::alignment_in_bytes-1); 946 asm_assert_eq("push_frame(Reg, Reg): unaligned", 0x203); 947 #endif 948 neg(tmp, bytes); 949 stdux(R1_SP, R1_SP, tmp); 950 } 951 952 // Push a frame of size `bytes'. 953 void MacroAssembler::push_frame(unsigned int bytes, Register tmp) { 954 long offset = align_addr(bytes, frame::alignment_in_bytes); 955 if (is_simm(-offset, 16)) { 956 stdu(R1_SP, -offset, R1_SP); 957 } else { 958 load_const_optimized(tmp, -offset); 959 stdux(R1_SP, R1_SP, tmp); 960 } 961 } 962 963 // Push a frame of size `bytes' plus abi_reg_args on top. 964 void MacroAssembler::push_frame_reg_args(unsigned int bytes, Register tmp) { 965 push_frame(bytes + frame::abi_reg_args_size, tmp); 966 } 967 968 // Setup up a new C frame with a spill area for non-volatile GPRs and 969 // additional space for local variables. 970 void MacroAssembler::push_frame_reg_args_nonvolatiles(unsigned int bytes, 971 Register tmp) { 972 push_frame(bytes + frame::abi_reg_args_size + frame::spill_nonvolatiles_size, tmp); 973 } 974 975 // Pop current C frame. 976 void MacroAssembler::pop_frame() { 977 ld(R1_SP, _abi(callers_sp), R1_SP); 978 } 979 980 #if defined(ABI_ELFv2) 981 address MacroAssembler::branch_to(Register r_function_entry, bool and_link) { 982 // TODO(asmundak): make sure the caller uses R12 as function descriptor 983 // most of the times. 984 if (R12 != r_function_entry) { 985 mr(R12, r_function_entry); 986 } 987 mtctr(R12); 988 // Do a call or a branch. 989 if (and_link) { 990 bctrl(); 991 } else { 992 bctr(); 993 } 994 _last_calls_return_pc = pc(); 995 996 return _last_calls_return_pc; 997 } 998 999 // Call a C function via a function descriptor and use full C 1000 // calling conventions. Updates and returns _last_calls_return_pc. 1001 address MacroAssembler::call_c(Register r_function_entry) { 1002 return branch_to(r_function_entry, /*and_link=*/true); 1003 } 1004 1005 // For tail calls: only branch, don't link, so callee returns to caller of this function. 1006 address MacroAssembler::call_c_and_return_to_caller(Register r_function_entry) { 1007 return branch_to(r_function_entry, /*and_link=*/false); 1008 } 1009 1010 address MacroAssembler::call_c(address function_entry, relocInfo::relocType rt) { 1011 load_const(R12, function_entry, R0); 1012 return branch_to(R12, /*and_link=*/true); 1013 } 1014 1015 #else 1016 // Generic version of a call to C function via a function descriptor 1017 // with variable support for C calling conventions (TOC, ENV, etc.). 1018 // Updates and returns _last_calls_return_pc. 1019 address MacroAssembler::branch_to(Register function_descriptor, bool and_link, bool save_toc_before_call, 1020 bool restore_toc_after_call, bool load_toc_of_callee, bool load_env_of_callee) { 1021 // we emit standard ptrgl glue code here 1022 assert((function_descriptor != R0), "function_descriptor cannot be R0"); 1023 1024 // retrieve necessary entries from the function descriptor 1025 ld(R0, in_bytes(FunctionDescriptor::entry_offset()), function_descriptor); 1026 mtctr(R0); 1027 1028 if (load_toc_of_callee) { 1029 ld(R2_TOC, in_bytes(FunctionDescriptor::toc_offset()), function_descriptor); 1030 } 1031 if (load_env_of_callee) { 1032 ld(R11, in_bytes(FunctionDescriptor::env_offset()), function_descriptor); 1033 } else if (load_toc_of_callee) { 1034 li(R11, 0); 1035 } 1036 1037 // do a call or a branch 1038 if (and_link) { 1039 bctrl(); 1040 } else { 1041 bctr(); 1042 } 1043 _last_calls_return_pc = pc(); 1044 1045 return _last_calls_return_pc; 1046 } 1047 1048 // Call a C function via a function descriptor and use full C calling 1049 // conventions. 1050 // We don't use the TOC in generated code, so there is no need to save 1051 // and restore its value. 1052 address MacroAssembler::call_c(Register fd) { 1053 return branch_to(fd, /*and_link=*/true, 1054 /*save toc=*/false, 1055 /*restore toc=*/false, 1056 /*load toc=*/true, 1057 /*load env=*/true); 1058 } 1059 1060 address MacroAssembler::call_c_and_return_to_caller(Register fd) { 1061 return branch_to(fd, /*and_link=*/false, 1062 /*save toc=*/false, 1063 /*restore toc=*/false, 1064 /*load toc=*/true, 1065 /*load env=*/true); 1066 } 1067 1068 address MacroAssembler::call_c(const FunctionDescriptor* fd, relocInfo::relocType rt) { 1069 if (rt != relocInfo::none) { 1070 // this call needs to be relocatable 1071 if (!ReoptimizeCallSequences 1072 || (rt != relocInfo::runtime_call_type && rt != relocInfo::none) 1073 || fd == NULL // support code-size estimation 1074 || !fd->is_friend_function() 1075 || fd->entry() == NULL) { 1076 // it's not a friend function as defined by class FunctionDescriptor, 1077 // so do a full call-c here. 1078 load_const(R11, (address)fd, R0); 1079 1080 bool has_env = (fd != NULL && fd->env() != NULL); 1081 return branch_to(R11, /*and_link=*/true, 1082 /*save toc=*/false, 1083 /*restore toc=*/false, 1084 /*load toc=*/true, 1085 /*load env=*/has_env); 1086 } else { 1087 // It's a friend function. Load the entry point and don't care about 1088 // toc and env. Use an optimizable call instruction, but ensure the 1089 // same code-size as in the case of a non-friend function. 1090 nop(); 1091 nop(); 1092 nop(); 1093 bl64_patchable(fd->entry(), rt); 1094 _last_calls_return_pc = pc(); 1095 return _last_calls_return_pc; 1096 } 1097 } else { 1098 // This call does not need to be relocatable, do more aggressive 1099 // optimizations. 1100 if (!ReoptimizeCallSequences 1101 || !fd->is_friend_function()) { 1102 // It's not a friend function as defined by class FunctionDescriptor, 1103 // so do a full call-c here. 1104 load_const(R11, (address)fd, R0); 1105 return branch_to(R11, /*and_link=*/true, 1106 /*save toc=*/false, 1107 /*restore toc=*/false, 1108 /*load toc=*/true, 1109 /*load env=*/true); 1110 } else { 1111 // it's a friend function, load the entry point and don't care about 1112 // toc and env. 1113 address dest = fd->entry(); 1114 if (is_within_range_of_b(dest, pc())) { 1115 bl(dest); 1116 } else { 1117 bl64_patchable(dest, rt); 1118 } 1119 _last_calls_return_pc = pc(); 1120 return _last_calls_return_pc; 1121 } 1122 } 1123 } 1124 1125 // Call a C function. All constants needed reside in TOC. 1126 // 1127 // Read the address to call from the TOC. 1128 // Read env from TOC, if fd specifies an env. 1129 // Read new TOC from TOC. 1130 address MacroAssembler::call_c_using_toc(const FunctionDescriptor* fd, 1131 relocInfo::relocType rt, Register toc) { 1132 if (!ReoptimizeCallSequences 1133 || (rt != relocInfo::runtime_call_type && rt != relocInfo::none) 1134 || !fd->is_friend_function()) { 1135 // It's not a friend function as defined by class FunctionDescriptor, 1136 // so do a full call-c here. 1137 assert(fd->entry() != NULL, "function must be linked"); 1138 1139 AddressLiteral fd_entry(fd->entry()); 1140 bool success = load_const_from_method_toc(R11, fd_entry, toc, /*fixed_size*/ true); 1141 mtctr(R11); 1142 if (fd->env() == NULL) { 1143 li(R11, 0); 1144 nop(); 1145 } else { 1146 AddressLiteral fd_env(fd->env()); 1147 success = success && load_const_from_method_toc(R11, fd_env, toc, /*fixed_size*/ true); 1148 } 1149 AddressLiteral fd_toc(fd->toc()); 1150 // Set R2_TOC (load from toc) 1151 success = success && load_const_from_method_toc(R2_TOC, fd_toc, toc, /*fixed_size*/ true); 1152 bctrl(); 1153 _last_calls_return_pc = pc(); 1154 if (!success) { return NULL; } 1155 } else { 1156 // It's a friend function, load the entry point and don't care about 1157 // toc and env. Use an optimizable call instruction, but ensure the 1158 // same code-size as in the case of a non-friend function. 1159 nop(); 1160 bl64_patchable(fd->entry(), rt); 1161 _last_calls_return_pc = pc(); 1162 } 1163 return _last_calls_return_pc; 1164 } 1165 #endif // ABI_ELFv2 1166 1167 void MacroAssembler::call_VM_base(Register oop_result, 1168 Register last_java_sp, 1169 address entry_point, 1170 bool check_exceptions) { 1171 BLOCK_COMMENT("call_VM {"); 1172 // Determine last_java_sp register. 1173 if (!last_java_sp->is_valid()) { 1174 last_java_sp = R1_SP; 1175 } 1176 set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1); 1177 1178 // ARG1 must hold thread address. 1179 mr(R3_ARG1, R16_thread); 1180 #if defined(ABI_ELFv2) 1181 address return_pc = call_c(entry_point, relocInfo::none); 1182 #else 1183 address return_pc = call_c((FunctionDescriptor*)entry_point, relocInfo::none); 1184 #endif 1185 1186 reset_last_Java_frame(); 1187 1188 // Check for pending exceptions. 1189 if (check_exceptions) { 1190 // We don't check for exceptions here. 1191 ShouldNotReachHere(); 1192 } 1193 1194 // Get oop result if there is one and reset the value in the thread. 1195 if (oop_result->is_valid()) { 1196 get_vm_result(oop_result); 1197 } 1198 1199 _last_calls_return_pc = return_pc; 1200 BLOCK_COMMENT("} call_VM"); 1201 } 1202 1203 void MacroAssembler::call_VM_leaf_base(address entry_point) { 1204 BLOCK_COMMENT("call_VM_leaf {"); 1205 #if defined(ABI_ELFv2) 1206 call_c(entry_point, relocInfo::none); 1207 #else 1208 call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, entry_point), relocInfo::none); 1209 #endif 1210 BLOCK_COMMENT("} call_VM_leaf"); 1211 } 1212 1213 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) { 1214 call_VM_base(oop_result, noreg, entry_point, check_exceptions); 1215 } 1216 1217 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, 1218 bool check_exceptions) { 1219 // R3_ARG1 is reserved for the thread. 1220 mr_if_needed(R4_ARG2, arg_1); 1221 call_VM(oop_result, entry_point, check_exceptions); 1222 } 1223 1224 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, 1225 bool check_exceptions) { 1226 // R3_ARG1 is reserved for the thread 1227 mr_if_needed(R4_ARG2, arg_1); 1228 assert(arg_2 != R4_ARG2, "smashed argument"); 1229 mr_if_needed(R5_ARG3, arg_2); 1230 call_VM(oop_result, entry_point, check_exceptions); 1231 } 1232 1233 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3, 1234 bool check_exceptions) { 1235 // R3_ARG1 is reserved for the thread 1236 mr_if_needed(R4_ARG2, arg_1); 1237 assert(arg_2 != R4_ARG2, "smashed argument"); 1238 mr_if_needed(R5_ARG3, arg_2); 1239 mr_if_needed(R6_ARG4, arg_3); 1240 call_VM(oop_result, entry_point, check_exceptions); 1241 } 1242 1243 void MacroAssembler::call_VM_leaf(address entry_point) { 1244 call_VM_leaf_base(entry_point); 1245 } 1246 1247 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) { 1248 mr_if_needed(R3_ARG1, arg_1); 1249 call_VM_leaf(entry_point); 1250 } 1251 1252 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) { 1253 mr_if_needed(R3_ARG1, arg_1); 1254 assert(arg_2 != R3_ARG1, "smashed argument"); 1255 mr_if_needed(R4_ARG2, arg_2); 1256 call_VM_leaf(entry_point); 1257 } 1258 1259 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) { 1260 mr_if_needed(R3_ARG1, arg_1); 1261 assert(arg_2 != R3_ARG1, "smashed argument"); 1262 mr_if_needed(R4_ARG2, arg_2); 1263 assert(arg_3 != R3_ARG1 && arg_3 != R4_ARG2, "smashed argument"); 1264 mr_if_needed(R5_ARG3, arg_3); 1265 call_VM_leaf(entry_point); 1266 } 1267 1268 // Check whether instruction is a read access to the polling page 1269 // which was emitted by load_from_polling_page(..). 1270 bool MacroAssembler::is_load_from_polling_page(int instruction, void* ucontext, 1271 address* polling_address_ptr) { 1272 if (!is_ld(instruction)) 1273 return false; // It's not a ld. Fail. 1274 1275 int rt = inv_rt_field(instruction); 1276 int ra = inv_ra_field(instruction); 1277 int ds = inv_ds_field(instruction); 1278 if (!(ds == 0 && ra != 0 && rt == 0)) { 1279 return false; // It's not a ld(r0, X, ra). Fail. 1280 } 1281 1282 if (!ucontext) { 1283 // Set polling address. 1284 if (polling_address_ptr != NULL) { 1285 *polling_address_ptr = NULL; 1286 } 1287 return true; // No ucontext given. Can't check value of ra. Assume true. 1288 } 1289 1290 #ifdef LINUX 1291 // Ucontext given. Check that register ra contains the address of 1292 // the safepoing polling page. 1293 ucontext_t* uc = (ucontext_t*) ucontext; 1294 // Set polling address. 1295 address addr = (address)uc->uc_mcontext.regs->gpr[ra] + (ssize_t)ds; 1296 if (polling_address_ptr != NULL) { 1297 *polling_address_ptr = addr; 1298 } 1299 return os::is_poll_address(addr); 1300 #else 1301 // Not on Linux, ucontext must be NULL. 1302 ShouldNotReachHere(); 1303 return false; 1304 #endif 1305 } 1306 1307 bool MacroAssembler::is_memory_serialization(int instruction, JavaThread* thread, void* ucontext) { 1308 #ifdef LINUX 1309 ucontext_t* uc = (ucontext_t*) ucontext; 1310 1311 if (is_stwx(instruction) || is_stwux(instruction)) { 1312 int ra = inv_ra_field(instruction); 1313 int rb = inv_rb_field(instruction); 1314 1315 // look up content of ra and rb in ucontext 1316 address ra_val=(address)uc->uc_mcontext.regs->gpr[ra]; 1317 long rb_val=(long)uc->uc_mcontext.regs->gpr[rb]; 1318 return os::is_memory_serialize_page(thread, ra_val+rb_val); 1319 } else if (is_stw(instruction) || is_stwu(instruction)) { 1320 int ra = inv_ra_field(instruction); 1321 int d1 = inv_d1_field(instruction); 1322 1323 // look up content of ra in ucontext 1324 address ra_val=(address)uc->uc_mcontext.regs->gpr[ra]; 1325 return os::is_memory_serialize_page(thread, ra_val+d1); 1326 } else { 1327 return false; 1328 } 1329 #else 1330 // workaround not needed on !LINUX :-) 1331 ShouldNotCallThis(); 1332 return false; 1333 #endif 1334 } 1335 1336 void MacroAssembler::bang_stack_with_offset(int offset) { 1337 // When increasing the stack, the old stack pointer will be written 1338 // to the new top of stack according to the PPC64 abi. 1339 // Therefore, stack banging is not necessary when increasing 1340 // the stack by <= os::vm_page_size() bytes. 1341 // When increasing the stack by a larger amount, this method is 1342 // called repeatedly to bang the intermediate pages. 1343 1344 // Stack grows down, caller passes positive offset. 1345 assert(offset > 0, "must bang with positive offset"); 1346 1347 long stdoffset = -offset; 1348 1349 if (is_simm(stdoffset, 16)) { 1350 // Signed 16 bit offset, a simple std is ok. 1351 if (UseLoadInstructionsForStackBangingPPC64) { 1352 ld(R0, (int)(signed short)stdoffset, R1_SP); 1353 } else { 1354 std(R0,(int)(signed short)stdoffset, R1_SP); 1355 } 1356 } else if (is_simm(stdoffset, 31)) { 1357 const int hi = MacroAssembler::largeoffset_si16_si16_hi(stdoffset); 1358 const int lo = MacroAssembler::largeoffset_si16_si16_lo(stdoffset); 1359 1360 Register tmp = R11; 1361 addis(tmp, R1_SP, hi); 1362 if (UseLoadInstructionsForStackBangingPPC64) { 1363 ld(R0, lo, tmp); 1364 } else { 1365 std(R0, lo, tmp); 1366 } 1367 } else { 1368 ShouldNotReachHere(); 1369 } 1370 } 1371 1372 // If instruction is a stack bang of the form 1373 // std R0, x(Ry), (see bang_stack_with_offset()) 1374 // stdu R1_SP, x(R1_SP), (see push_frame(), resize_frame()) 1375 // or stdux R1_SP, Rx, R1_SP (see push_frame(), resize_frame()) 1376 // return the banged address. Otherwise, return 0. 1377 address MacroAssembler::get_stack_bang_address(int instruction, void *ucontext) { 1378 #ifdef LINUX 1379 ucontext_t* uc = (ucontext_t*) ucontext; 1380 int rs = inv_rs_field(instruction); 1381 int ra = inv_ra_field(instruction); 1382 if ( (is_ld(instruction) && rs == 0 && UseLoadInstructionsForStackBangingPPC64) 1383 || (is_std(instruction) && rs == 0 && !UseLoadInstructionsForStackBangingPPC64) 1384 || (is_stdu(instruction) && rs == 1)) { 1385 int ds = inv_ds_field(instruction); 1386 // return banged address 1387 return ds+(address)uc->uc_mcontext.regs->gpr[ra]; 1388 } else if (is_stdux(instruction) && rs == 1) { 1389 int rb = inv_rb_field(instruction); 1390 address sp = (address)uc->uc_mcontext.regs->gpr[1]; 1391 long rb_val = (long)uc->uc_mcontext.regs->gpr[rb]; 1392 return ra != 1 || rb_val >= 0 ? NULL // not a stack bang 1393 : sp + rb_val; // banged address 1394 } 1395 return NULL; // not a stack bang 1396 #else 1397 // workaround not needed on !LINUX :-) 1398 ShouldNotCallThis(); 1399 return NULL; 1400 #endif 1401 } 1402 1403 void MacroAssembler::reserved_stack_check(Register return_pc) { 1404 // Test if reserved zone needs to be enabled. 1405 Label no_reserved_zone_enabling; 1406 1407 ld_ptr(R0, JavaThread::reserved_stack_activation_offset(), R16_thread); 1408 cmpld(CCR0, R1_SP, R0); 1409 blt_predict_taken(CCR0, no_reserved_zone_enabling); 1410 1411 // Enable reserved zone again, throw stack overflow exception. 1412 push_frame_reg_args(0, R0); 1413 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), R16_thread); 1414 pop_frame(); 1415 mtlr(return_pc); 1416 load_const_optimized(R0, StubRoutines::throw_delayed_StackOverflowError_entry()); 1417 mtctr(R0); 1418 bctr(); 1419 1420 should_not_reach_here(); 1421 1422 bind(no_reserved_zone_enabling); 1423 } 1424 1425 void MacroAssembler::getandsetd(Register dest_current_value, Register exchange_value, Register addr_base, 1426 bool cmpxchgx_hint) { 1427 Label retry; 1428 bind(retry); 1429 ldarx(dest_current_value, addr_base, cmpxchgx_hint); 1430 stdcx_(exchange_value, addr_base); 1431 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1432 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0. 1433 } else { 1434 bne( CCR0, retry); // StXcx_ sets CCR0. 1435 } 1436 } 1437 1438 void MacroAssembler::getandaddd(Register dest_current_value, Register inc_value, Register addr_base, 1439 Register tmp, bool cmpxchgx_hint) { 1440 Label retry; 1441 bind(retry); 1442 ldarx(dest_current_value, addr_base, cmpxchgx_hint); 1443 add(tmp, dest_current_value, inc_value); 1444 stdcx_(tmp, addr_base); 1445 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1446 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0. 1447 } else { 1448 bne( CCR0, retry); // StXcx_ sets CCR0. 1449 } 1450 } 1451 1452 // Word/sub-word atomic helper functions 1453 1454 // Temps and addr_base are killed if size < 4 and processor does not support respective instructions. 1455 // Only signed types are supported with size < 4. 1456 // Atomic add always kills tmp1. 1457 void MacroAssembler::atomic_get_and_modify_generic(Register dest_current_value, Register exchange_value, 1458 Register addr_base, Register tmp1, Register tmp2, Register tmp3, 1459 bool cmpxchgx_hint, bool is_add, int size) { 1460 // Sub-word instructions are available since Power 8. 1461 // For older processors, instruction_type != size holds, and we 1462 // emulate the sub-word instructions by constructing a 4-byte value 1463 // that leaves the other bytes unchanged. 1464 const int instruction_type = VM_Version::has_lqarx() ? size : 4; 1465 1466 Label retry; 1467 Register shift_amount = noreg, 1468 val32 = dest_current_value, 1469 modval = is_add ? tmp1 : exchange_value; 1470 1471 if (instruction_type != size) { 1472 assert_different_registers(tmp1, tmp2, tmp3, dest_current_value, exchange_value, addr_base); 1473 modval = tmp1; 1474 shift_amount = tmp2; 1475 val32 = tmp3; 1476 // Need some preperation: Compute shift amount, align address. Note: shorts must be 2 byte aligned. 1477 #ifdef VM_LITTLE_ENDIAN 1478 rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8; 1479 clrrdi(addr_base, addr_base, 2); 1480 #else 1481 xori(shift_amount, addr_base, (size == 1) ? 3 : 2); 1482 clrrdi(addr_base, addr_base, 2); 1483 rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16; 1484 #endif 1485 } 1486 1487 // atomic emulation loop 1488 bind(retry); 1489 1490 switch (instruction_type) { 1491 case 4: lwarx(val32, addr_base, cmpxchgx_hint); break; 1492 case 2: lharx(val32, addr_base, cmpxchgx_hint); break; 1493 case 1: lbarx(val32, addr_base, cmpxchgx_hint); break; 1494 default: ShouldNotReachHere(); 1495 } 1496 1497 if (instruction_type != size) { 1498 srw(dest_current_value, val32, shift_amount); 1499 } 1500 1501 if (is_add) { add(modval, dest_current_value, exchange_value); } 1502 1503 if (instruction_type != size) { 1504 // Transform exchange value such that the replacement can be done by one xor instruction. 1505 xorr(modval, dest_current_value, is_add ? modval : exchange_value); 1506 clrldi(modval, modval, (size == 1) ? 56 : 48); 1507 slw(modval, modval, shift_amount); 1508 xorr(modval, val32, modval); 1509 } 1510 1511 switch (instruction_type) { 1512 case 4: stwcx_(modval, addr_base); break; 1513 case 2: sthcx_(modval, addr_base); break; 1514 case 1: stbcx_(modval, addr_base); break; 1515 default: ShouldNotReachHere(); 1516 } 1517 1518 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1519 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0. 1520 } else { 1521 bne( CCR0, retry); // StXcx_ sets CCR0. 1522 } 1523 1524 // l?arx zero-extends, but Java wants byte/short values sign-extended. 1525 if (size == 1) { 1526 extsb(dest_current_value, dest_current_value); 1527 } else if (size == 2) { 1528 extsh(dest_current_value, dest_current_value); 1529 }; 1530 } 1531 1532 // Temps, addr_base and exchange_value are killed if size < 4 and processor does not support respective instructions. 1533 // Only signed types are supported with size < 4. 1534 void MacroAssembler::cmpxchg_loop_body(ConditionRegister flag, Register dest_current_value, 1535 Register compare_value, Register exchange_value, 1536 Register addr_base, Register tmp1, Register tmp2, 1537 Label &retry, Label &failed, bool cmpxchgx_hint, int size) { 1538 // Sub-word instructions are available since Power 8. 1539 // For older processors, instruction_type != size holds, and we 1540 // emulate the sub-word instructions by constructing a 4-byte value 1541 // that leaves the other bytes unchanged. 1542 const int instruction_type = VM_Version::has_lqarx() ? size : 4; 1543 1544 Register shift_amount = noreg, 1545 val32 = dest_current_value, 1546 modval = exchange_value; 1547 1548 if (instruction_type != size) { 1549 assert_different_registers(tmp1, tmp2, dest_current_value, compare_value, exchange_value, addr_base); 1550 shift_amount = tmp1; 1551 val32 = tmp2; 1552 modval = tmp2; 1553 // Need some preperation: Compute shift amount, align address. Note: shorts must be 2 byte aligned. 1554 #ifdef VM_LITTLE_ENDIAN 1555 rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8; 1556 clrrdi(addr_base, addr_base, 2); 1557 #else 1558 xori(shift_amount, addr_base, (size == 1) ? 3 : 2); 1559 clrrdi(addr_base, addr_base, 2); 1560 rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16; 1561 #endif 1562 // Transform exchange value such that the replacement can be done by one xor instruction. 1563 xorr(exchange_value, compare_value, exchange_value); 1564 clrldi(exchange_value, exchange_value, (size == 1) ? 56 : 48); 1565 slw(exchange_value, exchange_value, shift_amount); 1566 } 1567 1568 // atomic emulation loop 1569 bind(retry); 1570 1571 switch (instruction_type) { 1572 case 4: lwarx(val32, addr_base, cmpxchgx_hint); break; 1573 case 2: lharx(val32, addr_base, cmpxchgx_hint); break; 1574 case 1: lbarx(val32, addr_base, cmpxchgx_hint); break; 1575 default: ShouldNotReachHere(); 1576 } 1577 1578 if (instruction_type != size) { 1579 srw(dest_current_value, val32, shift_amount); 1580 } 1581 if (size == 1) { 1582 extsb(dest_current_value, dest_current_value); 1583 } else if (size == 2) { 1584 extsh(dest_current_value, dest_current_value); 1585 }; 1586 1587 cmpw(flag, dest_current_value, compare_value); 1588 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1589 bne_predict_not_taken(flag, failed); 1590 } else { 1591 bne( flag, failed); 1592 } 1593 // branch to done => (flag == ne), (dest_current_value != compare_value) 1594 // fall through => (flag == eq), (dest_current_value == compare_value) 1595 1596 if (instruction_type != size) { 1597 xorr(modval, val32, exchange_value); 1598 } 1599 1600 switch (instruction_type) { 1601 case 4: stwcx_(modval, addr_base); break; 1602 case 2: sthcx_(modval, addr_base); break; 1603 case 1: stbcx_(modval, addr_base); break; 1604 default: ShouldNotReachHere(); 1605 } 1606 } 1607 1608 // CmpxchgX sets condition register to cmpX(current, compare). 1609 void MacroAssembler::cmpxchg_generic(ConditionRegister flag, Register dest_current_value, 1610 Register compare_value, Register exchange_value, 1611 Register addr_base, Register tmp1, Register tmp2, 1612 int semantics, bool cmpxchgx_hint, 1613 Register int_flag_success, bool contention_hint, bool weak, int size) { 1614 Label retry; 1615 Label failed; 1616 Label done; 1617 1618 // Save one branch if result is returned via register and 1619 // result register is different from the other ones. 1620 bool use_result_reg = (int_flag_success != noreg); 1621 bool preset_result_reg = (int_flag_success != dest_current_value && int_flag_success != compare_value && 1622 int_flag_success != exchange_value && int_flag_success != addr_base && 1623 int_flag_success != tmp1 && int_flag_success != tmp2); 1624 assert(!weak || flag == CCR0, "weak only supported with CCR0"); 1625 assert(size == 1 || size == 2 || size == 4, "unsupported"); 1626 1627 if (use_result_reg && preset_result_reg) { 1628 li(int_flag_success, 0); // preset (assume cas failed) 1629 } 1630 1631 // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM). 1632 if (contention_hint) { // Don't try to reserve if cmp fails. 1633 switch (size) { 1634 case 1: lbz(dest_current_value, 0, addr_base); extsb(dest_current_value, dest_current_value); break; 1635 case 2: lha(dest_current_value, 0, addr_base); break; 1636 case 4: lwz(dest_current_value, 0, addr_base); break; 1637 default: ShouldNotReachHere(); 1638 } 1639 cmpw(flag, dest_current_value, compare_value); 1640 bne(flag, failed); 1641 } 1642 1643 // release/fence semantics 1644 if (semantics & MemBarRel) { 1645 release(); 1646 } 1647 1648 cmpxchg_loop_body(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2, 1649 retry, failed, cmpxchgx_hint, size); 1650 if (!weak || use_result_reg) { 1651 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1652 bne_predict_not_taken(CCR0, weak ? failed : retry); // StXcx_ sets CCR0. 1653 } else { 1654 bne( CCR0, weak ? failed : retry); // StXcx_ sets CCR0. 1655 } 1656 } 1657 // fall through => (flag == eq), (dest_current_value == compare_value), (swapped) 1658 1659 // Result in register (must do this at the end because int_flag_success can be the 1660 // same register as one above). 1661 if (use_result_reg) { 1662 li(int_flag_success, 1); 1663 } 1664 1665 if (semantics & MemBarFenceAfter) { 1666 fence(); 1667 } else if (semantics & MemBarAcq) { 1668 isync(); 1669 } 1670 1671 if (use_result_reg && !preset_result_reg) { 1672 b(done); 1673 } 1674 1675 bind(failed); 1676 if (use_result_reg && !preset_result_reg) { 1677 li(int_flag_success, 0); 1678 } 1679 1680 bind(done); 1681 // (flag == ne) => (dest_current_value != compare_value), (!swapped) 1682 // (flag == eq) => (dest_current_value == compare_value), ( swapped) 1683 } 1684 1685 // Preforms atomic compare exchange: 1686 // if (compare_value == *addr_base) 1687 // *addr_base = exchange_value 1688 // int_flag_success = 1; 1689 // else 1690 // int_flag_success = 0; 1691 // 1692 // ConditionRegister flag = cmp(compare_value, *addr_base) 1693 // Register dest_current_value = *addr_base 1694 // Register compare_value Used to compare with value in memory 1695 // Register exchange_value Written to memory if compare_value == *addr_base 1696 // Register addr_base The memory location to compareXChange 1697 // Register int_flag_success Set to 1 if exchange_value was written to *addr_base 1698 // 1699 // To avoid the costly compare exchange the value is tested beforehand. 1700 // Several special cases exist to avoid that unnecessary information is generated. 1701 // 1702 void MacroAssembler::cmpxchgd(ConditionRegister flag, 1703 Register dest_current_value, RegisterOrConstant compare_value, Register exchange_value, 1704 Register addr_base, int semantics, bool cmpxchgx_hint, 1705 Register int_flag_success, Label* failed_ext, bool contention_hint, bool weak) { 1706 Label retry; 1707 Label failed_int; 1708 Label& failed = (failed_ext != NULL) ? *failed_ext : failed_int; 1709 Label done; 1710 1711 // Save one branch if result is returned via register and result register is different from the other ones. 1712 bool use_result_reg = (int_flag_success!=noreg); 1713 bool preset_result_reg = (int_flag_success!=dest_current_value && int_flag_success!=compare_value.register_or_noreg() && 1714 int_flag_success!=exchange_value && int_flag_success!=addr_base); 1715 assert(!weak || flag == CCR0, "weak only supported with CCR0"); 1716 assert(int_flag_success == noreg || failed_ext == NULL, "cannot have both"); 1717 1718 if (use_result_reg && preset_result_reg) { 1719 li(int_flag_success, 0); // preset (assume cas failed) 1720 } 1721 1722 // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM). 1723 if (contention_hint) { // Don't try to reserve if cmp fails. 1724 ld(dest_current_value, 0, addr_base); 1725 cmpd(flag, compare_value, dest_current_value); 1726 bne(flag, failed); 1727 } 1728 1729 // release/fence semantics 1730 if (semantics & MemBarRel) { 1731 release(); 1732 } 1733 1734 // atomic emulation loop 1735 bind(retry); 1736 1737 ldarx(dest_current_value, addr_base, cmpxchgx_hint); 1738 cmpd(flag, compare_value, dest_current_value); 1739 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1740 bne_predict_not_taken(flag, failed); 1741 } else { 1742 bne( flag, failed); 1743 } 1744 1745 stdcx_(exchange_value, addr_base); 1746 if (!weak || use_result_reg || failed_ext) { 1747 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 1748 bne_predict_not_taken(CCR0, weak ? failed : retry); // stXcx_ sets CCR0 1749 } else { 1750 bne( CCR0, weak ? failed : retry); // stXcx_ sets CCR0 1751 } 1752 } 1753 1754 // result in register (must do this at the end because int_flag_success can be the same register as one above) 1755 if (use_result_reg) { 1756 li(int_flag_success, 1); 1757 } 1758 1759 if (semantics & MemBarFenceAfter) { 1760 fence(); 1761 } else if (semantics & MemBarAcq) { 1762 isync(); 1763 } 1764 1765 if (use_result_reg && !preset_result_reg) { 1766 b(done); 1767 } 1768 1769 bind(failed_int); 1770 if (use_result_reg && !preset_result_reg) { 1771 li(int_flag_success, 0); 1772 } 1773 1774 bind(done); 1775 // (flag == ne) => (dest_current_value != compare_value), (!swapped) 1776 // (flag == eq) => (dest_current_value == compare_value), ( swapped) 1777 } 1778 1779 // Look up the method for a megamorphic invokeinterface call. 1780 // The target method is determined by <intf_klass, itable_index>. 1781 // The receiver klass is in recv_klass. 1782 // On success, the result will be in method_result, and execution falls through. 1783 // On failure, execution transfers to the given label. 1784 void MacroAssembler::lookup_interface_method(Register recv_klass, 1785 Register intf_klass, 1786 RegisterOrConstant itable_index, 1787 Register method_result, 1788 Register scan_temp, 1789 Register sethi_temp, 1790 Label& L_no_such_interface) { 1791 assert_different_registers(recv_klass, intf_klass, method_result, scan_temp); 1792 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 1793 "caller must use same register for non-constant itable index as for method"); 1794 1795 // Compute start of first itableOffsetEntry (which is at the end of the vtable). 1796 int vtable_base = in_bytes(Klass::vtable_start_offset()); 1797 int itentry_off = itableMethodEntry::method_offset_in_bytes(); 1798 int logMEsize = exact_log2(itableMethodEntry::size() * wordSize); 1799 int scan_step = itableOffsetEntry::size() * wordSize; 1800 int log_vte_size= exact_log2(vtableEntry::size_in_bytes()); 1801 1802 lwz(scan_temp, in_bytes(Klass::vtable_length_offset()), recv_klass); 1803 // %%% We should store the aligned, prescaled offset in the klassoop. 1804 // Then the next several instructions would fold away. 1805 1806 sldi(scan_temp, scan_temp, log_vte_size); 1807 addi(scan_temp, scan_temp, vtable_base); 1808 add(scan_temp, recv_klass, scan_temp); 1809 1810 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 1811 if (itable_index.is_register()) { 1812 Register itable_offset = itable_index.as_register(); 1813 sldi(itable_offset, itable_offset, logMEsize); 1814 if (itentry_off) addi(itable_offset, itable_offset, itentry_off); 1815 add(recv_klass, itable_offset, recv_klass); 1816 } else { 1817 long itable_offset = (long)itable_index.as_constant(); 1818 load_const_optimized(sethi_temp, (itable_offset<<logMEsize)+itentry_off); // static address, no relocation 1819 add(recv_klass, sethi_temp, recv_klass); 1820 } 1821 1822 // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) { 1823 // if (scan->interface() == intf) { 1824 // result = (klass + scan->offset() + itable_index); 1825 // } 1826 // } 1827 Label search, found_method; 1828 1829 for (int peel = 1; peel >= 0; peel--) { 1830 // %%%% Could load both offset and interface in one ldx, if they were 1831 // in the opposite order. This would save a load. 1832 ld(method_result, itableOffsetEntry::interface_offset_in_bytes(), scan_temp); 1833 1834 // Check that this entry is non-null. A null entry means that 1835 // the receiver class doesn't implement the interface, and wasn't the 1836 // same as when the caller was compiled. 1837 cmpd(CCR0, method_result, intf_klass); 1838 1839 if (peel) { 1840 beq(CCR0, found_method); 1841 } else { 1842 bne(CCR0, search); 1843 // (invert the test to fall through to found_method...) 1844 } 1845 1846 if (!peel) break; 1847 1848 bind(search); 1849 1850 cmpdi(CCR0, method_result, 0); 1851 beq(CCR0, L_no_such_interface); 1852 addi(scan_temp, scan_temp, scan_step); 1853 } 1854 1855 bind(found_method); 1856 1857 // Got a hit. 1858 int ito_offset = itableOffsetEntry::offset_offset_in_bytes(); 1859 lwz(scan_temp, ito_offset, scan_temp); 1860 ldx(method_result, scan_temp, recv_klass); 1861 } 1862 1863 // virtual method calling 1864 void MacroAssembler::lookup_virtual_method(Register recv_klass, 1865 RegisterOrConstant vtable_index, 1866 Register method_result) { 1867 1868 assert_different_registers(recv_klass, method_result, vtable_index.register_or_noreg()); 1869 1870 const int base = in_bytes(Klass::vtable_start_offset()); 1871 assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 1872 1873 if (vtable_index.is_register()) { 1874 sldi(vtable_index.as_register(), vtable_index.as_register(), LogBytesPerWord); 1875 add(recv_klass, vtable_index.as_register(), recv_klass); 1876 } else { 1877 addi(recv_klass, recv_klass, vtable_index.as_constant() << LogBytesPerWord); 1878 } 1879 ld(R19_method, base + vtableEntry::method_offset_in_bytes(), recv_klass); 1880 } 1881 1882 /////////////////////////////////////////// subtype checking //////////////////////////////////////////// 1883 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 1884 Register super_klass, 1885 Register temp1_reg, 1886 Register temp2_reg, 1887 Label* L_success, 1888 Label* L_failure, 1889 Label* L_slow_path, 1890 RegisterOrConstant super_check_offset) { 1891 1892 const Register check_cache_offset = temp1_reg; 1893 const Register cached_super = temp2_reg; 1894 1895 assert_different_registers(sub_klass, super_klass, check_cache_offset, cached_super); 1896 1897 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 1898 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 1899 1900 bool must_load_sco = (super_check_offset.constant_or_zero() == -1); 1901 bool need_slow_path = (must_load_sco || super_check_offset.constant_or_zero() == sco_offset); 1902 1903 Label L_fallthrough; 1904 int label_nulls = 0; 1905 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } 1906 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } 1907 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; } 1908 assert(label_nulls <= 1 || 1909 (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path), 1910 "at most one NULL in the batch, usually"); 1911 1912 // If the pointers are equal, we are done (e.g., String[] elements). 1913 // This self-check enables sharing of secondary supertype arrays among 1914 // non-primary types such as array-of-interface. Otherwise, each such 1915 // type would need its own customized SSA. 1916 // We move this check to the front of the fast path because many 1917 // type checks are in fact trivially successful in this manner, 1918 // so we get a nicely predicted branch right at the start of the check. 1919 cmpd(CCR0, sub_klass, super_klass); 1920 beq(CCR0, *L_success); 1921 1922 // Check the supertype display: 1923 if (must_load_sco) { 1924 // The super check offset is always positive... 1925 lwz(check_cache_offset, sco_offset, super_klass); 1926 super_check_offset = RegisterOrConstant(check_cache_offset); 1927 // super_check_offset is register. 1928 assert_different_registers(sub_klass, super_klass, cached_super, super_check_offset.as_register()); 1929 } 1930 // The loaded value is the offset from KlassOopDesc. 1931 1932 ld(cached_super, super_check_offset, sub_klass); 1933 cmpd(CCR0, cached_super, super_klass); 1934 1935 // This check has worked decisively for primary supers. 1936 // Secondary supers are sought in the super_cache ('super_cache_addr'). 1937 // (Secondary supers are interfaces and very deeply nested subtypes.) 1938 // This works in the same check above because of a tricky aliasing 1939 // between the super_cache and the primary super display elements. 1940 // (The 'super_check_addr' can address either, as the case requires.) 1941 // Note that the cache is updated below if it does not help us find 1942 // what we need immediately. 1943 // So if it was a primary super, we can just fail immediately. 1944 // Otherwise, it's the slow path for us (no success at this point). 1945 1946 #define FINAL_JUMP(label) if (&(label) != &L_fallthrough) { b(label); } 1947 1948 if (super_check_offset.is_register()) { 1949 beq(CCR0, *L_success); 1950 cmpwi(CCR0, super_check_offset.as_register(), sc_offset); 1951 if (L_failure == &L_fallthrough) { 1952 beq(CCR0, *L_slow_path); 1953 } else { 1954 bne(CCR0, *L_failure); 1955 FINAL_JUMP(*L_slow_path); 1956 } 1957 } else { 1958 if (super_check_offset.as_constant() == sc_offset) { 1959 // Need a slow path; fast failure is impossible. 1960 if (L_slow_path == &L_fallthrough) { 1961 beq(CCR0, *L_success); 1962 } else { 1963 bne(CCR0, *L_slow_path); 1964 FINAL_JUMP(*L_success); 1965 } 1966 } else { 1967 // No slow path; it's a fast decision. 1968 if (L_failure == &L_fallthrough) { 1969 beq(CCR0, *L_success); 1970 } else { 1971 bne(CCR0, *L_failure); 1972 FINAL_JUMP(*L_success); 1973 } 1974 } 1975 } 1976 1977 bind(L_fallthrough); 1978 #undef FINAL_JUMP 1979 } 1980 1981 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 1982 Register super_klass, 1983 Register temp1_reg, 1984 Register temp2_reg, 1985 Label* L_success, 1986 Register result_reg) { 1987 const Register array_ptr = temp1_reg; // current value from cache array 1988 const Register temp = temp2_reg; 1989 1990 assert_different_registers(sub_klass, super_klass, array_ptr, temp); 1991 1992 int source_offset = in_bytes(Klass::secondary_supers_offset()); 1993 int target_offset = in_bytes(Klass::secondary_super_cache_offset()); 1994 1995 int length_offset = Array<Klass*>::length_offset_in_bytes(); 1996 int base_offset = Array<Klass*>::base_offset_in_bytes(); 1997 1998 Label hit, loop, failure, fallthru; 1999 2000 ld(array_ptr, source_offset, sub_klass); 2001 2002 // TODO: PPC port: assert(4 == arrayOopDesc::length_length_in_bytes(), "precondition violated."); 2003 lwz(temp, length_offset, array_ptr); 2004 cmpwi(CCR0, temp, 0); 2005 beq(CCR0, result_reg!=noreg ? failure : fallthru); // length 0 2006 2007 mtctr(temp); // load ctr 2008 2009 bind(loop); 2010 // Oops in table are NO MORE compressed. 2011 ld(temp, base_offset, array_ptr); 2012 cmpd(CCR0, temp, super_klass); 2013 beq(CCR0, hit); 2014 addi(array_ptr, array_ptr, BytesPerWord); 2015 bdnz(loop); 2016 2017 bind(failure); 2018 if (result_reg!=noreg) li(result_reg, 1); // load non-zero result (indicates a miss) 2019 b(fallthru); 2020 2021 bind(hit); 2022 std(super_klass, target_offset, sub_klass); // save result to cache 2023 if (result_reg != noreg) { li(result_reg, 0); } // load zero result (indicates a hit) 2024 if (L_success != NULL) { b(*L_success); } 2025 else if (result_reg == noreg) { blr(); } // return with CR0.eq if neither label nor result reg provided 2026 2027 bind(fallthru); 2028 } 2029 2030 // Try fast path, then go to slow one if not successful 2031 void MacroAssembler::check_klass_subtype(Register sub_klass, 2032 Register super_klass, 2033 Register temp1_reg, 2034 Register temp2_reg, 2035 Label& L_success) { 2036 Label L_failure; 2037 check_klass_subtype_fast_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success, &L_failure); 2038 check_klass_subtype_slow_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success); 2039 bind(L_failure); // Fallthru if not successful. 2040 } 2041 2042 void MacroAssembler::check_method_handle_type(Register mtype_reg, Register mh_reg, 2043 Register temp_reg, 2044 Label& wrong_method_type) { 2045 assert_different_registers(mtype_reg, mh_reg, temp_reg); 2046 // Compare method type against that of the receiver. 2047 load_heap_oop_not_null(temp_reg, delayed_value(java_lang_invoke_MethodHandle::type_offset_in_bytes, temp_reg), mh_reg); 2048 cmpd(CCR0, temp_reg, mtype_reg); 2049 bne(CCR0, wrong_method_type); 2050 } 2051 2052 RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot, 2053 Register temp_reg, 2054 int extra_slot_offset) { 2055 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 2056 int stackElementSize = Interpreter::stackElementSize; 2057 int offset = extra_slot_offset * stackElementSize; 2058 if (arg_slot.is_constant()) { 2059 offset += arg_slot.as_constant() * stackElementSize; 2060 return offset; 2061 } else { 2062 assert(temp_reg != noreg, "must specify"); 2063 sldi(temp_reg, arg_slot.as_register(), exact_log2(stackElementSize)); 2064 if (offset != 0) 2065 addi(temp_reg, temp_reg, offset); 2066 return temp_reg; 2067 } 2068 } 2069 2070 // Supports temp2_reg = R0. 2071 void MacroAssembler::biased_locking_enter(ConditionRegister cr_reg, Register obj_reg, 2072 Register mark_reg, Register temp_reg, 2073 Register temp2_reg, Label& done, Label* slow_case) { 2074 assert(UseBiasedLocking, "why call this otherwise?"); 2075 2076 #ifdef ASSERT 2077 assert_different_registers(obj_reg, mark_reg, temp_reg, temp2_reg); 2078 #endif 2079 2080 Label cas_label; 2081 2082 // Branch to done if fast path fails and no slow_case provided. 2083 Label *slow_case_int = (slow_case != NULL) ? slow_case : &done; 2084 2085 // Biased locking 2086 // See whether the lock is currently biased toward our thread and 2087 // whether the epoch is still valid 2088 // Note that the runtime guarantees sufficient alignment of JavaThread 2089 // pointers to allow age to be placed into low bits 2090 assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, 2091 "biased locking makes assumptions about bit layout"); 2092 2093 if (PrintBiasedLockingStatistics) { 2094 load_const(temp2_reg, (address) BiasedLocking::total_entry_count_addr(), temp_reg); 2095 lwzx(temp_reg, temp2_reg); 2096 addi(temp_reg, temp_reg, 1); 2097 stwx(temp_reg, temp2_reg); 2098 } 2099 2100 andi(temp_reg, mark_reg, markOopDesc::biased_lock_mask_in_place); 2101 cmpwi(cr_reg, temp_reg, markOopDesc::biased_lock_pattern); 2102 bne(cr_reg, cas_label); 2103 2104 load_klass(temp_reg, obj_reg); 2105 2106 load_const_optimized(temp2_reg, ~((int) markOopDesc::age_mask_in_place)); 2107 ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg); 2108 orr(temp_reg, R16_thread, temp_reg); 2109 xorr(temp_reg, mark_reg, temp_reg); 2110 andr(temp_reg, temp_reg, temp2_reg); 2111 cmpdi(cr_reg, temp_reg, 0); 2112 if (PrintBiasedLockingStatistics) { 2113 Label l; 2114 bne(cr_reg, l); 2115 load_const(temp2_reg, (address) BiasedLocking::biased_lock_entry_count_addr()); 2116 lwzx(mark_reg, temp2_reg); 2117 addi(mark_reg, mark_reg, 1); 2118 stwx(mark_reg, temp2_reg); 2119 // restore mark_reg 2120 ld(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg); 2121 bind(l); 2122 } 2123 beq(cr_reg, done); 2124 2125 Label try_revoke_bias; 2126 Label try_rebias; 2127 2128 // At this point we know that the header has the bias pattern and 2129 // that we are not the bias owner in the current epoch. We need to 2130 // figure out more details about the state of the header in order to 2131 // know what operations can be legally performed on the object's 2132 // header. 2133 2134 // If the low three bits in the xor result aren't clear, that means 2135 // the prototype header is no longer biased and we have to revoke 2136 // the bias on this object. 2137 andi(temp2_reg, temp_reg, markOopDesc::biased_lock_mask_in_place); 2138 cmpwi(cr_reg, temp2_reg, 0); 2139 bne(cr_reg, try_revoke_bias); 2140 2141 // Biasing is still enabled for this data type. See whether the 2142 // epoch of the current bias is still valid, meaning that the epoch 2143 // bits of the mark word are equal to the epoch bits of the 2144 // prototype header. (Note that the prototype header's epoch bits 2145 // only change at a safepoint.) If not, attempt to rebias the object 2146 // toward the current thread. Note that we must be absolutely sure 2147 // that the current epoch is invalid in order to do this because 2148 // otherwise the manipulations it performs on the mark word are 2149 // illegal. 2150 2151 int shift_amount = 64 - markOopDesc::epoch_shift; 2152 // rotate epoch bits to right (little) end and set other bits to 0 2153 // [ big part | epoch | little part ] -> [ 0..0 | epoch ] 2154 rldicl_(temp2_reg, temp_reg, shift_amount, 64 - markOopDesc::epoch_bits); 2155 // branch if epoch bits are != 0, i.e. they differ, because the epoch has been incremented 2156 bne(CCR0, try_rebias); 2157 2158 // The epoch of the current bias is still valid but we know nothing 2159 // about the owner; it might be set or it might be clear. Try to 2160 // acquire the bias of the object using an atomic operation. If this 2161 // fails we will go in to the runtime to revoke the object's bias. 2162 // Note that we first construct the presumed unbiased header so we 2163 // don't accidentally blow away another thread's valid bias. 2164 andi(mark_reg, mark_reg, (markOopDesc::biased_lock_mask_in_place | 2165 markOopDesc::age_mask_in_place | 2166 markOopDesc::epoch_mask_in_place)); 2167 orr(temp_reg, R16_thread, mark_reg); 2168 2169 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); 2170 2171 // CmpxchgX sets cr_reg to cmpX(temp2_reg, mark_reg). 2172 cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg, 2173 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg, 2174 /*where=*/obj_reg, 2175 MacroAssembler::MemBarAcq, 2176 MacroAssembler::cmpxchgx_hint_acquire_lock(), 2177 noreg, slow_case_int); // bail out if failed 2178 2179 // If the biasing toward our thread failed, this means that 2180 // another thread succeeded in biasing it toward itself and we 2181 // need to revoke that bias. The revocation will occur in the 2182 // interpreter runtime in the slow case. 2183 if (PrintBiasedLockingStatistics) { 2184 load_const(temp2_reg, (address) BiasedLocking::anonymously_biased_lock_entry_count_addr(), temp_reg); 2185 lwzx(temp_reg, temp2_reg); 2186 addi(temp_reg, temp_reg, 1); 2187 stwx(temp_reg, temp2_reg); 2188 } 2189 b(done); 2190 2191 bind(try_rebias); 2192 // At this point we know the epoch has expired, meaning that the 2193 // current "bias owner", if any, is actually invalid. Under these 2194 // circumstances _only_, we are allowed to use the current header's 2195 // value as the comparison value when doing the cas to acquire the 2196 // bias in the current epoch. In other words, we allow transfer of 2197 // the bias from one thread to another directly in this situation. 2198 load_klass(temp_reg, obj_reg); 2199 andi(temp2_reg, mark_reg, markOopDesc::age_mask_in_place); 2200 orr(temp2_reg, R16_thread, temp2_reg); 2201 ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg); 2202 orr(temp_reg, temp2_reg, temp_reg); 2203 2204 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); 2205 2206 cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg, 2207 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg, 2208 /*where=*/obj_reg, 2209 MacroAssembler::MemBarAcq, 2210 MacroAssembler::cmpxchgx_hint_acquire_lock(), 2211 noreg, slow_case_int); // bail out if failed 2212 2213 // If the biasing toward our thread failed, this means that 2214 // another thread succeeded in biasing it toward itself and we 2215 // need to revoke that bias. The revocation will occur in the 2216 // interpreter runtime in the slow case. 2217 if (PrintBiasedLockingStatistics) { 2218 load_const(temp2_reg, (address) BiasedLocking::rebiased_lock_entry_count_addr(), temp_reg); 2219 lwzx(temp_reg, temp2_reg); 2220 addi(temp_reg, temp_reg, 1); 2221 stwx(temp_reg, temp2_reg); 2222 } 2223 b(done); 2224 2225 bind(try_revoke_bias); 2226 // The prototype mark in the klass doesn't have the bias bit set any 2227 // more, indicating that objects of this data type are not supposed 2228 // to be biased any more. We are going to try to reset the mark of 2229 // this object to the prototype value and fall through to the 2230 // CAS-based locking scheme. Note that if our CAS fails, it means 2231 // that another thread raced us for the privilege of revoking the 2232 // bias of this particular object, so it's okay to continue in the 2233 // normal locking code. 2234 load_klass(temp_reg, obj_reg); 2235 ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg); 2236 andi(temp2_reg, mark_reg, markOopDesc::age_mask_in_place); 2237 orr(temp_reg, temp_reg, temp2_reg); 2238 2239 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); 2240 2241 // CmpxchgX sets cr_reg to cmpX(temp2_reg, mark_reg). 2242 cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg, 2243 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg, 2244 /*where=*/obj_reg, 2245 MacroAssembler::MemBarAcq, 2246 MacroAssembler::cmpxchgx_hint_acquire_lock()); 2247 2248 // reload markOop in mark_reg before continuing with lightweight locking 2249 ld(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg); 2250 2251 // Fall through to the normal CAS-based lock, because no matter what 2252 // the result of the above CAS, some thread must have succeeded in 2253 // removing the bias bit from the object's header. 2254 if (PrintBiasedLockingStatistics) { 2255 Label l; 2256 bne(cr_reg, l); 2257 load_const(temp2_reg, (address) BiasedLocking::revoked_lock_entry_count_addr(), temp_reg); 2258 lwzx(temp_reg, temp2_reg); 2259 addi(temp_reg, temp_reg, 1); 2260 stwx(temp_reg, temp2_reg); 2261 bind(l); 2262 } 2263 2264 bind(cas_label); 2265 } 2266 2267 void MacroAssembler::biased_locking_exit (ConditionRegister cr_reg, Register mark_addr, Register temp_reg, Label& done) { 2268 // Check for biased locking unlock case, which is a no-op 2269 // Note: we do not have to check the thread ID for two reasons. 2270 // First, the interpreter checks for IllegalMonitorStateException at 2271 // a higher level. Second, if the bias was revoked while we held the 2272 // lock, the object could not be rebiased toward another thread, so 2273 // the bias bit would be clear. 2274 2275 ld(temp_reg, 0, mark_addr); 2276 andi(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place); 2277 2278 cmpwi(cr_reg, temp_reg, markOopDesc::biased_lock_pattern); 2279 beq(cr_reg, done); 2280 } 2281 2282 // allocation (for C1) 2283 void MacroAssembler::eden_allocate( 2284 Register obj, // result: pointer to object after successful allocation 2285 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise 2286 int con_size_in_bytes, // object size in bytes if known at compile time 2287 Register t1, // temp register 2288 Register t2, // temp register 2289 Label& slow_case // continuation point if fast allocation fails 2290 ) { 2291 b(slow_case); 2292 } 2293 2294 void MacroAssembler::tlab_allocate( 2295 Register obj, // result: pointer to object after successful allocation 2296 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise 2297 int con_size_in_bytes, // object size in bytes if known at compile time 2298 Register t1, // temp register 2299 Label& slow_case // continuation point if fast allocation fails 2300 ) { 2301 // make sure arguments make sense 2302 assert_different_registers(obj, var_size_in_bytes, t1); 2303 assert(0 <= con_size_in_bytes && is_simm13(con_size_in_bytes), "illegal object size"); 2304 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment"); 2305 2306 const Register new_top = t1; 2307 //verify_tlab(); not implemented 2308 2309 ld(obj, in_bytes(JavaThread::tlab_top_offset()), R16_thread); 2310 ld(R0, in_bytes(JavaThread::tlab_end_offset()), R16_thread); 2311 if (var_size_in_bytes == noreg) { 2312 addi(new_top, obj, con_size_in_bytes); 2313 } else { 2314 add(new_top, obj, var_size_in_bytes); 2315 } 2316 cmpld(CCR0, new_top, R0); 2317 bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case); 2318 2319 #ifdef ASSERT 2320 // make sure new free pointer is properly aligned 2321 { 2322 Label L; 2323 andi_(R0, new_top, MinObjAlignmentInBytesMask); 2324 beq(CCR0, L); 2325 stop("updated TLAB free is not properly aligned", 0x934); 2326 bind(L); 2327 } 2328 #endif // ASSERT 2329 2330 // update the tlab top pointer 2331 std(new_top, in_bytes(JavaThread::tlab_top_offset()), R16_thread); 2332 //verify_tlab(); not implemented 2333 } 2334 void MacroAssembler::tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case) { 2335 unimplemented("tlab_refill"); 2336 } 2337 void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, Register t1, Register t2) { 2338 unimplemented("incr_allocated_bytes"); 2339 } 2340 2341 address MacroAssembler::emit_trampoline_stub(int destination_toc_offset, 2342 int insts_call_instruction_offset, Register Rtoc) { 2343 // Start the stub. 2344 address stub = start_a_stub(64); 2345 if (stub == NULL) { return NULL; } // CodeCache full: bail out 2346 2347 // Create a trampoline stub relocation which relates this trampoline stub 2348 // with the call instruction at insts_call_instruction_offset in the 2349 // instructions code-section. 2350 relocate(trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset)); 2351 const int stub_start_offset = offset(); 2352 2353 // For java_to_interp stubs we use R11_scratch1 as scratch register 2354 // and in call trampoline stubs we use R12_scratch2. This way we 2355 // can distinguish them (see is_NativeCallTrampolineStub_at()). 2356 Register reg_scratch = R12_scratch2; 2357 2358 // Now, create the trampoline stub's code: 2359 // - load the TOC 2360 // - load the call target from the constant pool 2361 // - call 2362 if (Rtoc == noreg) { 2363 calculate_address_from_global_toc(reg_scratch, method_toc()); 2364 Rtoc = reg_scratch; 2365 } 2366 2367 ld_largeoffset_unchecked(reg_scratch, destination_toc_offset, Rtoc, false); 2368 mtctr(reg_scratch); 2369 bctr(); 2370 2371 const address stub_start_addr = addr_at(stub_start_offset); 2372 2373 // Assert that the encoded destination_toc_offset can be identified and that it is correct. 2374 assert(destination_toc_offset == NativeCallTrampolineStub_at(stub_start_addr)->destination_toc_offset(), 2375 "encoded offset into the constant pool must match"); 2376 // Trampoline_stub_size should be good. 2377 assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size"); 2378 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline"); 2379 2380 // End the stub. 2381 end_a_stub(); 2382 return stub; 2383 } 2384 2385 // TM on PPC64. 2386 void MacroAssembler::atomic_inc_ptr(Register addr, Register result, int simm16) { 2387 Label retry; 2388 bind(retry); 2389 ldarx(result, addr, /*hint*/ false); 2390 addi(result, result, simm16); 2391 stdcx_(result, addr); 2392 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 2393 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0 2394 } else { 2395 bne( CCR0, retry); // stXcx_ sets CCR0 2396 } 2397 } 2398 2399 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) { 2400 Label retry; 2401 bind(retry); 2402 lwarx(result, addr, /*hint*/ false); 2403 ori(result, result, uimm16); 2404 stwcx_(result, addr); 2405 if (UseStaticBranchPredictionInCompareAndSwapPPC64) { 2406 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0 2407 } else { 2408 bne( CCR0, retry); // stXcx_ sets CCR0 2409 } 2410 } 2411 2412 #if INCLUDE_RTM_OPT 2413 2414 // Update rtm_counters based on abort status 2415 // input: abort_status 2416 // rtm_counters (RTMLockingCounters*) 2417 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) { 2418 // Mapping to keep PreciseRTMLockingStatistics similar to x86. 2419 // x86 ppc (! means inverted, ? means not the same) 2420 // 0 31 Set if abort caused by XABORT instruction. 2421 // 1 ! 7 If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set. 2422 // 2 13 Set if another logical processor conflicted with a memory address that was part of the transaction that aborted. 2423 // 3 10 Set if an internal buffer overflowed. 2424 // 4 ?12 Set if a debug breakpoint was hit. 2425 // 5 ?32 Set if an abort occurred during execution of a nested transaction. 2426 const int tm_failure_bit[] = {Assembler::tm_tabort, // Note: Seems like signal handler sets this, too. 2427 Assembler::tm_failure_persistent, // inverted: transient 2428 Assembler::tm_trans_cf, 2429 Assembler::tm_footprint_of, 2430 Assembler::tm_non_trans_cf, 2431 Assembler::tm_suspended}; 2432 const bool tm_failure_inv[] = {false, true, false, false, false, false}; 2433 assert(sizeof(tm_failure_bit)/sizeof(int) == RTMLockingCounters::ABORT_STATUS_LIMIT, "adapt mapping!"); 2434 2435 const Register addr_Reg = R0; 2436 // Keep track of offset to where rtm_counters_Reg had pointed to. 2437 int counters_offs = RTMLockingCounters::abort_count_offset(); 2438 addi(addr_Reg, rtm_counters_Reg, counters_offs); 2439 const Register temp_Reg = rtm_counters_Reg; 2440 2441 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically 2442 ldx(temp_Reg, addr_Reg); 2443 addi(temp_Reg, temp_Reg, 1); 2444 stdx(temp_Reg, addr_Reg); 2445 2446 if (PrintPreciseRTMLockingStatistics) { 2447 int counters_offs_delta = RTMLockingCounters::abortX_count_offset() - counters_offs; 2448 2449 //mftexasr(abort_status); done by caller 2450 for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) { 2451 counters_offs += counters_offs_delta; 2452 li(temp_Reg, counters_offs_delta); // can't use addi with R0 2453 add(addr_Reg, addr_Reg, temp_Reg); // point to next counter 2454 counters_offs_delta = sizeof(uintx); 2455 2456 Label check_abort; 2457 rldicr_(temp_Reg, abort_status, tm_failure_bit[i], 0); 2458 if (tm_failure_inv[i]) { 2459 bne(CCR0, check_abort); 2460 } else { 2461 beq(CCR0, check_abort); 2462 } 2463 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically 2464 ldx(temp_Reg, addr_Reg); 2465 addi(temp_Reg, temp_Reg, 1); 2466 stdx(temp_Reg, addr_Reg); 2467 bind(check_abort); 2468 } 2469 } 2470 li(temp_Reg, -counters_offs); // can't use addi with R0 2471 add(rtm_counters_Reg, addr_Reg, temp_Reg); // restore 2472 } 2473 2474 // Branch if (random & (count-1) != 0), count is 2^n 2475 // tmp and CR0 are killed 2476 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) { 2477 mftb(tmp); 2478 andi_(tmp, tmp, count-1); 2479 bne(CCR0, brLabel); 2480 } 2481 2482 // Perform abort ratio calculation, set no_rtm bit if high ratio. 2483 // input: rtm_counters_Reg (RTMLockingCounters* address) - KILLED 2484 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg, 2485 RTMLockingCounters* rtm_counters, 2486 Metadata* method_data) { 2487 Label L_done, L_check_always_rtm1, L_check_always_rtm2; 2488 2489 if (RTMLockingCalculationDelay > 0) { 2490 // Delay calculation. 2491 ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr()); 2492 cmpdi(CCR0, rtm_counters_Reg, 0); 2493 beq(CCR0, L_done); 2494 load_const_optimized(rtm_counters_Reg, (address)rtm_counters, R0); // reload 2495 } 2496 // Abort ratio calculation only if abort_count > RTMAbortThreshold. 2497 // Aborted transactions = abort_count * 100 2498 // All transactions = total_count * RTMTotalCountIncrRate 2499 // Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio) 2500 ld(R0, RTMLockingCounters::abort_count_offset(), rtm_counters_Reg); 2501 cmpdi(CCR0, R0, RTMAbortThreshold); 2502 blt(CCR0, L_check_always_rtm2); 2503 mulli(R0, R0, 100); 2504 2505 const Register tmpReg = rtm_counters_Reg; 2506 ld(tmpReg, RTMLockingCounters::total_count_offset(), rtm_counters_Reg); 2507 mulli(tmpReg, tmpReg, RTMTotalCountIncrRate); 2508 mulli(tmpReg, tmpReg, RTMAbortRatio); 2509 cmpd(CCR0, R0, tmpReg); 2510 blt(CCR0, L_check_always_rtm1); // jump to reload 2511 if (method_data != NULL) { 2512 // Set rtm_state to "no rtm" in MDO. 2513 // Not using a metadata relocation. Method and Class Loader are kept alive anyway. 2514 // (See nmethod::metadata_do and CodeBuffer::finalize_oop_references.) 2515 load_const(R0, (address)method_data + MethodData::rtm_state_offset_in_bytes(), tmpReg); 2516 atomic_ori_int(R0, tmpReg, NoRTM); 2517 } 2518 b(L_done); 2519 2520 bind(L_check_always_rtm1); 2521 load_const_optimized(rtm_counters_Reg, (address)rtm_counters, R0); // reload 2522 bind(L_check_always_rtm2); 2523 ld(tmpReg, RTMLockingCounters::total_count_offset(), rtm_counters_Reg); 2524 cmpdi(CCR0, tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate); 2525 blt(CCR0, L_done); 2526 if (method_data != NULL) { 2527 // Set rtm_state to "always rtm" in MDO. 2528 // Not using a metadata relocation. See above. 2529 load_const(R0, (address)method_data + MethodData::rtm_state_offset_in_bytes(), tmpReg); 2530 atomic_ori_int(R0, tmpReg, UseRTM); 2531 } 2532 bind(L_done); 2533 } 2534 2535 // Update counters and perform abort ratio calculation. 2536 // input: abort_status_Reg 2537 void MacroAssembler::rtm_profiling(Register abort_status_Reg, Register temp_Reg, 2538 RTMLockingCounters* rtm_counters, 2539 Metadata* method_data, 2540 bool profile_rtm) { 2541 2542 assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); 2543 // Update rtm counters based on state at abort. 2544 // Reads abort_status_Reg, updates flags. 2545 assert_different_registers(abort_status_Reg, temp_Reg); 2546 load_const_optimized(temp_Reg, (address)rtm_counters, R0); 2547 rtm_counters_update(abort_status_Reg, temp_Reg); 2548 if (profile_rtm) { 2549 assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); 2550 rtm_abort_ratio_calculation(temp_Reg, rtm_counters, method_data); 2551 } 2552 } 2553 2554 // Retry on abort if abort's status indicates non-persistent failure. 2555 // inputs: retry_count_Reg 2556 // : abort_status_Reg 2557 // output: retry_count_Reg decremented by 1 2558 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, 2559 Label& retryLabel, Label* checkRetry) { 2560 Label doneRetry; 2561 rldicr_(R0, abort_status_Reg, tm_failure_persistent, 0); 2562 bne(CCR0, doneRetry); 2563 if (checkRetry) { bind(*checkRetry); } 2564 addic_(retry_count_Reg, retry_count_Reg, -1); 2565 blt(CCR0, doneRetry); 2566 smt_yield(); // Can't use wait(). No permission (SIGILL). 2567 b(retryLabel); 2568 bind(doneRetry); 2569 } 2570 2571 // Spin and retry if lock is busy. 2572 // inputs: box_Reg (monitor address) 2573 // : retry_count_Reg 2574 // output: retry_count_Reg decremented by 1 2575 // CTR is killed 2576 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register owner_addr_Reg, Label& retryLabel) { 2577 Label SpinLoop, doneRetry; 2578 addic_(retry_count_Reg, retry_count_Reg, -1); 2579 blt(CCR0, doneRetry); 2580 li(R0, RTMSpinLoopCount); 2581 mtctr(R0); 2582 2583 bind(SpinLoop); 2584 smt_yield(); // Can't use waitrsv(). No permission (SIGILL). 2585 bdz(retryLabel); 2586 ld(R0, 0, owner_addr_Reg); 2587 cmpdi(CCR0, R0, 0); 2588 bne(CCR0, SpinLoop); 2589 b(retryLabel); 2590 2591 bind(doneRetry); 2592 } 2593 2594 // Use RTM for normal stack locks. 2595 // Input: objReg (object to lock) 2596 void MacroAssembler::rtm_stack_locking(ConditionRegister flag, 2597 Register obj, Register mark_word, Register tmp, 2598 Register retry_on_abort_count_Reg, 2599 RTMLockingCounters* stack_rtm_counters, 2600 Metadata* method_data, bool profile_rtm, 2601 Label& DONE_LABEL, Label& IsInflated) { 2602 assert(UseRTMForStackLocks, "why call this otherwise?"); 2603 assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); 2604 Label L_rtm_retry, L_decrement_retry, L_on_abort; 2605 2606 if (RTMRetryCount > 0) { 2607 load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort 2608 bind(L_rtm_retry); 2609 } 2610 andi_(R0, mark_word, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased 2611 bne(CCR0, IsInflated); 2612 2613 if (PrintPreciseRTMLockingStatistics || profile_rtm) { 2614 Label L_noincrement; 2615 if (RTMTotalCountIncrRate > 1) { 2616 branch_on_random_using_tb(tmp, (int)RTMTotalCountIncrRate, L_noincrement); 2617 } 2618 assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM"); 2619 load_const_optimized(tmp, (address)stack_rtm_counters->total_count_addr(), R0); 2620 //atomic_inc_ptr(tmp, /*temp, will be reloaded*/mark_word); We don't increment atomically 2621 ldx(mark_word, tmp); 2622 addi(mark_word, mark_word, 1); 2623 stdx(mark_word, tmp); 2624 bind(L_noincrement); 2625 } 2626 tbegin_(); 2627 beq(CCR0, L_on_abort); 2628 ld(mark_word, oopDesc::mark_offset_in_bytes(), obj); // Reload in transaction, conflicts need to be tracked. 2629 andi(R0, mark_word, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits 2630 cmpwi(flag, R0, markOopDesc::unlocked_value); // bits = 001 unlocked 2631 beq(flag, DONE_LABEL); // all done if unlocked 2632 2633 if (UseRTMXendForLockBusy) { 2634 tend_(); 2635 b(L_decrement_retry); 2636 } else { 2637 tabort_(); 2638 } 2639 bind(L_on_abort); 2640 const Register abort_status_Reg = tmp; 2641 mftexasr(abort_status_Reg); 2642 if (PrintPreciseRTMLockingStatistics || profile_rtm) { 2643 rtm_profiling(abort_status_Reg, /*temp*/mark_word, stack_rtm_counters, method_data, profile_rtm); 2644 } 2645 ld(mark_word, oopDesc::mark_offset_in_bytes(), obj); // reload 2646 if (RTMRetryCount > 0) { 2647 // Retry on lock abort if abort status is not permanent. 2648 rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry, &L_decrement_retry); 2649 } else { 2650 bind(L_decrement_retry); 2651 } 2652 } 2653 2654 // Use RTM for inflating locks 2655 // inputs: obj (object to lock) 2656 // mark_word (current header - KILLED) 2657 // boxReg (on-stack box address (displaced header location) - KILLED) 2658 void MacroAssembler::rtm_inflated_locking(ConditionRegister flag, 2659 Register obj, Register mark_word, Register boxReg, 2660 Register retry_on_busy_count_Reg, Register retry_on_abort_count_Reg, 2661 RTMLockingCounters* rtm_counters, 2662 Metadata* method_data, bool profile_rtm, 2663 Label& DONE_LABEL) { 2664 assert(UseRTMLocking, "why call this otherwise?"); 2665 Label L_rtm_retry, L_decrement_retry, L_on_abort; 2666 // Clean monitor_value bit to get valid pointer. 2667 int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value; 2668 2669 // Store non-null, using boxReg instead of (intptr_t)markOopDesc::unused_mark(). 2670 std(boxReg, BasicLock::displaced_header_offset_in_bytes(), boxReg); 2671 const Register tmpReg = boxReg; 2672 const Register owner_addr_Reg = mark_word; 2673 addi(owner_addr_Reg, mark_word, owner_offset); 2674 2675 if (RTMRetryCount > 0) { 2676 load_const_optimized(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy. 2677 load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort. 2678 bind(L_rtm_retry); 2679 } 2680 if (PrintPreciseRTMLockingStatistics || profile_rtm) { 2681 Label L_noincrement; 2682 if (RTMTotalCountIncrRate > 1) { 2683 branch_on_random_using_tb(R0, (int)RTMTotalCountIncrRate, L_noincrement); 2684 } 2685 assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); 2686 load_const(R0, (address)rtm_counters->total_count_addr(), tmpReg); 2687 //atomic_inc_ptr(R0, tmpReg); We don't increment atomically 2688 ldx(tmpReg, R0); 2689 addi(tmpReg, tmpReg, 1); 2690 stdx(tmpReg, R0); 2691 bind(L_noincrement); 2692 } 2693 tbegin_(); 2694 beq(CCR0, L_on_abort); 2695 // We don't reload mark word. Will only be reset at safepoint. 2696 ld(R0, 0, owner_addr_Reg); // Load in transaction, conflicts need to be tracked. 2697 cmpdi(flag, R0, 0); 2698 beq(flag, DONE_LABEL); 2699 2700 if (UseRTMXendForLockBusy) { 2701 tend_(); 2702 b(L_decrement_retry); 2703 } else { 2704 tabort_(); 2705 } 2706 bind(L_on_abort); 2707 const Register abort_status_Reg = tmpReg; 2708 mftexasr(abort_status_Reg); 2709 if (PrintPreciseRTMLockingStatistics || profile_rtm) { 2710 rtm_profiling(abort_status_Reg, /*temp*/ owner_addr_Reg, rtm_counters, method_data, profile_rtm); 2711 // Restore owner_addr_Reg 2712 ld(mark_word, oopDesc::mark_offset_in_bytes(), obj); 2713 #ifdef ASSERT 2714 andi_(R0, mark_word, markOopDesc::monitor_value); 2715 asm_assert_ne("must be inflated", 0xa754); // Deflating only allowed at safepoint. 2716 #endif 2717 addi(owner_addr_Reg, mark_word, owner_offset); 2718 } 2719 if (RTMRetryCount > 0) { 2720 // Retry on lock abort if abort status is not permanent. 2721 rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); 2722 } 2723 2724 // Appears unlocked - try to swing _owner from null to non-null. 2725 cmpxchgd(flag, /*current val*/ R0, (intptr_t)0, /*new val*/ R16_thread, owner_addr_Reg, 2726 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq, 2727 MacroAssembler::cmpxchgx_hint_acquire_lock(), noreg, &L_decrement_retry, true); 2728 2729 if (RTMRetryCount > 0) { 2730 // success done else retry 2731 b(DONE_LABEL); 2732 bind(L_decrement_retry); 2733 // Spin and retry if lock is busy. 2734 rtm_retry_lock_on_busy(retry_on_busy_count_Reg, owner_addr_Reg, L_rtm_retry); 2735 } else { 2736 bind(L_decrement_retry); 2737 } 2738 } 2739 2740 #endif // INCLUDE_RTM_OPT 2741 2742 // "The box" is the space on the stack where we copy the object mark. 2743 void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box, 2744 Register temp, Register displaced_header, Register current_header, 2745 bool try_bias, 2746 RTMLockingCounters* rtm_counters, 2747 RTMLockingCounters* stack_rtm_counters, 2748 Metadata* method_data, 2749 bool use_rtm, bool profile_rtm) { 2750 assert_different_registers(oop, box, temp, displaced_header, current_header); 2751 assert(flag != CCR0, "bad condition register"); 2752 Label cont; 2753 Label object_has_monitor; 2754 Label cas_failed; 2755 2756 // Load markOop from object into displaced_header. 2757 ld(displaced_header, oopDesc::mark_offset_in_bytes(), oop); 2758 2759 2760 // Always do locking in runtime. 2761 if (EmitSync & 0x01) { 2762 cmpdi(flag, oop, 0); // Oop can't be 0 here => always false. 2763 return; 2764 } 2765 2766 if (try_bias) { 2767 biased_locking_enter(flag, oop, displaced_header, temp, current_header, cont); 2768 } 2769 2770 #if INCLUDE_RTM_OPT 2771 if (UseRTMForStackLocks && use_rtm) { 2772 rtm_stack_locking(flag, oop, displaced_header, temp, /*temp*/ current_header, 2773 stack_rtm_counters, method_data, profile_rtm, 2774 cont, object_has_monitor); 2775 } 2776 #endif // INCLUDE_RTM_OPT 2777 2778 // Handle existing monitor. 2779 if ((EmitSync & 0x02) == 0) { 2780 // The object has an existing monitor iff (mark & monitor_value) != 0. 2781 andi_(temp, displaced_header, markOopDesc::monitor_value); 2782 bne(CCR0, object_has_monitor); 2783 } 2784 2785 // Set displaced_header to be (markOop of object | UNLOCK_VALUE). 2786 ori(displaced_header, displaced_header, markOopDesc::unlocked_value); 2787 2788 // Load Compare Value application register. 2789 2790 // Initialize the box. (Must happen before we update the object mark!) 2791 std(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box); 2792 2793 // Must fence, otherwise, preceding store(s) may float below cmpxchg. 2794 // Compare object markOop with mark and if equal exchange scratch1 with object markOop. 2795 cmpxchgd(/*flag=*/flag, 2796 /*current_value=*/current_header, 2797 /*compare_value=*/displaced_header, 2798 /*exchange_value=*/box, 2799 /*where=*/oop, 2800 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq, 2801 MacroAssembler::cmpxchgx_hint_acquire_lock(), 2802 noreg, 2803 &cas_failed, 2804 /*check without membar and ldarx first*/true); 2805 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); 2806 2807 // If the compare-and-exchange succeeded, then we found an unlocked 2808 // object and we have now locked it. 2809 b(cont); 2810 2811 bind(cas_failed); 2812 // We did not see an unlocked object so try the fast recursive case. 2813 2814 // Check if the owner is self by comparing the value in the markOop of object 2815 // (current_header) with the stack pointer. 2816 sub(current_header, current_header, R1_SP); 2817 load_const_optimized(temp, ~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place); 2818 2819 and_(R0/*==0?*/, current_header, temp); 2820 // If condition is true we are cont and hence we can store 0 as the 2821 // displaced header in the box, which indicates that it is a recursive lock. 2822 mcrf(flag,CCR0); 2823 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box); 2824 2825 // Handle existing monitor. 2826 if ((EmitSync & 0x02) == 0) { 2827 b(cont); 2828 2829 bind(object_has_monitor); 2830 // The object's monitor m is unlocked iff m->owner == NULL, 2831 // otherwise m->owner may contain a thread or a stack address. 2832 2833 #if INCLUDE_RTM_OPT 2834 // Use the same RTM locking code in 32- and 64-bit VM. 2835 if (use_rtm) { 2836 rtm_inflated_locking(flag, oop, displaced_header, box, temp, /*temp*/ current_header, 2837 rtm_counters, method_data, profile_rtm, cont); 2838 } else { 2839 #endif // INCLUDE_RTM_OPT 2840 2841 // Try to CAS m->owner from NULL to current thread. 2842 addi(temp, displaced_header, ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value); 2843 cmpxchgd(/*flag=*/flag, 2844 /*current_value=*/current_header, 2845 /*compare_value=*/(intptr_t)0, 2846 /*exchange_value=*/R16_thread, 2847 /*where=*/temp, 2848 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq, 2849 MacroAssembler::cmpxchgx_hint_acquire_lock()); 2850 2851 // Store a non-null value into the box. 2852 std(box, BasicLock::displaced_header_offset_in_bytes(), box); 2853 2854 # ifdef ASSERT 2855 bne(flag, cont); 2856 // We have acquired the monitor, check some invariants. 2857 addi(/*monitor=*/temp, temp, -ObjectMonitor::owner_offset_in_bytes()); 2858 // Invariant 1: _recursions should be 0. 2859 //assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size"); 2860 asm_assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), temp, 2861 "monitor->_recursions should be 0", -1); 2862 // Invariant 2: OwnerIsThread shouldn't be 0. 2863 //assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size"); 2864 //asm_assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), temp, 2865 // "monitor->OwnerIsThread shouldn't be 0", -1); 2866 # endif 2867 2868 #if INCLUDE_RTM_OPT 2869 } // use_rtm() 2870 #endif 2871 } 2872 2873 bind(cont); 2874 // flag == EQ indicates success 2875 // flag == NE indicates failure 2876 } 2877 2878 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box, 2879 Register temp, Register displaced_header, Register current_header, 2880 bool try_bias, bool use_rtm) { 2881 assert_different_registers(oop, box, temp, displaced_header, current_header); 2882 assert(flag != CCR0, "bad condition register"); 2883 Label cont; 2884 Label object_has_monitor; 2885 2886 // Always do locking in runtime. 2887 if (EmitSync & 0x01) { 2888 cmpdi(flag, oop, 0); // Oop can't be 0 here => always false. 2889 return; 2890 } 2891 2892 if (try_bias) { 2893 biased_locking_exit(flag, oop, current_header, cont); 2894 } 2895 2896 #if INCLUDE_RTM_OPT 2897 if (UseRTMForStackLocks && use_rtm) { 2898 assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); 2899 Label L_regular_unlock; 2900 ld(current_header, oopDesc::mark_offset_in_bytes(), oop); // fetch markword 2901 andi(R0, current_header, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits 2902 cmpwi(flag, R0, markOopDesc::unlocked_value); // bits = 001 unlocked 2903 bne(flag, L_regular_unlock); // else RegularLock 2904 tend_(); // otherwise end... 2905 b(cont); // ... and we're done 2906 bind(L_regular_unlock); 2907 } 2908 #endif 2909 2910 // Find the lock address and load the displaced header from the stack. 2911 ld(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box); 2912 2913 // If the displaced header is 0, we have a recursive unlock. 2914 cmpdi(flag, displaced_header, 0); 2915 beq(flag, cont); 2916 2917 // Handle existing monitor. 2918 if ((EmitSync & 0x02) == 0) { 2919 // The object has an existing monitor iff (mark & monitor_value) != 0. 2920 RTM_OPT_ONLY( if (!(UseRTMForStackLocks && use_rtm)) ) // skip load if already done 2921 ld(current_header, oopDesc::mark_offset_in_bytes(), oop); 2922 andi_(R0, current_header, markOopDesc::monitor_value); 2923 bne(CCR0, object_has_monitor); 2924 } 2925 2926 // Check if it is still a light weight lock, this is is true if we see 2927 // the stack address of the basicLock in the markOop of the object. 2928 // Cmpxchg sets flag to cmpd(current_header, box). 2929 cmpxchgd(/*flag=*/flag, 2930 /*current_value=*/current_header, 2931 /*compare_value=*/box, 2932 /*exchange_value=*/displaced_header, 2933 /*where=*/oop, 2934 MacroAssembler::MemBarRel, 2935 MacroAssembler::cmpxchgx_hint_release_lock(), 2936 noreg, 2937 &cont); 2938 2939 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0"); 2940 2941 // Handle existing monitor. 2942 if ((EmitSync & 0x02) == 0) { 2943 b(cont); 2944 2945 bind(object_has_monitor); 2946 addi(current_header, current_header, -markOopDesc::monitor_value); // monitor 2947 ld(temp, ObjectMonitor::owner_offset_in_bytes(), current_header); 2948 2949 // It's inflated. 2950 #if INCLUDE_RTM_OPT 2951 if (use_rtm) { 2952 Label L_regular_inflated_unlock; 2953 // Clean monitor_value bit to get valid pointer 2954 cmpdi(flag, temp, 0); 2955 bne(flag, L_regular_inflated_unlock); 2956 tend_(); 2957 b(cont); 2958 bind(L_regular_inflated_unlock); 2959 } 2960 #endif 2961 2962 ld(displaced_header, ObjectMonitor::recursions_offset_in_bytes(), current_header); 2963 xorr(temp, R16_thread, temp); // Will be 0 if we are the owner. 2964 orr(temp, temp, displaced_header); // Will be 0 if there are 0 recursions. 2965 cmpdi(flag, temp, 0); 2966 bne(flag, cont); 2967 2968 ld(temp, ObjectMonitor::EntryList_offset_in_bytes(), current_header); 2969 ld(displaced_header, ObjectMonitor::cxq_offset_in_bytes(), current_header); 2970 orr(temp, temp, displaced_header); // Will be 0 if both are 0. 2971 cmpdi(flag, temp, 0); 2972 bne(flag, cont); 2973 release(); 2974 std(temp, ObjectMonitor::owner_offset_in_bytes(), current_header); 2975 } 2976 2977 bind(cont); 2978 // flag == EQ indicates success 2979 // flag == NE indicates failure 2980 } 2981 2982 // Write serialization page so VM thread can do a pseudo remote membar. 2983 // We use the current thread pointer to calculate a thread specific 2984 // offset to write to within the page. This minimizes bus traffic 2985 // due to cache line collision. 2986 void MacroAssembler::serialize_memory(Register thread, Register tmp1, Register tmp2) { 2987 srdi(tmp2, thread, os::get_serialize_page_shift_count()); 2988 2989 int mask = os::vm_page_size() - sizeof(int); 2990 if (Assembler::is_simm(mask, 16)) { 2991 andi(tmp2, tmp2, mask); 2992 } else { 2993 lis(tmp1, (int)((signed short) (mask >> 16))); 2994 ori(tmp1, tmp1, mask & 0x0000ffff); 2995 andr(tmp2, tmp2, tmp1); 2996 } 2997 2998 load_const(tmp1, (long) os::get_memory_serialize_page()); 2999 release(); 3000 stwx(R0, tmp1, tmp2); 3001 } 3002 3003 3004 // GC barrier helper macros 3005 3006 // Write the card table byte if needed. 3007 void MacroAssembler::card_write_barrier_post(Register Rstore_addr, Register Rnew_val, Register Rtmp) { 3008 CardTableModRefBS* bs = 3009 barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set()); 3010 assert(bs->kind() == BarrierSet::CardTableForRS || 3011 bs->kind() == BarrierSet::CardTableExtension, "wrong barrier"); 3012 #ifdef ASSERT 3013 cmpdi(CCR0, Rnew_val, 0); 3014 asm_assert_ne("null oop not allowed", 0x321); 3015 #endif 3016 card_table_write(bs->byte_map_base, Rtmp, Rstore_addr); 3017 } 3018 3019 // Write the card table byte. 3020 void MacroAssembler::card_table_write(jbyte* byte_map_base, Register Rtmp, Register Robj) { 3021 assert_different_registers(Robj, Rtmp, R0); 3022 load_const_optimized(Rtmp, (address)byte_map_base, R0); 3023 srdi(Robj, Robj, CardTableModRefBS::card_shift); 3024 li(R0, 0); // dirty 3025 if (UseConcMarkSweepGC) membar(Assembler::StoreStore); 3026 stbx(R0, Rtmp, Robj); 3027 } 3028 3029 #if INCLUDE_ALL_GCS 3030 // General G1 pre-barrier generator. 3031 // Goal: record the previous value if it is not null. 3032 void MacroAssembler::g1_write_barrier_pre(Register Robj, RegisterOrConstant offset, Register Rpre_val, 3033 Register Rtmp1, Register Rtmp2, bool needs_frame) { 3034 Label runtime, filtered; 3035 3036 // Is marking active? 3037 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 3038 lwz(Rtmp1, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active()), R16_thread); 3039 } else { 3040 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 3041 lbz(Rtmp1, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active()), R16_thread); 3042 } 3043 cmpdi(CCR0, Rtmp1, 0); 3044 beq(CCR0, filtered); 3045 3046 // Do we need to load the previous value? 3047 if (Robj != noreg) { 3048 // Load the previous value... 3049 if (UseCompressedOops) { 3050 lwz(Rpre_val, offset, Robj); 3051 } else { 3052 ld(Rpre_val, offset, Robj); 3053 } 3054 // Previous value has been loaded into Rpre_val. 3055 } 3056 assert(Rpre_val != noreg, "must have a real register"); 3057 3058 // Is the previous value null? 3059 cmpdi(CCR0, Rpre_val, 0); 3060 beq(CCR0, filtered); 3061 3062 if (Robj != noreg && UseCompressedOops) { 3063 decode_heap_oop_not_null(Rpre_val); 3064 } 3065 3066 // OK, it's not filtered, so we'll need to call enqueue. In the normal 3067 // case, pre_val will be a scratch G-reg, but there are some cases in 3068 // which it's an O-reg. In the first case, do a normal call. In the 3069 // latter, do a save here and call the frameless version. 3070 3071 // Can we store original value in the thread's buffer? 3072 // Is index == 0? 3073 // (The index field is typed as size_t.) 3074 const Register Rbuffer = Rtmp1, Rindex = Rtmp2; 3075 3076 ld(Rindex, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index()), R16_thread); 3077 cmpdi(CCR0, Rindex, 0); 3078 beq(CCR0, runtime); // If index == 0, goto runtime. 3079 ld(Rbuffer, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf()), R16_thread); 3080 3081 addi(Rindex, Rindex, -wordSize); // Decrement index. 3082 std(Rindex, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index()), R16_thread); 3083 3084 // Record the previous value. 3085 stdx(Rpre_val, Rbuffer, Rindex); 3086 b(filtered); 3087 3088 bind(runtime); 3089 3090 // VM call need frame to access(write) O register. 3091 if (needs_frame) { 3092 save_LR_CR(Rtmp1); 3093 push_frame_reg_args(0, Rtmp2); 3094 } 3095 3096 if (Rpre_val->is_volatile() && Robj == noreg) mr(R31, Rpre_val); // Save pre_val across C call if it was preloaded. 3097 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), Rpre_val, R16_thread); 3098 if (Rpre_val->is_volatile() && Robj == noreg) mr(Rpre_val, R31); // restore 3099 3100 if (needs_frame) { 3101 pop_frame(); 3102 restore_LR_CR(Rtmp1); 3103 } 3104 3105 bind(filtered); 3106 } 3107 3108 // General G1 post-barrier generator 3109 // Store cross-region card. 3110 void MacroAssembler::g1_write_barrier_post(Register Rstore_addr, Register Rnew_val, Register Rtmp1, Register Rtmp2, Register Rtmp3, Label *filtered_ext) { 3111 Label runtime, filtered_int; 3112 Label& filtered = (filtered_ext != NULL) ? *filtered_ext : filtered_int; 3113 assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2); 3114 3115 G1SATBCardTableLoggingModRefBS* bs = 3116 barrier_set_cast<G1SATBCardTableLoggingModRefBS>(Universe::heap()->barrier_set()); 3117 3118 // Does store cross heap regions? 3119 if (G1RSBarrierRegionFilter) { 3120 xorr(Rtmp1, Rstore_addr, Rnew_val); 3121 srdi_(Rtmp1, Rtmp1, HeapRegion::LogOfHRGrainBytes); 3122 beq(CCR0, filtered); 3123 } 3124 3125 // Crosses regions, storing NULL? 3126 #ifdef ASSERT 3127 cmpdi(CCR0, Rnew_val, 0); 3128 asm_assert_ne("null oop not allowed (G1)", 0x322); // Checked by caller on PPC64, so following branch is obsolete: 3129 //beq(CCR0, filtered); 3130 #endif 3131 3132 // Storing region crossing non-NULL, is card already dirty? 3133 assert(sizeof(*bs->byte_map_base) == sizeof(jbyte), "adjust this code"); 3134 const Register Rcard_addr = Rtmp1; 3135 Register Rbase = Rtmp2; 3136 load_const_optimized(Rbase, (address)bs->byte_map_base, /*temp*/ Rtmp3); 3137 3138 srdi(Rcard_addr, Rstore_addr, CardTableModRefBS::card_shift); 3139 3140 // Get the address of the card. 3141 lbzx(/*card value*/ Rtmp3, Rbase, Rcard_addr); 3142 cmpwi(CCR0, Rtmp3, (int)G1SATBCardTableModRefBS::g1_young_card_val()); 3143 beq(CCR0, filtered); 3144 3145 membar(Assembler::StoreLoad); 3146 lbzx(/*card value*/ Rtmp3, Rbase, Rcard_addr); // Reload after membar. 3147 cmpwi(CCR0, Rtmp3 /* card value */, CardTableModRefBS::dirty_card_val()); 3148 beq(CCR0, filtered); 3149 3150 // Storing a region crossing, non-NULL oop, card is clean. 3151 // Dirty card and log. 3152 li(Rtmp3, CardTableModRefBS::dirty_card_val()); 3153 //release(); // G1: oops are allowed to get visible after dirty marking. 3154 stbx(Rtmp3, Rbase, Rcard_addr); 3155 3156 add(Rcard_addr, Rbase, Rcard_addr); // This is the address which needs to get enqueued. 3157 Rbase = noreg; // end of lifetime 3158 3159 const Register Rqueue_index = Rtmp2, 3160 Rqueue_buf = Rtmp3; 3161 ld(Rqueue_index, in_bytes(JavaThread::dirty_card_queue_offset() + DirtyCardQueue::byte_offset_of_index()), R16_thread); 3162 cmpdi(CCR0, Rqueue_index, 0); 3163 beq(CCR0, runtime); // index == 0 then jump to runtime 3164 ld(Rqueue_buf, in_bytes(JavaThread::dirty_card_queue_offset() + DirtyCardQueue::byte_offset_of_buf()), R16_thread); 3165 3166 addi(Rqueue_index, Rqueue_index, -wordSize); // decrement index 3167 std(Rqueue_index, in_bytes(JavaThread::dirty_card_queue_offset() + DirtyCardQueue::byte_offset_of_index()), R16_thread); 3168 3169 stdx(Rcard_addr, Rqueue_buf, Rqueue_index); // store card 3170 b(filtered); 3171 3172 bind(runtime); 3173 3174 // Save the live input values. 3175 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), Rcard_addr, R16_thread); 3176 3177 bind(filtered_int); 3178 } 3179 #endif // INCLUDE_ALL_GCS 3180 3181 // Values for last_Java_pc, and last_Java_sp must comply to the rules 3182 // in frame_ppc.hpp. 3183 void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Java_pc) { 3184 // Always set last_Java_pc and flags first because once last_Java_sp 3185 // is visible has_last_Java_frame is true and users will look at the 3186 // rest of the fields. (Note: flags should always be zero before we 3187 // get here so doesn't need to be set.) 3188 3189 // Verify that last_Java_pc was zeroed on return to Java 3190 asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_pc_offset()), R16_thread, 3191 "last_Java_pc not zeroed before leaving Java", 0x200); 3192 3193 // When returning from calling out from Java mode the frame anchor's 3194 // last_Java_pc will always be set to NULL. It is set here so that 3195 // if we are doing a call to native (not VM) that we capture the 3196 // known pc and don't have to rely on the native call having a 3197 // standard frame linkage where we can find the pc. 3198 if (last_Java_pc != noreg) 3199 std(last_Java_pc, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread); 3200 3201 // Set last_Java_sp last. 3202 std(last_Java_sp, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread); 3203 } 3204 3205 void MacroAssembler::reset_last_Java_frame(void) { 3206 asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()), 3207 R16_thread, "SP was not set, still zero", 0x202); 3208 3209 BLOCK_COMMENT("reset_last_Java_frame {"); 3210 li(R0, 0); 3211 3212 // _last_Java_sp = 0 3213 std(R0, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread); 3214 3215 // _last_Java_pc = 0 3216 std(R0, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread); 3217 BLOCK_COMMENT("} reset_last_Java_frame"); 3218 } 3219 3220 void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) { 3221 assert_different_registers(sp, tmp1); 3222 3223 // sp points to a TOP_IJAVA_FRAME, retrieve frame's PC via 3224 // TOP_IJAVA_FRAME_ABI. 3225 // FIXME: assert that we really have a TOP_IJAVA_FRAME here! 3226 address entry = pc(); 3227 load_const_optimized(tmp1, entry); 3228 3229 set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1); 3230 } 3231 3232 void MacroAssembler::get_vm_result(Register oop_result) { 3233 // Read: 3234 // R16_thread 3235 // R16_thread->in_bytes(JavaThread::vm_result_offset()) 3236 // 3237 // Updated: 3238 // oop_result 3239 // R16_thread->in_bytes(JavaThread::vm_result_offset()) 3240 3241 verify_thread(); 3242 3243 ld(oop_result, in_bytes(JavaThread::vm_result_offset()), R16_thread); 3244 li(R0, 0); 3245 std(R0, in_bytes(JavaThread::vm_result_offset()), R16_thread); 3246 3247 verify_oop(oop_result); 3248 } 3249 3250 void MacroAssembler::get_vm_result_2(Register metadata_result) { 3251 // Read: 3252 // R16_thread 3253 // R16_thread->in_bytes(JavaThread::vm_result_2_offset()) 3254 // 3255 // Updated: 3256 // metadata_result 3257 // R16_thread->in_bytes(JavaThread::vm_result_2_offset()) 3258 3259 ld(metadata_result, in_bytes(JavaThread::vm_result_2_offset()), R16_thread); 3260 li(R0, 0); 3261 std(R0, in_bytes(JavaThread::vm_result_2_offset()), R16_thread); 3262 } 3263 3264 Register MacroAssembler::encode_klass_not_null(Register dst, Register src) { 3265 Register current = (src != noreg) ? src : dst; // Klass is in dst if no src provided. 3266 if (Universe::narrow_klass_base() != 0) { 3267 // Use dst as temp if it is free. 3268 sub_const_optimized(dst, current, Universe::narrow_klass_base(), R0); 3269 current = dst; 3270 } 3271 if (Universe::narrow_klass_shift() != 0) { 3272 srdi(dst, current, Universe::narrow_klass_shift()); 3273 current = dst; 3274 } 3275 return current; 3276 } 3277 3278 void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) { 3279 if (UseCompressedClassPointers) { 3280 Register compressedKlass = encode_klass_not_null(ck, klass); 3281 stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop); 3282 } else { 3283 std(klass, oopDesc::klass_offset_in_bytes(), dst_oop); 3284 } 3285 } 3286 3287 void MacroAssembler::store_klass_gap(Register dst_oop, Register val) { 3288 if (UseCompressedClassPointers) { 3289 if (val == noreg) { 3290 val = R0; 3291 li(val, 0); 3292 } 3293 stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed 3294 } 3295 } 3296 3297 int MacroAssembler::instr_size_for_decode_klass_not_null() { 3298 if (!UseCompressedClassPointers) return 0; 3299 int num_instrs = 1; // shift or move 3300 if (Universe::narrow_klass_base() != 0) num_instrs = 7; // shift + load const + add 3301 return num_instrs * BytesPerInstWord; 3302 } 3303 3304 void MacroAssembler::decode_klass_not_null(Register dst, Register src) { 3305 assert(dst != R0, "Dst reg may not be R0, as R0 is used here."); 3306 if (src == noreg) src = dst; 3307 Register shifted_src = src; 3308 if (Universe::narrow_klass_shift() != 0 || 3309 Universe::narrow_klass_base() == 0 && src != dst) { // Move required. 3310 shifted_src = dst; 3311 sldi(shifted_src, src, Universe::narrow_klass_shift()); 3312 } 3313 if (Universe::narrow_klass_base() != 0) { 3314 add_const_optimized(dst, shifted_src, Universe::narrow_klass_base(), R0); 3315 } 3316 } 3317 3318 void MacroAssembler::load_klass(Register dst, Register src) { 3319 if (UseCompressedClassPointers) { 3320 lwz(dst, oopDesc::klass_offset_in_bytes(), src); 3321 // Attention: no null check here! 3322 decode_klass_not_null(dst, dst); 3323 } else { 3324 ld(dst, oopDesc::klass_offset_in_bytes(), src); 3325 } 3326 } 3327 3328 void MacroAssembler::load_mirror_from_const_method(Register mirror, Register const_method) { 3329 ld(mirror, in_bytes(ConstMethod::constants_offset()), const_method); 3330 ld(mirror, ConstantPool::pool_holder_offset_in_bytes(), mirror); 3331 ld(mirror, in_bytes(Klass::java_mirror_offset()), mirror); 3332 } 3333 3334 // Clear Array 3335 // For very short arrays. tmp == R0 is allowed. 3336 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) { 3337 if (cnt_dwords > 0) { li(tmp, 0); } 3338 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); } 3339 } 3340 3341 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed. 3342 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) { 3343 if (cnt_dwords < 8) { 3344 clear_memory_unrolled(base_ptr, cnt_dwords, tmp); 3345 return; 3346 } 3347 3348 Label loop; 3349 const long loopcnt = cnt_dwords >> 1, 3350 remainder = cnt_dwords & 1; 3351 3352 li(tmp, loopcnt); 3353 mtctr(tmp); 3354 li(tmp, 0); 3355 bind(loop); 3356 std(tmp, 0, base_ptr); 3357 std(tmp, 8, base_ptr); 3358 addi(base_ptr, base_ptr, 16); 3359 bdnz(loop); 3360 if (remainder) { std(tmp, 0, base_ptr); } 3361 } 3362 3363 // Kills both input registers. tmp == R0 is allowed. 3364 void MacroAssembler::clear_memory_doubleword(Register base_ptr, Register cnt_dwords, Register tmp, long const_cnt) { 3365 // Procedure for large arrays (uses data cache block zero instruction). 3366 Label startloop, fast, fastloop, small_rest, restloop, done; 3367 const int cl_size = VM_Version::L1_data_cache_line_size(), 3368 cl_dwords = cl_size >> 3, 3369 cl_dw_addr_bits = exact_log2(cl_dwords), 3370 dcbz_min = 1, // Min count of dcbz executions, needs to be >0. 3371 min_cnt = ((dcbz_min + 1) << cl_dw_addr_bits) - 1; 3372 3373 if (const_cnt >= 0) { 3374 // Constant case. 3375 if (const_cnt < min_cnt) { 3376 clear_memory_constlen(base_ptr, const_cnt, tmp); 3377 return; 3378 } 3379 load_const_optimized(cnt_dwords, const_cnt, tmp); 3380 } else { 3381 // cnt_dwords already loaded in register. Need to check size. 3382 cmpdi(CCR1, cnt_dwords, min_cnt); // Big enough? (ensure >= dcbz_min lines included). 3383 blt(CCR1, small_rest); 3384 } 3385 rldicl_(tmp, base_ptr, 64-3, 64-cl_dw_addr_bits); // Extract dword offset within first cache line. 3386 beq(CCR0, fast); // Already 128byte aligned. 3387 3388 subfic(tmp, tmp, cl_dwords); 3389 mtctr(tmp); // Set ctr to hit 128byte boundary (0<ctr<cl_dwords). 3390 subf(cnt_dwords, tmp, cnt_dwords); // rest. 3391 li(tmp, 0); 3392 3393 bind(startloop); // Clear at the beginning to reach 128byte boundary. 3394 std(tmp, 0, base_ptr); // Clear 8byte aligned block. 3395 addi(base_ptr, base_ptr, 8); 3396 bdnz(startloop); 3397 3398 bind(fast); // Clear 128byte blocks. 3399 srdi(tmp, cnt_dwords, cl_dw_addr_bits); // Loop count for 128byte loop (>0). 3400 andi(cnt_dwords, cnt_dwords, cl_dwords-1); // Rest in dwords. 3401 mtctr(tmp); // Load counter. 3402 3403 bind(fastloop); 3404 dcbz(base_ptr); // Clear 128byte aligned block. 3405 addi(base_ptr, base_ptr, cl_size); 3406 bdnz(fastloop); 3407 3408 bind(small_rest); 3409 cmpdi(CCR0, cnt_dwords, 0); // size 0? 3410 beq(CCR0, done); // rest == 0 3411 li(tmp, 0); 3412 mtctr(cnt_dwords); // Load counter. 3413 3414 bind(restloop); // Clear rest. 3415 std(tmp, 0, base_ptr); // Clear 8byte aligned block. 3416 addi(base_ptr, base_ptr, 8); 3417 bdnz(restloop); 3418 3419 bind(done); 3420 } 3421 3422 /////////////////////////////////////////// String intrinsics //////////////////////////////////////////// 3423 3424 #ifdef COMPILER2 3425 // Intrinsics for CompactStrings 3426 3427 // Compress char[] to byte[] by compressing 16 bytes at once. 3428 void MacroAssembler::string_compress_16(Register src, Register dst, Register cnt, 3429 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, 3430 Label& Lfailure) { 3431 3432 const Register tmp0 = R0; 3433 assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); 3434 Label Lloop, Lslow; 3435 3436 // Check if cnt >= 8 (= 16 bytes) 3437 lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF00FF00FF 3438 srwi_(tmp2, cnt, 3); 3439 beq(CCR0, Lslow); 3440 ori(tmp1, tmp1, 0xFF); 3441 rldimi(tmp1, tmp1, 32, 0); 3442 mtctr(tmp2); 3443 3444 // 2x unrolled loop 3445 bind(Lloop); 3446 ld(tmp2, 0, src); // _0_1_2_3 (Big Endian) 3447 ld(tmp4, 8, src); // _4_5_6_7 3448 3449 orr(tmp0, tmp2, tmp4); 3450 rldicl(tmp3, tmp2, 6*8, 64-24); // _____1_2 3451 rldimi(tmp2, tmp2, 2*8, 2*8); // _0_2_3_3 3452 rldicl(tmp5, tmp4, 6*8, 64-24); // _____5_6 3453 rldimi(tmp4, tmp4, 2*8, 2*8); // _4_6_7_7 3454 3455 andc_(tmp0, tmp0, tmp1); 3456 bne(CCR0, Lfailure); // Not latin1. 3457 addi(src, src, 16); 3458 3459 rlwimi(tmp3, tmp2, 0*8, 24, 31);// _____1_3 3460 srdi(tmp2, tmp2, 3*8); // ____0_2_ 3461 rlwimi(tmp5, tmp4, 0*8, 24, 31);// _____5_7 3462 srdi(tmp4, tmp4, 3*8); // ____4_6_ 3463 3464 orr(tmp2, tmp2, tmp3); // ____0123 3465 orr(tmp4, tmp4, tmp5); // ____4567 3466 3467 stw(tmp2, 0, dst); 3468 stw(tmp4, 4, dst); 3469 addi(dst, dst, 8); 3470 bdnz(Lloop); 3471 3472 bind(Lslow); // Fallback to slow version 3473 } 3474 3475 // Compress char[] to byte[]. cnt must be positive int. 3476 void MacroAssembler::string_compress(Register src, Register dst, Register cnt, Register tmp, Label& Lfailure) { 3477 Label Lloop; 3478 mtctr(cnt); 3479 3480 bind(Lloop); 3481 lhz(tmp, 0, src); 3482 cmplwi(CCR0, tmp, 0xff); 3483 bgt(CCR0, Lfailure); // Not latin1. 3484 addi(src, src, 2); 3485 stb(tmp, 0, dst); 3486 addi(dst, dst, 1); 3487 bdnz(Lloop); 3488 } 3489 3490 // Inflate byte[] to char[] by inflating 16 bytes at once. 3491 void MacroAssembler::string_inflate_16(Register src, Register dst, Register cnt, 3492 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) { 3493 const Register tmp0 = R0; 3494 assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5); 3495 Label Lloop, Lslow; 3496 3497 // Check if cnt >= 8 3498 srwi_(tmp2, cnt, 3); 3499 beq(CCR0, Lslow); 3500 lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF 3501 ori(tmp1, tmp1, 0xFF); 3502 mtctr(tmp2); 3503 3504 // 2x unrolled loop 3505 bind(Lloop); 3506 lwz(tmp2, 0, src); // ____0123 (Big Endian) 3507 lwz(tmp4, 4, src); // ____4567 3508 addi(src, src, 8); 3509 3510 rldicl(tmp3, tmp2, 7*8, 64-8); // _______2 3511 rlwimi(tmp2, tmp2, 3*8, 16, 23);// ____0113 3512 rldicl(tmp5, tmp4, 7*8, 64-8); // _______6 3513 rlwimi(tmp4, tmp4, 3*8, 16, 23);// ____4557 3514 3515 andc(tmp0, tmp2, tmp1); // ____0_1_ 3516 rlwimi(tmp2, tmp3, 2*8, 0, 23); // _____2_3 3517 andc(tmp3, tmp4, tmp1); // ____4_5_ 3518 rlwimi(tmp4, tmp5, 2*8, 0, 23); // _____6_7 3519 3520 rldimi(tmp2, tmp0, 3*8, 0*8); // _0_1_2_3 3521 rldimi(tmp4, tmp3, 3*8, 0*8); // _4_5_6_7 3522 3523 std(tmp2, 0, dst); 3524 std(tmp4, 8, dst); 3525 addi(dst, dst, 16); 3526 bdnz(Lloop); 3527 3528 bind(Lslow); // Fallback to slow version 3529 } 3530 3531 // Inflate byte[] to char[]. cnt must be positive int. 3532 void MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) { 3533 Label Lloop; 3534 mtctr(cnt); 3535 3536 bind(Lloop); 3537 lbz(tmp, 0, src); 3538 addi(src, src, 1); 3539 sth(tmp, 0, dst); 3540 addi(dst, dst, 2); 3541 bdnz(Lloop); 3542 } 3543 3544 void MacroAssembler::string_compare(Register str1, Register str2, 3545 Register cnt1, Register cnt2, 3546 Register tmp1, Register result, int ae) { 3547 const Register tmp0 = R0, 3548 diff = tmp1; 3549 3550 assert_different_registers(str1, str2, cnt1, cnt2, tmp0, tmp1, result); 3551 Label Ldone, Lslow, Lloop, Lreturn_diff; 3552 3553 // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a) 3554 // we interchange str1 and str2 in the UL case and negate the result. 3555 // Like this, str1 is always latin1 encoded, except for the UU case. 3556 // In addition, we need 0 (or sign which is 0) extend. 3557 3558 if (ae == StrIntrinsicNode::UU) { 3559 srwi(cnt1, cnt1, 1); 3560 } else { 3561 clrldi(cnt1, cnt1, 32); 3562 } 3563 3564 if (ae != StrIntrinsicNode::LL) { 3565 srwi(cnt2, cnt2, 1); 3566 } else { 3567 clrldi(cnt2, cnt2, 32); 3568 } 3569 3570 // See if the lengths are different, and calculate min in cnt1. 3571 // Save diff in case we need it for a tie-breaker. 3572 subf_(diff, cnt2, cnt1); // diff = cnt1 - cnt2 3573 // if (diff > 0) { cnt1 = cnt2; } 3574 if (VM_Version::has_isel()) { 3575 isel(cnt1, CCR0, Assembler::greater, /*invert*/ false, cnt2); 3576 } else { 3577 Label Lskip; 3578 blt(CCR0, Lskip); 3579 mr(cnt1, cnt2); 3580 bind(Lskip); 3581 } 3582 3583 // Rename registers 3584 Register chr1 = result; 3585 Register chr2 = tmp0; 3586 3587 // Compare multiple characters in fast loop (only implemented for same encoding). 3588 int stride1 = 8, stride2 = 8; 3589 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) { 3590 int log2_chars_per_iter = (ae == StrIntrinsicNode::LL) ? 3 : 2; 3591 Label Lfastloop, Lskipfast; 3592 3593 srwi_(tmp0, cnt1, log2_chars_per_iter); 3594 beq(CCR0, Lskipfast); 3595 rldicl(cnt2, cnt1, 0, 64 - log2_chars_per_iter); // Remaining characters. 3596 li(cnt1, 1 << log2_chars_per_iter); // Initialize for failure case: Rescan characters from current iteration. 3597 mtctr(tmp0); 3598 3599 bind(Lfastloop); 3600 ld(chr1, 0, str1); 3601 ld(chr2, 0, str2); 3602 cmpd(CCR0, chr1, chr2); 3603 bne(CCR0, Lslow); 3604 addi(str1, str1, stride1); 3605 addi(str2, str2, stride2); 3606 bdnz(Lfastloop); 3607 mr(cnt1, cnt2); // Remaining characters. 3608 bind(Lskipfast); 3609 } 3610 3611 // Loop which searches the first difference character by character. 3612 cmpwi(CCR0, cnt1, 0); 3613 beq(CCR0, Lreturn_diff); 3614 bind(Lslow); 3615 mtctr(cnt1); 3616 3617 switch (ae) { 3618 case StrIntrinsicNode::LL: stride1 = 1; stride2 = 1; break; 3619 case StrIntrinsicNode::UL: // fallthru (see comment above) 3620 case StrIntrinsicNode::LU: stride1 = 1; stride2 = 2; break; 3621 case StrIntrinsicNode::UU: stride1 = 2; stride2 = 2; break; 3622 default: ShouldNotReachHere(); break; 3623 } 3624 3625 bind(Lloop); 3626 if (stride1 == 1) { lbz(chr1, 0, str1); } else { lhz(chr1, 0, str1); } 3627 if (stride2 == 1) { lbz(chr2, 0, str2); } else { lhz(chr2, 0, str2); } 3628 subf_(result, chr2, chr1); // result = chr1 - chr2 3629 bne(CCR0, Ldone); 3630 addi(str1, str1, stride1); 3631 addi(str2, str2, stride2); 3632 bdnz(Lloop); 3633 3634 // If strings are equal up to min length, return the length difference. 3635 bind(Lreturn_diff); 3636 mr(result, diff); 3637 3638 // Otherwise, return the difference between the first mismatched chars. 3639 bind(Ldone); 3640 if (ae == StrIntrinsicNode::UL) { 3641 neg(result, result); // Negate result (see note above). 3642 } 3643 } 3644 3645 void MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2, 3646 Register limit, Register tmp1, Register result, bool is_byte) { 3647 const Register tmp0 = R0; 3648 assert_different_registers(ary1, ary2, limit, tmp0, tmp1, result); 3649 Label Ldone, Lskiploop, Lloop, Lfastloop, Lskipfast; 3650 bool limit_needs_shift = false; 3651 3652 if (is_array_equ) { 3653 const int length_offset = arrayOopDesc::length_offset_in_bytes(); 3654 const int base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR); 3655 3656 // Return true if the same array. 3657 cmpd(CCR0, ary1, ary2); 3658 beq(CCR0, Lskiploop); 3659 3660 // Return false if one of them is NULL. 3661 cmpdi(CCR0, ary1, 0); 3662 cmpdi(CCR1, ary2, 0); 3663 li(result, 0); 3664 cror(CCR0, Assembler::equal, CCR1, Assembler::equal); 3665 beq(CCR0, Ldone); 3666 3667 // Load the lengths of arrays. 3668 lwz(limit, length_offset, ary1); 3669 lwz(tmp0, length_offset, ary2); 3670 3671 // Return false if the two arrays are not equal length. 3672 cmpw(CCR0, limit, tmp0); 3673 bne(CCR0, Ldone); 3674 3675 // Load array addresses. 3676 addi(ary1, ary1, base_offset); 3677 addi(ary2, ary2, base_offset); 3678 } else { 3679 limit_needs_shift = !is_byte; 3680 li(result, 0); // Assume not equal. 3681 } 3682 3683 // Rename registers 3684 Register chr1 = tmp0; 3685 Register chr2 = tmp1; 3686 3687 // Compare 8 bytes per iteration in fast loop. 3688 const int log2_chars_per_iter = is_byte ? 3 : 2; 3689 3690 srwi_(tmp0, limit, log2_chars_per_iter + (limit_needs_shift ? 1 : 0)); 3691 beq(CCR0, Lskipfast); 3692 mtctr(tmp0); 3693 3694 bind(Lfastloop); 3695 ld(chr1, 0, ary1); 3696 ld(chr2, 0, ary2); 3697 addi(ary1, ary1, 8); 3698 addi(ary2, ary2, 8); 3699 cmpd(CCR0, chr1, chr2); 3700 bne(CCR0, Ldone); 3701 bdnz(Lfastloop); 3702 3703 bind(Lskipfast); 3704 rldicl_(limit, limit, limit_needs_shift ? 64 - 1 : 0, 64 - log2_chars_per_iter); // Remaining characters. 3705 beq(CCR0, Lskiploop); 3706 mtctr(limit); 3707 3708 // Character by character. 3709 bind(Lloop); 3710 if (is_byte) { 3711 lbz(chr1, 0, ary1); 3712 lbz(chr2, 0, ary2); 3713 addi(ary1, ary1, 1); 3714 addi(ary2, ary2, 1); 3715 } else { 3716 lhz(chr1, 0, ary1); 3717 lhz(chr2, 0, ary2); 3718 addi(ary1, ary1, 2); 3719 addi(ary2, ary2, 2); 3720 } 3721 cmpw(CCR0, chr1, chr2); 3722 bne(CCR0, Ldone); 3723 bdnz(Lloop); 3724 3725 bind(Lskiploop); 3726 li(result, 1); // All characters are equal. 3727 bind(Ldone); 3728 } 3729 3730 void MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt, 3731 Register needle, ciTypeArray* needle_values, Register needlecnt, int needlecntval, 3732 Register tmp1, Register tmp2, Register tmp3, Register tmp4, int ae) { 3733 3734 // Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite! 3735 Label L_TooShort, L_Found, L_NotFound, L_End; 3736 Register last_addr = haycnt, // Kill haycnt at the beginning. 3737 addr = tmp1, 3738 n_start = tmp2, 3739 ch1 = tmp3, 3740 ch2 = R0; 3741 3742 assert(ae != StrIntrinsicNode::LU, "Invalid encoding"); 3743 const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2; 3744 const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1; 3745 3746 // ************************************************************************************************** 3747 // Prepare for main loop: optimized for needle count >=2, bail out otherwise. 3748 // ************************************************************************************************** 3749 3750 // Compute last haystack addr to use if no match gets found. 3751 clrldi(haycnt, haycnt, 32); // Ensure positive int is valid as 64 bit value. 3752 addi(addr, haystack, -h_csize); // Accesses use pre-increment. 3753 if (needlecntval == 0) { // variable needlecnt 3754 cmpwi(CCR6, needlecnt, 2); 3755 clrldi(needlecnt, needlecnt, 32); // Ensure positive int is valid as 64 bit value. 3756 blt(CCR6, L_TooShort); // Variable needlecnt: handle short needle separately. 3757 } 3758 3759 if (n_csize == 2) { lwz(n_start, 0, needle); } else { lhz(n_start, 0, needle); } // Load first 2 characters of needle. 3760 3761 if (needlecntval == 0) { // variable needlecnt 3762 subf(ch1, needlecnt, haycnt); // Last character index to compare is haycnt-needlecnt. 3763 addi(needlecnt, needlecnt, -2); // Rest of needle. 3764 } else { // constant needlecnt 3765 guarantee(needlecntval != 1, "IndexOf with single-character needle must be handled separately"); 3766 assert((needlecntval & 0x7fff) == needlecntval, "wrong immediate"); 3767 addi(ch1, haycnt, -needlecntval); // Last character index to compare is haycnt-needlecnt. 3768 if (needlecntval > 3) { li(needlecnt, needlecntval - 2); } // Rest of needle. 3769 } 3770 3771 if (h_csize == 2) { slwi(ch1, ch1, 1); } // Scale to number of bytes. 3772 3773 if (ae ==StrIntrinsicNode::UL) { 3774 srwi(tmp4, n_start, 1*8); // ___0 3775 rlwimi(n_start, tmp4, 2*8, 0, 23); // _0_1 3776 } 3777 3778 add(last_addr, haystack, ch1); // Point to last address to compare (haystack+2*(haycnt-needlecnt)). 3779 3780 // Main Loop (now we have at least 2 characters). 3781 Label L_OuterLoop, L_InnerLoop, L_FinalCheck, L_Comp1, L_Comp2; 3782 bind(L_OuterLoop); // Search for 1st 2 characters. 3783 Register addr_diff = tmp4; 3784 subf(addr_diff, addr, last_addr); // Difference between already checked address and last address to check. 3785 addi(addr, addr, h_csize); // This is the new address we want to use for comparing. 3786 srdi_(ch2, addr_diff, h_csize); 3787 beq(CCR0, L_FinalCheck); // 2 characters left? 3788 mtctr(ch2); // num of characters / 2 3789 bind(L_InnerLoop); // Main work horse (2x unrolled search loop) 3790 if (h_csize == 2) { // Load 2 characters of haystack (ignore alignment). 3791 lwz(ch1, 0, addr); 3792 lwz(ch2, 2, addr); 3793 } else { 3794 lhz(ch1, 0, addr); 3795 lhz(ch2, 1, addr); 3796 } 3797 cmpw(CCR0, ch1, n_start); // Compare 2 characters (1 would be sufficient but try to reduce branches to CompLoop). 3798 cmpw(CCR1, ch2, n_start); 3799 beq(CCR0, L_Comp1); // Did we find the needle start? 3800 beq(CCR1, L_Comp2); 3801 addi(addr, addr, 2 * h_csize); 3802 bdnz(L_InnerLoop); 3803 bind(L_FinalCheck); 3804 andi_(addr_diff, addr_diff, h_csize); // Remaining characters not covered by InnerLoop: (num of characters) & 1. 3805 beq(CCR0, L_NotFound); 3806 if (h_csize == 2) { lwz(ch1, 0, addr); } else { lhz(ch1, 0, addr); } // One position left at which we have to compare. 3807 cmpw(CCR1, ch1, n_start); 3808 beq(CCR1, L_Comp1); 3809 bind(L_NotFound); 3810 li(result, -1); // not found 3811 b(L_End); 3812 3813 // ************************************************************************************************** 3814 // Special Case: unfortunately, the variable needle case can be called with needlecnt<2 3815 // ************************************************************************************************** 3816 if (needlecntval == 0) { // We have to handle these cases separately. 3817 Label L_OneCharLoop; 3818 bind(L_TooShort); 3819 mtctr(haycnt); 3820 if (n_csize == 2) { lhz(n_start, 0, needle); } else { lbz(n_start, 0, needle); } // First character of needle 3821 bind(L_OneCharLoop); 3822 if (h_csize == 2) { lhzu(ch1, 2, addr); } else { lbzu(ch1, 1, addr); } 3823 cmpw(CCR1, ch1, n_start); 3824 beq(CCR1, L_Found); // Did we find the one character needle? 3825 bdnz(L_OneCharLoop); 3826 li(result, -1); // Not found. 3827 b(L_End); 3828 } 3829 3830 // ************************************************************************************************** 3831 // Regular Case Part II: compare rest of needle (first 2 characters have been compared already) 3832 // ************************************************************************************************** 3833 3834 // Compare the rest 3835 bind(L_Comp2); 3836 addi(addr, addr, h_csize); // First comparison has failed, 2nd one hit. 3837 bind(L_Comp1); // Addr points to possible needle start. 3838 if (needlecntval != 2) { // Const needlecnt==2? 3839 if (needlecntval != 3) { 3840 if (needlecntval == 0) { beq(CCR6, L_Found); } // Variable needlecnt==2? 3841 Register n_ind = tmp4, 3842 h_ind = n_ind; 3843 li(n_ind, 2 * n_csize); // First 2 characters are already compared, use index 2. 3844 mtctr(needlecnt); // Decremented by 2, still > 0. 3845 Label L_CompLoop; 3846 bind(L_CompLoop); 3847 if (ae ==StrIntrinsicNode::UL) { 3848 h_ind = ch1; 3849 sldi(h_ind, n_ind, 1); 3850 } 3851 if (n_csize == 2) { lhzx(ch2, needle, n_ind); } else { lbzx(ch2, needle, n_ind); } 3852 if (h_csize == 2) { lhzx(ch1, addr, h_ind); } else { lbzx(ch1, addr, h_ind); } 3853 cmpw(CCR1, ch1, ch2); 3854 bne(CCR1, L_OuterLoop); 3855 addi(n_ind, n_ind, n_csize); 3856 bdnz(L_CompLoop); 3857 } else { // No loop required if there's only one needle character left. 3858 if (n_csize == 2) { lhz(ch2, 2 * 2, needle); } else { lbz(ch2, 2 * 1, needle); } 3859 if (h_csize == 2) { lhz(ch1, 2 * 2, addr); } else { lbz(ch1, 2 * 1, addr); } 3860 cmpw(CCR1, ch1, ch2); 3861 bne(CCR1, L_OuterLoop); 3862 } 3863 } 3864 // Return index ... 3865 bind(L_Found); 3866 subf(result, haystack, addr); // relative to haystack, ... 3867 if (h_csize == 2) { srdi(result, result, 1); } // in characters. 3868 bind(L_End); 3869 } // string_indexof 3870 3871 void MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt, 3872 Register needle, jchar needleChar, Register tmp1, Register tmp2, bool is_byte) { 3873 assert_different_registers(haystack, haycnt, needle, tmp1, tmp2); 3874 3875 Label L_InnerLoop, L_FinalCheck, L_Found1, L_Found2, L_NotFound, L_End; 3876 Register addr = tmp1, 3877 ch1 = tmp2, 3878 ch2 = R0; 3879 3880 const int h_csize = is_byte ? 1 : 2; 3881 3882 //4: 3883 srwi_(tmp2, haycnt, 1); // Shift right by exact_log2(UNROLL_FACTOR). 3884 mr(addr, haystack); 3885 beq(CCR0, L_FinalCheck); 3886 mtctr(tmp2); // Move to count register. 3887 //8: 3888 bind(L_InnerLoop); // Main work horse (2x unrolled search loop). 3889 if (!is_byte) { 3890 lhz(ch1, 0, addr); 3891 lhz(ch2, 2, addr); 3892 } else { 3893 lbz(ch1, 0, addr); 3894 lbz(ch2, 1, addr); 3895 } 3896 (needle != R0) ? cmpw(CCR0, ch1, needle) : cmplwi(CCR0, ch1, (unsigned int)needleChar); 3897 (needle != R0) ? cmpw(CCR1, ch2, needle) : cmplwi(CCR1, ch2, (unsigned int)needleChar); 3898 beq(CCR0, L_Found1); // Did we find the needle? 3899 beq(CCR1, L_Found2); 3900 addi(addr, addr, 2 * h_csize); 3901 bdnz(L_InnerLoop); 3902 //16: 3903 bind(L_FinalCheck); 3904 andi_(R0, haycnt, 1); 3905 beq(CCR0, L_NotFound); 3906 if (!is_byte) { lhz(ch1, 0, addr); } else { lbz(ch1, 0, addr); } // One position left at which we have to compare. 3907 (needle != R0) ? cmpw(CCR1, ch1, needle) : cmplwi(CCR1, ch1, (unsigned int)needleChar); 3908 beq(CCR1, L_Found1); 3909 //21: 3910 bind(L_NotFound); 3911 li(result, -1); // Not found. 3912 b(L_End); 3913 3914 bind(L_Found2); 3915 addi(addr, addr, h_csize); 3916 //24: 3917 bind(L_Found1); // Return index ... 3918 subf(result, haystack, addr); // relative to haystack, ... 3919 if (!is_byte) { srdi(result, result, 1); } // in characters. 3920 bind(L_End); 3921 } // string_indexof_char 3922 3923 3924 void MacroAssembler::has_negatives(Register src, Register cnt, Register result, 3925 Register tmp1, Register tmp2) { 3926 const Register tmp0 = R0; 3927 assert_different_registers(src, result, cnt, tmp0, tmp1, tmp2); 3928 Label Lfastloop, Lslow, Lloop, Lnoneg, Ldone; 3929 3930 // Check if cnt >= 8 (= 16 bytes) 3931 lis(tmp1, (int)(short)0x8080); // tmp1 = 0x8080808080808080 3932 srwi_(tmp2, cnt, 4); 3933 li(result, 1); // Assume there's a negative byte. 3934 beq(CCR0, Lslow); 3935 ori(tmp1, tmp1, 0x8080); 3936 rldimi(tmp1, tmp1, 32, 0); 3937 mtctr(tmp2); 3938 3939 // 2x unrolled loop 3940 bind(Lfastloop); 3941 ld(tmp2, 0, src); 3942 ld(tmp0, 8, src); 3943 3944 orr(tmp0, tmp2, tmp0); 3945 3946 and_(tmp0, tmp0, tmp1); 3947 bne(CCR0, Ldone); // Found negative byte. 3948 addi(src, src, 16); 3949 3950 bdnz(Lfastloop); 3951 3952 bind(Lslow); // Fallback to slow version 3953 rldicl_(tmp0, cnt, 0, 64-4); 3954 beq(CCR0, Lnoneg); 3955 mtctr(tmp0); 3956 bind(Lloop); 3957 lbz(tmp0, 0, src); 3958 addi(src, src, 1); 3959 andi_(tmp0, tmp0, 0x80); 3960 bne(CCR0, Ldone); // Found negative byte. 3961 bdnz(Lloop); 3962 bind(Lnoneg); 3963 li(result, 0); 3964 3965 bind(Ldone); 3966 } 3967 3968 #endif // Compiler2 3969 3970 // Helpers for Intrinsic Emitters 3971 // 3972 // Revert the byte order of a 32bit value in a register 3973 // src: 0x44556677 3974 // dst: 0x77665544 3975 // Three steps to obtain the result: 3976 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word 3977 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared. 3978 // This value initializes dst. 3979 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost 3980 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go. 3981 // This value is mask inserted into dst with a [0..23] mask of 1s. 3982 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position. 3983 // This value is mask inserted into dst with a [8..15] mask of 1s. 3984 void MacroAssembler::load_reverse_32(Register dst, Register src) { 3985 assert_different_registers(dst, src); 3986 3987 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left. 3988 rlwimi(dst, src, 3*8, 0, 23); // Insert byte 5 into position 6, 7 into 4, leave pos 7 alone. 3989 rlwimi(dst, src, 1*8, 8, 15); // Insert byte 6 into position 5, leave the rest alone. 3990 } 3991 3992 // Calculate the column addresses of the crc32 lookup table into distinct registers. 3993 // This loop-invariant calculation is moved out of the loop body, reducing the loop 3994 // body size from 20 to 16 instructions. 3995 // Returns the offset that was used to calculate the address of column tc3. 3996 // Due to register shortage, setting tc3 may overwrite table. With the return offset 3997 // at hand, the original table address can be easily reconstructed. 3998 int MacroAssembler::crc32_table_columns(Register table, Register tc0, Register tc1, Register tc2, Register tc3) { 3999 4000 #ifdef VM_LITTLE_ENDIAN 4001 // This is what we implement (the DOLIT4 part): 4002 // ========================================================================= */ 4003 // #define DOLIT4 c ^= *buf4++; \ 4004 // c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ 4005 // crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24] 4006 // #define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4 4007 // ========================================================================= */ 4008 const int ix0 = 3*(4*CRC32_COLUMN_SIZE); 4009 const int ix1 = 2*(4*CRC32_COLUMN_SIZE); 4010 const int ix2 = 1*(4*CRC32_COLUMN_SIZE); 4011 const int ix3 = 0*(4*CRC32_COLUMN_SIZE); 4012 #else 4013 // This is what we implement (the DOBIG4 part): 4014 // ========================================================================= 4015 // #define DOBIG4 c ^= *++buf4; \ 4016 // c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ 4017 // crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] 4018 // #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 4019 // ========================================================================= 4020 const int ix0 = 4*(4*CRC32_COLUMN_SIZE); 4021 const int ix1 = 5*(4*CRC32_COLUMN_SIZE); 4022 const int ix2 = 6*(4*CRC32_COLUMN_SIZE); 4023 const int ix3 = 7*(4*CRC32_COLUMN_SIZE); 4024 #endif 4025 assert_different_registers(table, tc0, tc1, tc2); 4026 assert(table == tc3, "must be!"); 4027 4028 addi(tc0, table, ix0); 4029 addi(tc1, table, ix1); 4030 addi(tc2, table, ix2); 4031 if (ix3 != 0) addi(tc3, table, ix3); 4032 4033 return ix3; 4034 } 4035 4036 /** 4037 * uint32_t crc; 4038 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 4039 */ 4040 void MacroAssembler::fold_byte_crc32(Register crc, Register val, Register table, Register tmp) { 4041 assert_different_registers(crc, table, tmp); 4042 assert_different_registers(val, table); 4043 4044 if (crc == val) { // Must rotate first to use the unmodified value. 4045 rlwinm(tmp, val, 2, 24-2, 31-2); // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest. 4046 // As we use a word (4-byte) instruction, we have to adapt the mask bit positions. 4047 srwi(crc, crc, 8); // Unsigned shift, clear leftmost 8 bits. 4048 } else { 4049 srwi(crc, crc, 8); // Unsigned shift, clear leftmost 8 bits. 4050 rlwinm(tmp, val, 2, 24-2, 31-2); // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest. 4051 } 4052 lwzx(tmp, table, tmp); 4053 xorr(crc, crc, tmp); 4054 } 4055 4056 /** 4057 * uint32_t crc; 4058 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8); 4059 */ 4060 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) { 4061 fold_byte_crc32(crc, crc, table, tmp); 4062 } 4063 4064 /** 4065 * Emits code to update CRC-32 with a byte value according to constants in table. 4066 * 4067 * @param [in,out]crc Register containing the crc. 4068 * @param [in]val Register containing the byte to fold into the CRC. 4069 * @param [in]table Register containing the table of crc constants. 4070 * 4071 * uint32_t crc; 4072 * val = crc_table[(val ^ crc) & 0xFF]; 4073 * crc = val ^ (crc >> 8); 4074 */ 4075 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 4076 BLOCK_COMMENT("update_byte_crc32:"); 4077 xorr(val, val, crc); 4078 fold_byte_crc32(crc, val, table, val); 4079 } 4080 4081 /** 4082 * @param crc register containing existing CRC (32-bit) 4083 * @param buf register pointing to input byte buffer (byte*) 4084 * @param len register containing number of bytes 4085 * @param table register pointing to CRC table 4086 */ 4087 void MacroAssembler::update_byteLoop_crc32(Register crc, Register buf, Register len, Register table, 4088 Register data, bool loopAlignment, bool invertCRC) { 4089 assert_different_registers(crc, buf, len, table, data); 4090 4091 Label L_mainLoop, L_done; 4092 const int mainLoop_stepping = 1; 4093 const int mainLoop_alignment = loopAlignment ? 32 : 4; // (InputForNewCode > 4 ? InputForNewCode : 32) : 4; 4094 4095 // Process all bytes in a single-byte loop. 4096 clrldi_(len, len, 32); // Enforce 32 bit. Anything to do? 4097 beq(CCR0, L_done); 4098 4099 if (invertCRC) { 4100 nand(crc, crc, crc); // ~c 4101 } 4102 4103 mtctr(len); 4104 align(mainLoop_alignment); 4105 BIND(L_mainLoop); 4106 lbz(data, 0, buf); // Byte from buffer, zero-extended. 4107 addi(buf, buf, mainLoop_stepping); // Advance buffer position. 4108 update_byte_crc32(crc, data, table); 4109 bdnz(L_mainLoop); // Iterate. 4110 4111 if (invertCRC) { 4112 nand(crc, crc, crc); // ~c 4113 } 4114 4115 bind(L_done); 4116 } 4117 4118 /** 4119 * Emits code to update CRC-32 with a 4-byte value according to constants in table 4120 * Implementation according to jdk/src/share/native/java/util/zip/zlib-1.2.8/crc32.c 4121 */ 4122 // A not on the lookup table address(es): 4123 // The lookup table consists of two sets of four columns each. 4124 // The columns {0..3} are used for little-endian machines. 4125 // The columns {4..7} are used for big-endian machines. 4126 // To save the effort of adding the column offset to the table address each time 4127 // a table element is looked up, it is possible to pass the pre-calculated 4128 // column addresses. 4129 // Uses R9..R12 as work register. Must be saved/restored by caller, if necessary. 4130 void MacroAssembler::update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc, 4131 Register t0, Register t1, Register t2, Register t3, 4132 Register tc0, Register tc1, Register tc2, Register tc3) { 4133 assert_different_registers(crc, t3); 4134 4135 // XOR crc with next four bytes of buffer. 4136 lwz(t3, bufDisp, buf); 4137 if (bufInc != 0) { 4138 addi(buf, buf, bufInc); 4139 } 4140 xorr(t3, t3, crc); 4141 4142 // Chop crc into 4 single-byte pieces, shifted left 2 bits, to form the table indices. 4143 rlwinm(t0, t3, 2, 24-2, 31-2); // ((t1 >> 0) & 0xff) << 2 4144 rlwinm(t1, t3, 32+(2- 8), 24-2, 31-2); // ((t1 >> 8) & 0xff) << 2 4145 rlwinm(t2, t3, 32+(2-16), 24-2, 31-2); // ((t1 >> 16) & 0xff) << 2 4146 rlwinm(t3, t3, 32+(2-24), 24-2, 31-2); // ((t1 >> 24) & 0xff) << 2 4147 4148 // Use the pre-calculated column addresses. 4149 // Load pre-calculated table values. 4150 lwzx(t0, tc0, t0); 4151 lwzx(t1, tc1, t1); 4152 lwzx(t2, tc2, t2); 4153 lwzx(t3, tc3, t3); 4154 4155 // Calculate new crc from table values. 4156 xorr(t0, t0, t1); 4157 xorr(t2, t2, t3); 4158 xorr(crc, t0, t2); // Now crc contains the final checksum value. 4159 } 4160 4161 /** 4162 * @param crc register containing existing CRC (32-bit) 4163 * @param buf register pointing to input byte buffer (byte*) 4164 * @param len register containing number of bytes 4165 * @param table register pointing to CRC table 4166 * 4167 * Uses R9..R12 as work register. Must be saved/restored by caller! 4168 */ 4169 void MacroAssembler::kernel_crc32_2word(Register crc, Register buf, Register len, Register table, 4170 Register t0, Register t1, Register t2, Register t3, 4171 Register tc0, Register tc1, Register tc2, Register tc3) { 4172 assert_different_registers(crc, buf, len, table); 4173 4174 Label L_mainLoop, L_tail; 4175 Register tmp = t0; 4176 Register data = t0; 4177 Register tmp2 = t1; 4178 const int mainLoop_stepping = 8; 4179 const int tailLoop_stepping = 1; 4180 const int log_stepping = exact_log2(mainLoop_stepping); 4181 const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32; 4182 const int complexThreshold = 2*mainLoop_stepping; 4183 4184 // Don't test for len <= 0 here. This pathological case should not occur anyway. 4185 // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles. 4186 // The situation itself is detected and handled correctly by the conditional branches 4187 // following aghi(len, -stepping) and aghi(len, +stepping). 4188 assert(tailLoop_stepping == 1, "check tailLoop_stepping!"); 4189 4190 BLOCK_COMMENT("kernel_crc32_2word {"); 4191 4192 nand(crc, crc, crc); // ~c 4193 4194 // Check for short (<mainLoop_stepping) buffer. 4195 cmpdi(CCR0, len, complexThreshold); 4196 blt(CCR0, L_tail); 4197 4198 // Pre-mainLoop alignment did show a slight (1%) positive effect on performance. 4199 // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions. 4200 { 4201 // Align buf addr to mainLoop_stepping boundary. 4202 neg(tmp2, buf); // Calculate # preLoop iterations for alignment. 4203 rldicl(tmp2, tmp2, 0, 64-log_stepping); // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63. 4204 4205 if (complexThreshold > mainLoop_stepping) { 4206 sub(len, len, tmp2); // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed). 4207 } else { 4208 sub(tmp, len, tmp2); // Remaining bytes for main loop. 4209 cmpdi(CCR0, tmp, mainLoop_stepping); 4210 blt(CCR0, L_tail); // For less than one mainloop_stepping left, do only tail processing 4211 mr(len, tmp); // remaining bytes for main loop (>=mainLoop_stepping is guaranteed). 4212 } 4213 update_byteLoop_crc32(crc, buf, tmp2, table, data, false, false); 4214 } 4215 4216 srdi(tmp2, len, log_stepping); // #iterations for mainLoop 4217 andi(len, len, mainLoop_stepping-1); // remaining bytes for tailLoop 4218 mtctr(tmp2); 4219 4220 #ifdef VM_LITTLE_ENDIAN 4221 Register crc_rv = crc; 4222 #else 4223 Register crc_rv = tmp; // Load_reverse needs separate registers to work on. 4224 // Occupies tmp, but frees up crc. 4225 load_reverse_32(crc_rv, crc); // Revert byte order because we are dealing with big-endian data. 4226 tmp = crc; 4227 #endif 4228 4229 int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3); 4230 4231 align(mainLoop_alignment); // Octoword-aligned loop address. Shows 2% improvement. 4232 BIND(L_mainLoop); 4233 update_1word_crc32(crc_rv, buf, table, 0, 0, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3); 4234 update_1word_crc32(crc_rv, buf, table, 4, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3); 4235 bdnz(L_mainLoop); 4236 4237 #ifndef VM_LITTLE_ENDIAN 4238 load_reverse_32(crc, crc_rv); // Revert byte order because we are dealing with big-endian data. 4239 tmp = crc_rv; // Tmp uses it's original register again. 4240 #endif 4241 4242 // Restore original table address for tailLoop. 4243 if (reconstructTableOffset != 0) { 4244 addi(table, table, -reconstructTableOffset); 4245 } 4246 4247 // Process last few (<complexThreshold) bytes of buffer. 4248 BIND(L_tail); 4249 update_byteLoop_crc32(crc, buf, len, table, data, false, false); 4250 4251 nand(crc, crc, crc); // ~c 4252 BLOCK_COMMENT("} kernel_crc32_2word"); 4253 } 4254 4255 /** 4256 * @param crc register containing existing CRC (32-bit) 4257 * @param buf register pointing to input byte buffer (byte*) 4258 * @param len register containing number of bytes 4259 * @param table register pointing to CRC table 4260 * 4261 * uses R9..R12 as work register. Must be saved/restored by caller! 4262 */ 4263 void MacroAssembler::kernel_crc32_1word(Register crc, Register buf, Register len, Register table, 4264 Register t0, Register t1, Register t2, Register t3, 4265 Register tc0, Register tc1, Register tc2, Register tc3) { 4266 assert_different_registers(crc, buf, len, table); 4267 4268 Label L_mainLoop, L_tail; 4269 Register tmp = t0; 4270 Register data = t0; 4271 Register tmp2 = t1; 4272 const int mainLoop_stepping = 4; 4273 const int tailLoop_stepping = 1; 4274 const int log_stepping = exact_log2(mainLoop_stepping); 4275 const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32; 4276 const int complexThreshold = 2*mainLoop_stepping; 4277 4278 // Don't test for len <= 0 here. This pathological case should not occur anyway. 4279 // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles. 4280 // The situation itself is detected and handled correctly by the conditional branches 4281 // following aghi(len, -stepping) and aghi(len, +stepping). 4282 assert(tailLoop_stepping == 1, "check tailLoop_stepping!"); 4283 4284 BLOCK_COMMENT("kernel_crc32_1word {"); 4285 4286 nand(crc, crc, crc); // ~c 4287 4288 // Check for short (<mainLoop_stepping) buffer. 4289 cmpdi(CCR0, len, complexThreshold); 4290 blt(CCR0, L_tail); 4291 4292 // Pre-mainLoop alignment did show a slight (1%) positive effect on performance. 4293 // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions. 4294 { 4295 // Align buf addr to mainLoop_stepping boundary. 4296 neg(tmp2, buf); // Calculate # preLoop iterations for alignment. 4297 rldicl(tmp2, tmp2, 0, 64-log_stepping); // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63. 4298 4299 if (complexThreshold > mainLoop_stepping) { 4300 sub(len, len, tmp2); // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed). 4301 } else { 4302 sub(tmp, len, tmp2); // Remaining bytes for main loop. 4303 cmpdi(CCR0, tmp, mainLoop_stepping); 4304 blt(CCR0, L_tail); // For less than one mainloop_stepping left, do only tail processing 4305 mr(len, tmp); // remaining bytes for main loop (>=mainLoop_stepping is guaranteed). 4306 } 4307 update_byteLoop_crc32(crc, buf, tmp2, table, data, false, false); 4308 } 4309 4310 srdi(tmp2, len, log_stepping); // #iterations for mainLoop 4311 andi(len, len, mainLoop_stepping-1); // remaining bytes for tailLoop 4312 mtctr(tmp2); 4313 4314 #ifdef VM_LITTLE_ENDIAN 4315 Register crc_rv = crc; 4316 #else 4317 Register crc_rv = tmp; // Load_reverse needs separate registers to work on. 4318 // Occupies tmp, but frees up crc. 4319 load_reverse_32(crc_rv, crc); // Revert byte order because we are dealing with big-endian data. 4320 tmp = crc; 4321 #endif 4322 4323 int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3); 4324 4325 align(mainLoop_alignment); // Octoword-aligned loop address. Shows 2% improvement. 4326 BIND(L_mainLoop); 4327 update_1word_crc32(crc_rv, buf, table, 0, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3); 4328 bdnz(L_mainLoop); 4329 4330 #ifndef VM_LITTLE_ENDIAN 4331 load_reverse_32(crc, crc_rv); // Revert byte order because we are dealing with big-endian data. 4332 tmp = crc_rv; // Tmp uses it's original register again. 4333 #endif 4334 4335 // Restore original table address for tailLoop. 4336 if (reconstructTableOffset != 0) { 4337 addi(table, table, -reconstructTableOffset); 4338 } 4339 4340 // Process last few (<complexThreshold) bytes of buffer. 4341 BIND(L_tail); 4342 update_byteLoop_crc32(crc, buf, len, table, data, false, false); 4343 4344 nand(crc, crc, crc); // ~c 4345 BLOCK_COMMENT("} kernel_crc32_1word"); 4346 } 4347 4348 /** 4349 * @param crc register containing existing CRC (32-bit) 4350 * @param buf register pointing to input byte buffer (byte*) 4351 * @param len register containing number of bytes 4352 * @param table register pointing to CRC table 4353 * 4354 * Uses R7_ARG5, R8_ARG6 as work registers. 4355 */ 4356 void MacroAssembler::kernel_crc32_1byte(Register crc, Register buf, Register len, Register table, 4357 Register t0, Register t1, Register t2, Register t3) { 4358 assert_different_registers(crc, buf, len, table); 4359 4360 Register data = t0; // Holds the current byte to be folded into crc. 4361 4362 BLOCK_COMMENT("kernel_crc32_1byte {"); 4363 4364 // Process all bytes in a single-byte loop. 4365 update_byteLoop_crc32(crc, buf, len, table, data, true, true); 4366 4367 BLOCK_COMMENT("} kernel_crc32_1byte"); 4368 } 4369 4370 /** 4371 * @param crc register containing existing CRC (32-bit) 4372 * @param buf register pointing to input byte buffer (byte*) 4373 * @param len register containing number of bytes 4374 * @param table register pointing to CRC table 4375 * @param constants register pointing to CRC table for 128-bit aligned memory 4376 * @param barretConstants register pointing to table for barrett reduction 4377 * @param t0 volatile register 4378 * @param t1 volatile register 4379 * @param t2 volatile register 4380 * @param t3 volatile register 4381 */ 4382 void MacroAssembler::kernel_crc32_1word_vpmsumd(Register crc, Register buf, Register len, Register table, 4383 Register constants, Register barretConstants, 4384 Register t0, Register t1, Register t2, Register t3, Register t4) { 4385 assert_different_registers(crc, buf, len, table); 4386 4387 Label L_alignedHead, L_tail, L_alignTail, L_start, L_end; 4388 4389 Register prealign = t0; 4390 Register postalign = t0; 4391 4392 BLOCK_COMMENT("kernel_crc32_1word_vpmsumb {"); 4393 4394 // 1. use kernel_crc32_1word for shorter than 384bit 4395 clrldi(len, len, 32); 4396 cmpdi(CCR0, len, 384); 4397 bge(CCR0, L_start); 4398 4399 Register tc0 = t4; 4400 Register tc1 = constants; 4401 Register tc2 = barretConstants; 4402 kernel_crc32_1word(crc, buf, len, table,t0, t1, t2, t3, tc0, tc1, tc2, table); 4403 b(L_end); 4404 4405 BIND(L_start); 4406 4407 // 2. ~c 4408 nand(crc, crc, crc); 4409 4410 // 3. calculate from 0 to first 128bit-aligned address 4411 clrldi_(prealign, buf, 57); 4412 beq(CCR0, L_alignedHead); 4413 4414 subfic(prealign, prealign, 128); 4415 4416 subf(len, prealign, len); 4417 update_byteLoop_crc32(crc, buf, prealign, table, t2, false, false); 4418 4419 // 4. calculate from first 128bit-aligned address to last 128bit-aligned address 4420 BIND(L_alignedHead); 4421 4422 clrldi(postalign, len, 57); 4423 subf(len, postalign, len); 4424 4425 // len must be more than 256bit 4426 kernel_crc32_1word_aligned(crc, buf, len, constants, barretConstants, t1, t2, t3); 4427 4428 // 5. calculate remaining 4429 cmpdi(CCR0, postalign, 0); 4430 beq(CCR0, L_tail); 4431 4432 update_byteLoop_crc32(crc, buf, postalign, table, t2, false, false); 4433 4434 BIND(L_tail); 4435 4436 // 6. ~c 4437 nand(crc, crc, crc); 4438 4439 BIND(L_end); 4440 4441 BLOCK_COMMENT("} kernel_crc32_1word_vpmsumb"); 4442 } 4443 4444 /** 4445 * @param crc register containing existing CRC (32-bit) 4446 * @param buf register pointing to input byte buffer (byte*) 4447 * @param len register containing number of bytes 4448 * @param constants register pointing to CRC table for 128-bit aligned memory 4449 * @param barretConstants register pointing to table for barrett reduction 4450 * @param t0 volatile register 4451 * @param t1 volatile register 4452 * @param t2 volatile register 4453 */ 4454 void MacroAssembler::kernel_crc32_1word_aligned(Register crc, Register buf, Register len, 4455 Register constants, Register barretConstants, Register t0, Register t1, Register t2) { 4456 Label L_mainLoop, L_tail, L_alignTail, L_barrett_reduction, L_end, L_first_warm_up_done, L_first_cool_down, L_second_cool_down, L_XOR, L_test; 4457 Label L_lv0, L_lv1, L_lv2, L_lv3, L_lv4, L_lv5, L_lv6, L_lv7, L_lv8, L_lv9, L_lv10, L_lv11, L_lv12, L_lv13, L_lv14, L_lv15; 4458 Label L_1, L_2, L_3, L_4; 4459 4460 Register rLoaded = t0; 4461 Register rTmp1 = t1; 4462 Register rTmp2 = t2; 4463 Register off16 = R22; 4464 Register off32 = R23; 4465 Register off48 = R24; 4466 Register off64 = R25; 4467 Register off80 = R26; 4468 Register off96 = R27; 4469 Register off112 = R28; 4470 Register rIdx = R29; 4471 Register rMax = R30; 4472 Register constantsPos = R31; 4473 4474 VectorRegister mask_32bit = VR24; 4475 VectorRegister mask_64bit = VR25; 4476 VectorRegister zeroes = VR26; 4477 VectorRegister const1 = VR27; 4478 VectorRegister const2 = VR28; 4479 4480 // Save non-volatile vector registers (frameless). 4481 Register offset = t1; int offsetInt = 0; 4482 offsetInt -= 16; li(offset, -16); stvx(VR20, offset, R1_SP); 4483 offsetInt -= 16; addi(offset, offset, -16); stvx(VR21, offset, R1_SP); 4484 offsetInt -= 16; addi(offset, offset, -16); stvx(VR22, offset, R1_SP); 4485 offsetInt -= 16; addi(offset, offset, -16); stvx(VR23, offset, R1_SP); 4486 offsetInt -= 16; addi(offset, offset, -16); stvx(VR24, offset, R1_SP); 4487 offsetInt -= 16; addi(offset, offset, -16); stvx(VR25, offset, R1_SP); 4488 offsetInt -= 16; addi(offset, offset, -16); stvx(VR26, offset, R1_SP); 4489 offsetInt -= 16; addi(offset, offset, -16); stvx(VR27, offset, R1_SP); 4490 offsetInt -= 16; addi(offset, offset, -16); stvx(VR28, offset, R1_SP); 4491 offsetInt -= 8; std(R22, offsetInt, R1_SP); 4492 offsetInt -= 8; std(R23, offsetInt, R1_SP); 4493 offsetInt -= 8; std(R24, offsetInt, R1_SP); 4494 offsetInt -= 8; std(R25, offsetInt, R1_SP); 4495 offsetInt -= 8; std(R26, offsetInt, R1_SP); 4496 offsetInt -= 8; std(R27, offsetInt, R1_SP); 4497 offsetInt -= 8; std(R28, offsetInt, R1_SP); 4498 offsetInt -= 8; std(R29, offsetInt, R1_SP); 4499 offsetInt -= 8; std(R30, offsetInt, R1_SP); 4500 offsetInt -= 8; std(R31, offsetInt, R1_SP); 4501 4502 // Set constants 4503 li(off16, 16); 4504 li(off32, 32); 4505 li(off48, 48); 4506 li(off64, 64); 4507 li(off80, 80); 4508 li(off96, 96); 4509 li(off112, 112); 4510 4511 clrldi(crc, crc, 32); 4512 4513 vxor(zeroes, zeroes, zeroes); 4514 vspltisw(VR0, -1); 4515 4516 vsldoi(mask_32bit, zeroes, VR0, 4); 4517 vsldoi(mask_64bit, zeroes, VR0, -8); 4518 4519 // Get the initial value into v8 4520 vxor(VR8, VR8, VR8); 4521 mtvrd(VR8, crc); 4522 vsldoi(VR8, zeroes, VR8, -8); // shift into bottom 32 bits 4523 4524 li (rLoaded, 0); 4525 4526 rldicr(rIdx, len, 0, 56); 4527 4528 { 4529 BIND(L_1); 4530 // Checksum in blocks of MAX_SIZE (32768) 4531 lis(rMax, 0); 4532 ori(rMax, rMax, 32768); 4533 mr(rTmp2, rMax); 4534 cmpd(CCR0, rIdx, rMax); 4535 bgt(CCR0, L_2); 4536 mr(rMax, rIdx); 4537 4538 BIND(L_2); 4539 subf(rIdx, rMax, rIdx); 4540 4541 // our main loop does 128 bytes at a time 4542 srdi(rMax, rMax, 7); 4543 4544 /* 4545 * Work out the offset into the constants table to start at. Each 4546 * constant is 16 bytes, and it is used against 128 bytes of input 4547 * data - 128 / 16 = 8 4548 */ 4549 sldi(rTmp1, rMax, 4); 4550 srdi(rTmp2, rTmp2, 3); 4551 subf(rTmp1, rTmp1, rTmp2); 4552 4553 // We reduce our final 128 bytes in a separate step 4554 addi(rMax, rMax, -1); 4555 mtctr(rMax); 4556 4557 // Find the start of our constants 4558 add(constantsPos, constants, rTmp1); 4559 4560 // zero VR0-v7 which will contain our checksums 4561 vxor(VR0, VR0, VR0); 4562 vxor(VR1, VR1, VR1); 4563 vxor(VR2, VR2, VR2); 4564 vxor(VR3, VR3, VR3); 4565 vxor(VR4, VR4, VR4); 4566 vxor(VR5, VR5, VR5); 4567 vxor(VR6, VR6, VR6); 4568 vxor(VR7, VR7, VR7); 4569 4570 lvx(const1, constantsPos); 4571 4572 /* 4573 * If we are looping back to consume more data we use the values 4574 * already in VR16-v23. 4575 */ 4576 cmpdi(CCR0, rLoaded, 1); 4577 beq(CCR0, L_3); 4578 { 4579 4580 // First warm up pass 4581 lvx(VR16, buf); 4582 lvx(VR17, off16, buf); 4583 lvx(VR18, off32, buf); 4584 lvx(VR19, off48, buf); 4585 lvx(VR20, off64, buf); 4586 lvx(VR21, off80, buf); 4587 lvx(VR22, off96, buf); 4588 lvx(VR23, off112, buf); 4589 addi(buf, buf, 8*16); 4590 4591 // xor in initial value 4592 vxor(VR16, VR16, VR8); 4593 } 4594 4595 BIND(L_3); 4596 bdz(L_first_warm_up_done); 4597 4598 addi(constantsPos, constantsPos, 16); 4599 lvx(const2, constantsPos); 4600 4601 // Second warm up pass 4602 vpmsumd(VR8, VR16, const1); 4603 lvx(VR16, buf); 4604 4605 vpmsumd(VR9, VR17, const1); 4606 lvx(VR17, off16, buf); 4607 4608 vpmsumd(VR10, VR18, const1); 4609 lvx(VR18, off32, buf); 4610 4611 vpmsumd(VR11, VR19, const1); 4612 lvx(VR19, off48, buf); 4613 4614 vpmsumd(VR12, VR20, const1); 4615 lvx(VR20, off64, buf); 4616 4617 vpmsumd(VR13, VR21, const1); 4618 lvx(VR21, off80, buf); 4619 4620 vpmsumd(VR14, VR22, const1); 4621 lvx(VR22, off96, buf); 4622 4623 vpmsumd(VR15, VR23, const1); 4624 lvx(VR23, off112, buf); 4625 4626 addi(buf, buf, 8 * 16); 4627 4628 bdz(L_first_cool_down); 4629 4630 /* 4631 * main loop. We modulo schedule it such that it takes three iterations 4632 * to complete - first iteration load, second iteration vpmsum, third 4633 * iteration xor. 4634 */ 4635 { 4636 BIND(L_4); 4637 lvx(const1, constantsPos); addi(constantsPos, constantsPos, 16); 4638 4639 vxor(VR0, VR0, VR8); 4640 vpmsumd(VR8, VR16, const2); 4641 lvx(VR16, buf); 4642 4643 vxor(VR1, VR1, VR9); 4644 vpmsumd(VR9, VR17, const2); 4645 lvx(VR17, off16, buf); 4646 4647 vxor(VR2, VR2, VR10); 4648 vpmsumd(VR10, VR18, const2); 4649 lvx(VR18, off32, buf); 4650 4651 vxor(VR3, VR3, VR11); 4652 vpmsumd(VR11, VR19, const2); 4653 lvx(VR19, off48, buf); 4654 lvx(const2, constantsPos); 4655 4656 vxor(VR4, VR4, VR12); 4657 vpmsumd(VR12, VR20, const1); 4658 lvx(VR20, off64, buf); 4659 4660 vxor(VR5, VR5, VR13); 4661 vpmsumd(VR13, VR21, const1); 4662 lvx(VR21, off80, buf); 4663 4664 vxor(VR6, VR6, VR14); 4665 vpmsumd(VR14, VR22, const1); 4666 lvx(VR22, off96, buf); 4667 4668 vxor(VR7, VR7, VR15); 4669 vpmsumd(VR15, VR23, const1); 4670 lvx(VR23, off112, buf); 4671 4672 addi(buf, buf, 8 * 16); 4673 4674 bdnz(L_4); 4675 } 4676 4677 BIND(L_first_cool_down); 4678 4679 // First cool down pass 4680 lvx(const1, constantsPos); 4681 addi(constantsPos, constantsPos, 16); 4682 4683 vxor(VR0, VR0, VR8); 4684 vpmsumd(VR8, VR16, const1); 4685 4686 vxor(VR1, VR1, VR9); 4687 vpmsumd(VR9, VR17, const1); 4688 4689 vxor(VR2, VR2, VR10); 4690 vpmsumd(VR10, VR18, const1); 4691 4692 vxor(VR3, VR3, VR11); 4693 vpmsumd(VR11, VR19, const1); 4694 4695 vxor(VR4, VR4, VR12); 4696 vpmsumd(VR12, VR20, const1); 4697 4698 vxor(VR5, VR5, VR13); 4699 vpmsumd(VR13, VR21, const1); 4700 4701 vxor(VR6, VR6, VR14); 4702 vpmsumd(VR14, VR22, const1); 4703 4704 vxor(VR7, VR7, VR15); 4705 vpmsumd(VR15, VR23, const1); 4706 4707 BIND(L_second_cool_down); 4708 // Second cool down pass 4709 vxor(VR0, VR0, VR8); 4710 vxor(VR1, VR1, VR9); 4711 vxor(VR2, VR2, VR10); 4712 vxor(VR3, VR3, VR11); 4713 vxor(VR4, VR4, VR12); 4714 vxor(VR5, VR5, VR13); 4715 vxor(VR6, VR6, VR14); 4716 vxor(VR7, VR7, VR15); 4717 4718 /* 4719 * vpmsumd produces a 96 bit result in the least significant bits 4720 * of the register. Since we are bit reflected we have to shift it 4721 * left 32 bits so it occupies the least significant bits in the 4722 * bit reflected domain. 4723 */ 4724 vsldoi(VR0, VR0, zeroes, 4); 4725 vsldoi(VR1, VR1, zeroes, 4); 4726 vsldoi(VR2, VR2, zeroes, 4); 4727 vsldoi(VR3, VR3, zeroes, 4); 4728 vsldoi(VR4, VR4, zeroes, 4); 4729 vsldoi(VR5, VR5, zeroes, 4); 4730 vsldoi(VR6, VR6, zeroes, 4); 4731 vsldoi(VR7, VR7, zeroes, 4); 4732 4733 // xor with last 1024 bits 4734 lvx(VR8, buf); 4735 lvx(VR9, off16, buf); 4736 lvx(VR10, off32, buf); 4737 lvx(VR11, off48, buf); 4738 lvx(VR12, off64, buf); 4739 lvx(VR13, off80, buf); 4740 lvx(VR14, off96, buf); 4741 lvx(VR15, off112, buf); 4742 addi(buf, buf, 8 * 16); 4743 4744 vxor(VR16, VR0, VR8); 4745 vxor(VR17, VR1, VR9); 4746 vxor(VR18, VR2, VR10); 4747 vxor(VR19, VR3, VR11); 4748 vxor(VR20, VR4, VR12); 4749 vxor(VR21, VR5, VR13); 4750 vxor(VR22, VR6, VR14); 4751 vxor(VR23, VR7, VR15); 4752 4753 li(rLoaded, 1); 4754 cmpdi(CCR0, rIdx, 0); 4755 addi(rIdx, rIdx, 128); 4756 bne(CCR0, L_1); 4757 } 4758 4759 // Work out how many bytes we have left 4760 andi_(len, len, 127); 4761 4762 // Calculate where in the constant table we need to start 4763 subfic(rTmp1, len, 128); 4764 add(constantsPos, constantsPos, rTmp1); 4765 4766 // How many 16 byte chunks are in the tail 4767 srdi(rIdx, len, 4); 4768 mtctr(rIdx); 4769 4770 /* 4771 * Reduce the previously calculated 1024 bits to 64 bits, shifting 4772 * 32 bits to include the trailing 32 bits of zeros 4773 */ 4774 lvx(VR0, constantsPos); 4775 lvx(VR1, off16, constantsPos); 4776 lvx(VR2, off32, constantsPos); 4777 lvx(VR3, off48, constantsPos); 4778 lvx(VR4, off64, constantsPos); 4779 lvx(VR5, off80, constantsPos); 4780 lvx(VR6, off96, constantsPos); 4781 lvx(VR7, off112, constantsPos); 4782 addi(constantsPos, constantsPos, 8 * 16); 4783 4784 vpmsumw(VR0, VR16, VR0); 4785 vpmsumw(VR1, VR17, VR1); 4786 vpmsumw(VR2, VR18, VR2); 4787 vpmsumw(VR3, VR19, VR3); 4788 vpmsumw(VR4, VR20, VR4); 4789 vpmsumw(VR5, VR21, VR5); 4790 vpmsumw(VR6, VR22, VR6); 4791 vpmsumw(VR7, VR23, VR7); 4792 4793 // Now reduce the tail (0 - 112 bytes) 4794 cmpdi(CCR0, rIdx, 0); 4795 beq(CCR0, L_XOR); 4796 4797 lvx(VR16, buf); addi(buf, buf, 16); 4798 lvx(VR17, constantsPos); 4799 vpmsumw(VR16, VR16, VR17); 4800 vxor(VR0, VR0, VR16); 4801 beq(CCR0, L_XOR); 4802 4803 lvx(VR16, buf); addi(buf, buf, 16); 4804 lvx(VR17, off16, constantsPos); 4805 vpmsumw(VR16, VR16, VR17); 4806 vxor(VR0, VR0, VR16); 4807 beq(CCR0, L_XOR); 4808 4809 lvx(VR16, buf); addi(buf, buf, 16); 4810 lvx(VR17, off32, constantsPos); 4811 vpmsumw(VR16, VR16, VR17); 4812 vxor(VR0, VR0, VR16); 4813 beq(CCR0, L_XOR); 4814 4815 lvx(VR16, buf); addi(buf, buf, 16); 4816 lvx(VR17, off48,constantsPos); 4817 vpmsumw(VR16, VR16, VR17); 4818 vxor(VR0, VR0, VR16); 4819 beq(CCR0, L_XOR); 4820 4821 lvx(VR16, buf); addi(buf, buf, 16); 4822 lvx(VR17, off64, constantsPos); 4823 vpmsumw(VR16, VR16, VR17); 4824 vxor(VR0, VR0, VR16); 4825 beq(CCR0, L_XOR); 4826 4827 lvx(VR16, buf); addi(buf, buf, 16); 4828 lvx(VR17, off80, constantsPos); 4829 vpmsumw(VR16, VR16, VR17); 4830 vxor(VR0, VR0, VR16); 4831 beq(CCR0, L_XOR); 4832 4833 lvx(VR16, buf); addi(buf, buf, 16); 4834 lvx(VR17, off96, constantsPos); 4835 vpmsumw(VR16, VR16, VR17); 4836 vxor(VR0, VR0, VR16); 4837 4838 // Now xor all the parallel chunks together 4839 BIND(L_XOR); 4840 vxor(VR0, VR0, VR1); 4841 vxor(VR2, VR2, VR3); 4842 vxor(VR4, VR4, VR5); 4843 vxor(VR6, VR6, VR7); 4844 4845 vxor(VR0, VR0, VR2); 4846 vxor(VR4, VR4, VR6); 4847 4848 vxor(VR0, VR0, VR4); 4849 4850 b(L_barrett_reduction); 4851 4852 BIND(L_first_warm_up_done); 4853 lvx(const1, constantsPos); 4854 addi(constantsPos, constantsPos, 16); 4855 vpmsumd(VR8, VR16, const1); 4856 vpmsumd(VR9, VR17, const1); 4857 vpmsumd(VR10, VR18, const1); 4858 vpmsumd(VR11, VR19, const1); 4859 vpmsumd(VR12, VR20, const1); 4860 vpmsumd(VR13, VR21, const1); 4861 vpmsumd(VR14, VR22, const1); 4862 vpmsumd(VR15, VR23, const1); 4863 b(L_second_cool_down); 4864 4865 BIND(L_barrett_reduction); 4866 4867 lvx(const1, barretConstants); 4868 addi(barretConstants, barretConstants, 16); 4869 lvx(const2, barretConstants); 4870 4871 vsldoi(VR1, VR0, VR0, -8); 4872 vxor(VR0, VR0, VR1); // xor two 64 bit results together 4873 4874 // shift left one bit 4875 vspltisb(VR1, 1); 4876 vsl(VR0, VR0, VR1); 4877 4878 vand(VR0, VR0, mask_64bit); 4879 4880 /* 4881 * The reflected version of Barrett reduction. Instead of bit 4882 * reflecting our data (which is expensive to do), we bit reflect our 4883 * constants and our algorithm, which means the intermediate data in 4884 * our vector registers goes from 0-63 instead of 63-0. We can reflect 4885 * the algorithm because we don't carry in mod 2 arithmetic. 4886 */ 4887 vand(VR1, VR0, mask_32bit); // bottom 32 bits of a 4888 vpmsumd(VR1, VR1, const1); // ma 4889 vand(VR1, VR1, mask_32bit); // bottom 32bits of ma 4890 vpmsumd(VR1, VR1, const2); // qn */ 4891 vxor(VR0, VR0, VR1); // a - qn, subtraction is xor in GF(2) 4892 4893 /* 4894 * Since we are bit reflected, the result (ie the low 32 bits) is in 4895 * the high 32 bits. We just need to shift it left 4 bytes 4896 * V0 [ 0 1 X 3 ] 4897 * V0 [ 0 X 2 3 ] 4898 */ 4899 vsldoi(VR0, VR0, zeroes, 4); // shift result into top 64 bits of 4900 4901 // Get it into r3 4902 mfvrd(crc, VR0); 4903 4904 BIND(L_end); 4905 4906 offsetInt = 0; 4907 // Restore non-volatile Vector registers (frameless). 4908 offsetInt -= 16; li(offset, -16); lvx(VR20, offset, R1_SP); 4909 offsetInt -= 16; addi(offset, offset, -16); lvx(VR21, offset, R1_SP); 4910 offsetInt -= 16; addi(offset, offset, -16); lvx(VR22, offset, R1_SP); 4911 offsetInt -= 16; addi(offset, offset, -16); lvx(VR23, offset, R1_SP); 4912 offsetInt -= 16; addi(offset, offset, -16); lvx(VR24, offset, R1_SP); 4913 offsetInt -= 16; addi(offset, offset, -16); lvx(VR25, offset, R1_SP); 4914 offsetInt -= 16; addi(offset, offset, -16); lvx(VR26, offset, R1_SP); 4915 offsetInt -= 16; addi(offset, offset, -16); lvx(VR27, offset, R1_SP); 4916 offsetInt -= 16; addi(offset, offset, -16); lvx(VR28, offset, R1_SP); 4917 offsetInt -= 8; ld(R22, offsetInt, R1_SP); 4918 offsetInt -= 8; ld(R23, offsetInt, R1_SP); 4919 offsetInt -= 8; ld(R24, offsetInt, R1_SP); 4920 offsetInt -= 8; ld(R25, offsetInt, R1_SP); 4921 offsetInt -= 8; ld(R26, offsetInt, R1_SP); 4922 offsetInt -= 8; ld(R27, offsetInt, R1_SP); 4923 offsetInt -= 8; ld(R28, offsetInt, R1_SP); 4924 offsetInt -= 8; ld(R29, offsetInt, R1_SP); 4925 offsetInt -= 8; ld(R30, offsetInt, R1_SP); 4926 offsetInt -= 8; ld(R31, offsetInt, R1_SP); 4927 } 4928 4929 void MacroAssembler::kernel_crc32_singleByte(Register crc, Register buf, Register len, Register table, Register tmp) { 4930 assert_different_registers(crc, buf, /* len, not used!! */ table, tmp); 4931 4932 BLOCK_COMMENT("kernel_crc32_singleByte:"); 4933 nand(crc, crc, crc); // ~c 4934 4935 lbz(tmp, 0, buf); // Byte from buffer, zero-extended. 4936 update_byte_crc32(crc, tmp, table); 4937 4938 nand(crc, crc, crc); // ~c 4939 } 4940 4941 // dest_lo += src1 + src2 4942 // dest_hi += carry1 + carry2 4943 void MacroAssembler::add2_with_carry(Register dest_hi, 4944 Register dest_lo, 4945 Register src1, Register src2) { 4946 li(R0, 0); 4947 addc(dest_lo, dest_lo, src1); 4948 adde(dest_hi, dest_hi, R0); 4949 addc(dest_lo, dest_lo, src2); 4950 adde(dest_hi, dest_hi, R0); 4951 } 4952 4953 // Multiply 64 bit by 64 bit first loop. 4954 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, 4955 Register x_xstart, 4956 Register y, Register y_idx, 4957 Register z, 4958 Register carry, 4959 Register product_high, Register product, 4960 Register idx, Register kdx, 4961 Register tmp) { 4962 // jlong carry, x[], y[], z[]; 4963 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { 4964 // huge_128 product = y[idx] * x[xstart] + carry; 4965 // z[kdx] = (jlong)product; 4966 // carry = (jlong)(product >>> 64); 4967 // } 4968 // z[xstart] = carry; 4969 4970 Label L_first_loop, L_first_loop_exit; 4971 Label L_one_x, L_one_y, L_multiply; 4972 4973 addic_(xstart, xstart, -1); 4974 blt(CCR0, L_one_x); // Special case: length of x is 1. 4975 4976 // Load next two integers of x. 4977 sldi(tmp, xstart, LogBytesPerInt); 4978 ldx(x_xstart, x, tmp); 4979 #ifdef VM_LITTLE_ENDIAN 4980 rldicl(x_xstart, x_xstart, 32, 0); 4981 #endif 4982 4983 align(32, 16); 4984 bind(L_first_loop); 4985 4986 cmpdi(CCR0, idx, 1); 4987 blt(CCR0, L_first_loop_exit); 4988 addi(idx, idx, -2); 4989 beq(CCR0, L_one_y); 4990 4991 // Load next two integers of y. 4992 sldi(tmp, idx, LogBytesPerInt); 4993 ldx(y_idx, y, tmp); 4994 #ifdef VM_LITTLE_ENDIAN 4995 rldicl(y_idx, y_idx, 32, 0); 4996 #endif 4997 4998 4999 bind(L_multiply); 5000 multiply64(product_high, product, x_xstart, y_idx); 5001 5002 li(tmp, 0); 5003 addc(product, product, carry); // Add carry to result. 5004 adde(product_high, product_high, tmp); // Add carry of the last addition. 5005 addi(kdx, kdx, -2); 5006 5007 // Store result. 5008 #ifdef VM_LITTLE_ENDIAN 5009 rldicl(product, product, 32, 0); 5010 #endif 5011 sldi(tmp, kdx, LogBytesPerInt); 5012 stdx(product, z, tmp); 5013 mr_if_needed(carry, product_high); 5014 b(L_first_loop); 5015 5016 5017 bind(L_one_y); // Load one 32 bit portion of y as (0,value). 5018 5019 lwz(y_idx, 0, y); 5020 b(L_multiply); 5021 5022 5023 bind(L_one_x); // Load one 32 bit portion of x as (0,value). 5024 5025 lwz(x_xstart, 0, x); 5026 b(L_first_loop); 5027 5028 bind(L_first_loop_exit); 5029 } 5030 5031 // Multiply 64 bit by 64 bit and add 128 bit. 5032 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, 5033 Register z, Register yz_idx, 5034 Register idx, Register carry, 5035 Register product_high, Register product, 5036 Register tmp, int offset) { 5037 5038 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry; 5039 // z[kdx] = (jlong)product; 5040 5041 sldi(tmp, idx, LogBytesPerInt); 5042 if (offset) { 5043 addi(tmp, tmp, offset); 5044 } 5045 ldx(yz_idx, y, tmp); 5046 #ifdef VM_LITTLE_ENDIAN 5047 rldicl(yz_idx, yz_idx, 32, 0); 5048 #endif 5049 5050 multiply64(product_high, product, x_xstart, yz_idx); 5051 ldx(yz_idx, z, tmp); 5052 #ifdef VM_LITTLE_ENDIAN 5053 rldicl(yz_idx, yz_idx, 32, 0); 5054 #endif 5055 5056 add2_with_carry(product_high, product, carry, yz_idx); 5057 5058 sldi(tmp, idx, LogBytesPerInt); 5059 if (offset) { 5060 addi(tmp, tmp, offset); 5061 } 5062 #ifdef VM_LITTLE_ENDIAN 5063 rldicl(product, product, 32, 0); 5064 #endif 5065 stdx(product, z, tmp); 5066 } 5067 5068 // Multiply 128 bit by 128 bit. Unrolled inner loop. 5069 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, 5070 Register y, Register z, 5071 Register yz_idx, Register idx, Register carry, 5072 Register product_high, Register product, 5073 Register carry2, Register tmp) { 5074 5075 // jlong carry, x[], y[], z[]; 5076 // int kdx = ystart+1; 5077 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 5078 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry; 5079 // z[kdx+idx+1] = (jlong)product; 5080 // jlong carry2 = (jlong)(product >>> 64); 5081 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2; 5082 // z[kdx+idx] = (jlong)product; 5083 // carry = (jlong)(product >>> 64); 5084 // } 5085 // idx += 2; 5086 // if (idx > 0) { 5087 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry; 5088 // z[kdx+idx] = (jlong)product; 5089 // carry = (jlong)(product >>> 64); 5090 // } 5091 5092 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 5093 const Register jdx = R0; 5094 5095 // Scale the index. 5096 srdi_(jdx, idx, 2); 5097 beq(CCR0, L_third_loop_exit); 5098 mtctr(jdx); 5099 5100 align(32, 16); 5101 bind(L_third_loop); 5102 5103 addi(idx, idx, -4); 5104 5105 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 8); 5106 mr_if_needed(carry2, product_high); 5107 5108 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product_high, product, tmp, 0); 5109 mr_if_needed(carry, product_high); 5110 bdnz(L_third_loop); 5111 5112 bind(L_third_loop_exit); // Handle any left-over operand parts. 5113 5114 andi_(idx, idx, 0x3); 5115 beq(CCR0, L_post_third_loop_done); 5116 5117 Label L_check_1; 5118 5119 addic_(idx, idx, -2); 5120 blt(CCR0, L_check_1); 5121 5122 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 0); 5123 mr_if_needed(carry, product_high); 5124 5125 bind(L_check_1); 5126 5127 addi(idx, idx, 0x2); 5128 andi_(idx, idx, 0x1); 5129 addic_(idx, idx, -1); 5130 blt(CCR0, L_post_third_loop_done); 5131 5132 sldi(tmp, idx, LogBytesPerInt); 5133 lwzx(yz_idx, y, tmp); 5134 multiply64(product_high, product, x_xstart, yz_idx); 5135 lwzx(yz_idx, z, tmp); 5136 5137 add2_with_carry(product_high, product, yz_idx, carry); 5138 5139 sldi(tmp, idx, LogBytesPerInt); 5140 stwx(product, z, tmp); 5141 srdi(product, product, 32); 5142 5143 sldi(product_high, product_high, 32); 5144 orr(product, product, product_high); 5145 mr_if_needed(carry, product); 5146 5147 bind(L_post_third_loop_done); 5148 } // multiply_128_x_128_loop 5149 5150 void MacroAssembler::multiply_to_len(Register x, Register xlen, 5151 Register y, Register ylen, 5152 Register z, Register zlen, 5153 Register tmp1, Register tmp2, 5154 Register tmp3, Register tmp4, 5155 Register tmp5, Register tmp6, 5156 Register tmp7, Register tmp8, 5157 Register tmp9, Register tmp10, 5158 Register tmp11, Register tmp12, 5159 Register tmp13) { 5160 5161 ShortBranchVerifier sbv(this); 5162 5163 assert_different_registers(x, xlen, y, ylen, z, zlen, 5164 tmp1, tmp2, tmp3, tmp4, tmp5, tmp6); 5165 assert_different_registers(x, xlen, y, ylen, z, zlen, 5166 tmp1, tmp2, tmp3, tmp4, tmp5, tmp7); 5167 assert_different_registers(x, xlen, y, ylen, z, zlen, 5168 tmp1, tmp2, tmp3, tmp4, tmp5, tmp8); 5169 5170 const Register idx = tmp1; 5171 const Register kdx = tmp2; 5172 const Register xstart = tmp3; 5173 5174 const Register y_idx = tmp4; 5175 const Register carry = tmp5; 5176 const Register product = tmp6; 5177 const Register product_high = tmp7; 5178 const Register x_xstart = tmp8; 5179 const Register tmp = tmp9; 5180 5181 // First Loop. 5182 // 5183 // final static long LONG_MASK = 0xffffffffL; 5184 // int xstart = xlen - 1; 5185 // int ystart = ylen - 1; 5186 // long carry = 0; 5187 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) { 5188 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry; 5189 // z[kdx] = (int)product; 5190 // carry = product >>> 32; 5191 // } 5192 // z[xstart] = (int)carry; 5193 5194 mr_if_needed(idx, ylen); // idx = ylen 5195 mr_if_needed(kdx, zlen); // kdx = xlen + ylen 5196 li(carry, 0); // carry = 0 5197 5198 Label L_done; 5199 5200 addic_(xstart, xlen, -1); 5201 blt(CCR0, L_done); 5202 5203 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, 5204 carry, product_high, product, idx, kdx, tmp); 5205 5206 Label L_second_loop; 5207 5208 cmpdi(CCR0, kdx, 0); 5209 beq(CCR0, L_second_loop); 5210 5211 Label L_carry; 5212 5213 addic_(kdx, kdx, -1); 5214 beq(CCR0, L_carry); 5215 5216 // Store lower 32 bits of carry. 5217 sldi(tmp, kdx, LogBytesPerInt); 5218 stwx(carry, z, tmp); 5219 srdi(carry, carry, 32); 5220 addi(kdx, kdx, -1); 5221 5222 5223 bind(L_carry); 5224 5225 // Store upper 32 bits of carry. 5226 sldi(tmp, kdx, LogBytesPerInt); 5227 stwx(carry, z, tmp); 5228 5229 // Second and third (nested) loops. 5230 // 5231 // for (int i = xstart-1; i >= 0; i--) { // Second loop 5232 // carry = 0; 5233 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 5234 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 5235 // (z[k] & LONG_MASK) + carry; 5236 // z[k] = (int)product; 5237 // carry = product >>> 32; 5238 // } 5239 // z[i] = (int)carry; 5240 // } 5241 // 5242 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx 5243 5244 bind(L_second_loop); 5245 5246 li(carry, 0); // carry = 0; 5247 5248 addic_(xstart, xstart, -1); // i = xstart-1; 5249 blt(CCR0, L_done); 5250 5251 Register zsave = tmp10; 5252 5253 mr(zsave, z); 5254 5255 5256 Label L_last_x; 5257 5258 sldi(tmp, xstart, LogBytesPerInt); 5259 add(z, z, tmp); // z = z + k - j 5260 addi(z, z, 4); 5261 addic_(xstart, xstart, -1); // i = xstart-1; 5262 blt(CCR0, L_last_x); 5263 5264 sldi(tmp, xstart, LogBytesPerInt); 5265 ldx(x_xstart, x, tmp); 5266 #ifdef VM_LITTLE_ENDIAN 5267 rldicl(x_xstart, x_xstart, 32, 0); 5268 #endif 5269 5270 5271 Label L_third_loop_prologue; 5272 5273 bind(L_third_loop_prologue); 5274 5275 Register xsave = tmp11; 5276 Register xlensave = tmp12; 5277 Register ylensave = tmp13; 5278 5279 mr(xsave, x); 5280 mr(xlensave, xstart); 5281 mr(ylensave, ylen); 5282 5283 5284 multiply_128_x_128_loop(x_xstart, y, z, y_idx, ylen, 5285 carry, product_high, product, x, tmp); 5286 5287 mr(z, zsave); 5288 mr(x, xsave); 5289 mr(xlen, xlensave); // This is the decrement of the loop counter! 5290 mr(ylen, ylensave); 5291 5292 addi(tmp3, xlen, 1); 5293 sldi(tmp, tmp3, LogBytesPerInt); 5294 stwx(carry, z, tmp); 5295 addic_(tmp3, tmp3, -1); 5296 blt(CCR0, L_done); 5297 5298 srdi(carry, carry, 32); 5299 sldi(tmp, tmp3, LogBytesPerInt); 5300 stwx(carry, z, tmp); 5301 b(L_second_loop); 5302 5303 // Next infrequent code is moved outside loops. 5304 bind(L_last_x); 5305 5306 lwz(x_xstart, 0, x); 5307 b(L_third_loop_prologue); 5308 5309 bind(L_done); 5310 } // multiply_to_len 5311 5312 void MacroAssembler::asm_assert(bool check_equal, const char *msg, int id) { 5313 #ifdef ASSERT 5314 Label ok; 5315 if (check_equal) { 5316 beq(CCR0, ok); 5317 } else { 5318 bne(CCR0, ok); 5319 } 5320 stop(msg, id); 5321 bind(ok); 5322 #endif 5323 } 5324 5325 void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset, 5326 Register mem_base, const char* msg, int id) { 5327 #ifdef ASSERT 5328 switch (size) { 5329 case 4: 5330 lwz(R0, mem_offset, mem_base); 5331 cmpwi(CCR0, R0, 0); 5332 break; 5333 case 8: 5334 ld(R0, mem_offset, mem_base); 5335 cmpdi(CCR0, R0, 0); 5336 break; 5337 default: 5338 ShouldNotReachHere(); 5339 } 5340 asm_assert(check_equal, msg, id); 5341 #endif // ASSERT 5342 } 5343 5344 void MacroAssembler::verify_thread() { 5345 if (VerifyThread) { 5346 unimplemented("'VerifyThread' currently not implemented on PPC"); 5347 } 5348 } 5349 5350 // READ: oop. KILL: R0. Volatile floats perhaps. 5351 void MacroAssembler::verify_oop(Register oop, const char* msg) { 5352 if (!VerifyOops) { 5353 return; 5354 } 5355 5356 address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address(); 5357 const Register tmp = R11; // Will be preserved. 5358 const int nbytes_save = MacroAssembler::num_volatile_regs * 8; 5359 save_volatile_gprs(R1_SP, -nbytes_save); // except R0 5360 5361 mr_if_needed(R4_ARG2, oop); 5362 save_LR_CR(tmp); // save in old frame 5363 push_frame_reg_args(nbytes_save, tmp); 5364 // load FunctionDescriptor** / entry_address * 5365 load_const_optimized(tmp, fd, R0); 5366 // load FunctionDescriptor* / entry_address 5367 ld(tmp, 0, tmp); 5368 load_const_optimized(R3_ARG1, (address)msg, R0); 5369 // Call destination for its side effect. 5370 call_c(tmp); 5371 5372 pop_frame(); 5373 restore_LR_CR(tmp); 5374 restore_volatile_gprs(R1_SP, -nbytes_save); // except R0 5375 } 5376 5377 void MacroAssembler::verify_oop_addr(RegisterOrConstant offs, Register base, const char* msg) { 5378 if (!VerifyOops) { 5379 return; 5380 } 5381 5382 address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address(); 5383 const Register tmp = R11; // Will be preserved. 5384 const int nbytes_save = MacroAssembler::num_volatile_regs * 8; 5385 save_volatile_gprs(R1_SP, -nbytes_save); // except R0 5386 5387 ld(R4_ARG2, offs, base); 5388 save_LR_CR(tmp); // save in old frame 5389 push_frame_reg_args(nbytes_save, tmp); 5390 // load FunctionDescriptor** / entry_address * 5391 load_const_optimized(tmp, fd, R0); 5392 // load FunctionDescriptor* / entry_address 5393 ld(tmp, 0, tmp); 5394 load_const_optimized(R3_ARG1, (address)msg, R0); 5395 // Call destination for its side effect. 5396 call_c(tmp); 5397 5398 pop_frame(); 5399 restore_LR_CR(tmp); 5400 restore_volatile_gprs(R1_SP, -nbytes_save); // except R0 5401 } 5402 5403 const char* stop_types[] = { 5404 "stop", 5405 "untested", 5406 "unimplemented", 5407 "shouldnotreachhere" 5408 }; 5409 5410 static void stop_on_request(int tp, const char* msg) { 5411 tty->print("PPC assembly code requires stop: (%s) %s\n", stop_types[tp%/*stop_end*/4], msg); 5412 guarantee(false, "PPC assembly code requires stop: %s", msg); 5413 } 5414 5415 // Call a C-function that prints output. 5416 void MacroAssembler::stop(int type, const char* msg, int id) { 5417 #ifndef PRODUCT 5418 block_comment(err_msg("stop: %s %s {", stop_types[type%stop_end], msg)); 5419 #else 5420 block_comment("stop {"); 5421 #endif 5422 5423 // setup arguments 5424 load_const_optimized(R3_ARG1, type); 5425 load_const_optimized(R4_ARG2, (void *)msg, /*tmp=*/R0); 5426 call_VM_leaf(CAST_FROM_FN_PTR(address, stop_on_request), R3_ARG1, R4_ARG2); 5427 illtrap(); 5428 emit_int32(id); 5429 block_comment("} stop;"); 5430 } 5431 5432 #ifndef PRODUCT 5433 // Write pattern 0x0101010101010101 in memory region [low-before, high+after]. 5434 // Val, addr are temp registers. 5435 // If low == addr, addr is killed. 5436 // High is preserved. 5437 void MacroAssembler::zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) { 5438 if (!ZapMemory) return; 5439 5440 assert_different_registers(low, val); 5441 5442 BLOCK_COMMENT("zap memory region {"); 5443 load_const_optimized(val, 0x0101010101010101); 5444 int size = before + after; 5445 if (low == high && size < 5 && size > 0) { 5446 int offset = -before*BytesPerWord; 5447 for (int i = 0; i < size; ++i) { 5448 std(val, offset, low); 5449 offset += (1*BytesPerWord); 5450 } 5451 } else { 5452 addi(addr, low, -before*BytesPerWord); 5453 assert_different_registers(high, val); 5454 if (after) addi(high, high, after * BytesPerWord); 5455 Label loop; 5456 bind(loop); 5457 std(val, 0, addr); 5458 addi(addr, addr, 8); 5459 cmpd(CCR6, addr, high); 5460 ble(CCR6, loop); 5461 if (after) addi(high, high, -after * BytesPerWord); // Correct back to old value. 5462 } 5463 BLOCK_COMMENT("} zap memory region"); 5464 } 5465 5466 #endif // !PRODUCT 5467 5468 SkipIfEqualZero::SkipIfEqualZero(MacroAssembler* masm, Register temp, const bool* flag_addr) : _masm(masm), _label() { 5469 int simm16_offset = masm->load_const_optimized(temp, (address)flag_addr, R0, true); 5470 assert(sizeof(bool) == 1, "PowerPC ABI"); 5471 masm->lbz(temp, simm16_offset, temp); 5472 masm->cmpwi(CCR0, temp, 0); 5473 masm->beq(CCR0, _label); 5474 } 5475 5476 SkipIfEqualZero::~SkipIfEqualZero() { 5477 _masm->bind(_label); 5478 }