1 /*
2 * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2012, 2017, SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "compiler/disassembler.hpp"
29 #include "gc/shared/cardTableModRefBS.hpp"
30 #include "gc/shared/collectedHeap.inline.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "memory/resourceArea.hpp"
33 #include "nativeInst_ppc.hpp"
34 #include "prims/methodHandles.hpp"
35 #include "runtime/biasedLocking.hpp"
36 #include "runtime/icache.hpp"
37 #include "runtime/interfaceSupport.hpp"
38 #include "runtime/objectMonitor.hpp"
39 #include "runtime/os.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/stubRoutines.hpp"
42 #include "utilities/macros.hpp"
43 #if INCLUDE_ALL_GCS
44 #include "gc/g1/g1CollectedHeap.inline.hpp"
45 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
46 #include "gc/g1/heapRegion.hpp"
47 #endif // INCLUDE_ALL_GCS
48 #ifdef COMPILER2
49 #include "opto/intrinsicnode.hpp"
50 #endif
51
52 #ifdef PRODUCT
53 #define BLOCK_COMMENT(str) // nothing
54 #else
55 #define BLOCK_COMMENT(str) block_comment(str)
56 #endif
57 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
58
59 #ifdef ASSERT
60 // On RISC, there's no benefit to verifying instruction boundaries.
61 bool AbstractAssembler::pd_check_instruction_mark() { return false; }
62 #endif
63
64 void MacroAssembler::ld_largeoffset_unchecked(Register d, int si31, Register a, int emit_filler_nop) {
65 assert(Assembler::is_simm(si31, 31) && si31 >= 0, "si31 out of range");
66 if (Assembler::is_simm(si31, 16)) {
67 ld(d, si31, a);
68 if (emit_filler_nop) nop();
69 } else {
70 const int hi = MacroAssembler::largeoffset_si16_si16_hi(si31);
71 const int lo = MacroAssembler::largeoffset_si16_si16_lo(si31);
72 addis(d, a, hi);
73 ld(d, lo, d);
74 }
75 }
76
77 void MacroAssembler::ld_largeoffset(Register d, int si31, Register a, int emit_filler_nop) {
78 assert_different_registers(d, a);
79 ld_largeoffset_unchecked(d, si31, a, emit_filler_nop);
80 }
81
82 void MacroAssembler::load_sized_value(Register dst, RegisterOrConstant offs, Register base,
83 size_t size_in_bytes, bool is_signed) {
84 switch (size_in_bytes) {
85 case 8: ld(dst, offs, base); break;
86 case 4: is_signed ? lwa(dst, offs, base) : lwz(dst, offs, base); break;
87 case 2: is_signed ? lha(dst, offs, base) : lhz(dst, offs, base); break;
88 case 1: lbz(dst, offs, base); if (is_signed) extsb(dst, dst); break; // lba doesn't exist :(
89 default: ShouldNotReachHere();
90 }
91 }
92
93 void MacroAssembler::store_sized_value(Register dst, RegisterOrConstant offs, Register base,
94 size_t size_in_bytes) {
95 switch (size_in_bytes) {
96 case 8: std(dst, offs, base); break;
97 case 4: stw(dst, offs, base); break;
98 case 2: sth(dst, offs, base); break;
99 case 1: stb(dst, offs, base); break;
100 default: ShouldNotReachHere();
101 }
102 }
103
104 void MacroAssembler::align(int modulus, int max, int rem) {
105 int padding = (rem + modulus - (offset() % modulus)) % modulus;
106 if (padding > max) return;
107 for (int c = (padding >> 2); c > 0; --c) { nop(); }
108 }
109
110 // Issue instructions that calculate given TOC from global TOC.
111 void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16,
112 bool add_relocation, bool emit_dummy_addr) {
113 int offset = -1;
114 if (emit_dummy_addr) {
115 offset = -128; // dummy address
116 } else if (addr != (address)(intptr_t)-1) {
117 offset = MacroAssembler::offset_to_global_toc(addr);
118 }
119
120 if (hi16) {
121 addis(dst, R29_TOC, MacroAssembler::largeoffset_si16_si16_hi(offset));
122 }
123 if (lo16) {
124 if (add_relocation) {
125 // Relocate at the addi to avoid confusion with a load from the method's TOC.
126 relocate(internal_word_Relocation::spec(addr));
127 }
128 addi(dst, dst, MacroAssembler::largeoffset_si16_si16_lo(offset));
129 }
130 }
131
132 int MacroAssembler::patch_calculate_address_from_global_toc_at(address a, address bound, address addr) {
133 const int offset = MacroAssembler::offset_to_global_toc(addr);
134
135 const address inst2_addr = a;
136 const int inst2 = *(int *)inst2_addr;
137
138 // The relocation points to the second instruction, the addi,
139 // and the addi reads and writes the same register dst.
140 const int dst = inv_rt_field(inst2);
141 assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
142
143 // Now, find the preceding addis which writes to dst.
144 int inst1 = 0;
145 address inst1_addr = inst2_addr - BytesPerInstWord;
146 while (inst1_addr >= bound) {
147 inst1 = *(int *) inst1_addr;
148 if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
149 // Stop, found the addis which writes dst.
150 break;
151 }
152 inst1_addr -= BytesPerInstWord;
153 }
154
155 assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
156 set_imm((int *)inst1_addr, MacroAssembler::largeoffset_si16_si16_hi(offset));
157 set_imm((int *)inst2_addr, MacroAssembler::largeoffset_si16_si16_lo(offset));
158 return (int)((intptr_t)addr - (intptr_t)inst1_addr);
159 }
160
161 address MacroAssembler::get_address_of_calculate_address_from_global_toc_at(address a, address bound) {
162 const address inst2_addr = a;
163 const int inst2 = *(int *)inst2_addr;
164
165 // The relocation points to the second instruction, the addi,
166 // and the addi reads and writes the same register dst.
167 const int dst = inv_rt_field(inst2);
168 assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
169
170 // Now, find the preceding addis which writes to dst.
171 int inst1 = 0;
172 address inst1_addr = inst2_addr - BytesPerInstWord;
173 while (inst1_addr >= bound) {
174 inst1 = *(int *) inst1_addr;
175 if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
176 // stop, found the addis which writes dst
177 break;
178 }
179 inst1_addr -= BytesPerInstWord;
180 }
181
182 assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
183
184 int offset = (get_imm(inst1_addr, 0) << 16) + get_imm(inst2_addr, 0);
185 // -1 is a special case
186 if (offset == -1) {
187 return (address)(intptr_t)-1;
188 } else {
189 return global_toc() + offset;
190 }
191 }
192
193 #ifdef _LP64
194 // Patch compressed oops or klass constants.
195 // Assembler sequence is
196 // 1) compressed oops:
197 // lis rx = const.hi
198 // ori rx = rx | const.lo
199 // 2) compressed klass:
200 // lis rx = const.hi
201 // clrldi rx = rx & 0xFFFFffff // clearMS32b, optional
202 // ori rx = rx | const.lo
203 // Clrldi will be passed by.
204 int MacroAssembler::patch_set_narrow_oop(address a, address bound, narrowOop data) {
205 assert(UseCompressedOops, "Should only patch compressed oops");
206
207 const address inst2_addr = a;
208 const int inst2 = *(int *)inst2_addr;
209
210 // The relocation points to the second instruction, the ori,
211 // and the ori reads and writes the same register dst.
212 const int dst = inv_rta_field(inst2);
213 assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
214 // Now, find the preceding addis which writes to dst.
215 int inst1 = 0;
216 address inst1_addr = inst2_addr - BytesPerInstWord;
217 bool inst1_found = false;
218 while (inst1_addr >= bound) {
219 inst1 = *(int *)inst1_addr;
220 if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break; }
221 inst1_addr -= BytesPerInstWord;
222 }
223 assert(inst1_found, "inst is not lis");
224
225 int xc = (data >> 16) & 0xffff;
226 int xd = (data >> 0) & 0xffff;
227
228 set_imm((int *)inst1_addr, (short)(xc)); // see enc_load_con_narrow_hi/_lo
229 set_imm((int *)inst2_addr, (xd)); // unsigned int
230 return (int)((intptr_t)inst2_addr - (intptr_t)inst1_addr);
231 }
232
233 // Get compressed oop or klass constant.
234 narrowOop MacroAssembler::get_narrow_oop(address a, address bound) {
235 assert(UseCompressedOops, "Should only patch compressed oops");
236
237 const address inst2_addr = a;
238 const int inst2 = *(int *)inst2_addr;
239
240 // The relocation points to the second instruction, the ori,
241 // and the ori reads and writes the same register dst.
242 const int dst = inv_rta_field(inst2);
243 assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
244 // Now, find the preceding lis which writes to dst.
245 int inst1 = 0;
246 address inst1_addr = inst2_addr - BytesPerInstWord;
247 bool inst1_found = false;
248
249 while (inst1_addr >= bound) {
250 inst1 = *(int *) inst1_addr;
251 if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break;}
252 inst1_addr -= BytesPerInstWord;
253 }
254 assert(inst1_found, "inst is not lis");
255
256 uint xl = ((unsigned int) (get_imm(inst2_addr, 0) & 0xffff));
257 uint xh = (((get_imm(inst1_addr, 0)) & 0xffff) << 16);
258
259 return (int) (xl | xh);
260 }
261 #endif // _LP64
262
263 // Returns true if successful.
264 bool MacroAssembler::load_const_from_method_toc(Register dst, AddressLiteral& a,
265 Register toc, bool fixed_size) {
266 int toc_offset = 0;
267 // Use RelocationHolder::none for the constant pool entry, otherwise
268 // we will end up with a failing NativeCall::verify(x) where x is
269 // the address of the constant pool entry.
270 // FIXME: We should insert relocation information for oops at the constant
271 // pool entries instead of inserting it at the loads; patching of a constant
272 // pool entry should be less expensive.
273 address const_address = address_constant((address)a.value(), RelocationHolder::none);
274 if (const_address == NULL) { return false; } // allocation failure
275 // Relocate at the pc of the load.
276 relocate(a.rspec());
277 toc_offset = (int)(const_address - code()->consts()->start());
278 ld_largeoffset_unchecked(dst, toc_offset, toc, fixed_size);
279 return true;
280 }
281
282 bool MacroAssembler::is_load_const_from_method_toc_at(address a) {
283 const address inst1_addr = a;
284 const int inst1 = *(int *)inst1_addr;
285
286 // The relocation points to the ld or the addis.
287 return (is_ld(inst1)) ||
288 (is_addis(inst1) && inv_ra_field(inst1) != 0);
289 }
290
291 int MacroAssembler::get_offset_of_load_const_from_method_toc_at(address a) {
292 assert(is_load_const_from_method_toc_at(a), "must be load_const_from_method_toc");
293
294 const address inst1_addr = a;
295 const int inst1 = *(int *)inst1_addr;
296
297 if (is_ld(inst1)) {
298 return inv_d1_field(inst1);
299 } else if (is_addis(inst1)) {
300 const int dst = inv_rt_field(inst1);
301
302 // Now, find the succeeding ld which reads and writes to dst.
303 address inst2_addr = inst1_addr + BytesPerInstWord;
304 int inst2 = 0;
305 while (true) {
306 inst2 = *(int *) inst2_addr;
307 if (is_ld(inst2) && inv_ra_field(inst2) == dst && inv_rt_field(inst2) == dst) {
308 // Stop, found the ld which reads and writes dst.
309 break;
310 }
311 inst2_addr += BytesPerInstWord;
312 }
313 return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2);
314 }
315 ShouldNotReachHere();
316 return 0;
317 }
318
319 // Get the constant from a `load_const' sequence.
320 long MacroAssembler::get_const(address a) {
321 assert(is_load_const_at(a), "not a load of a constant");
322 const int *p = (const int*) a;
323 unsigned long x = (((unsigned long) (get_imm(a,0) & 0xffff)) << 48);
324 if (is_ori(*(p+1))) {
325 x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 32);
326 x |= (((unsigned long) (get_imm(a,3) & 0xffff)) << 16);
327 x |= (((unsigned long) (get_imm(a,4) & 0xffff)));
328 } else if (is_lis(*(p+1))) {
329 x |= (((unsigned long) (get_imm(a,2) & 0xffff)) << 32);
330 x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 16);
331 x |= (((unsigned long) (get_imm(a,3) & 0xffff)));
332 } else {
333 ShouldNotReachHere();
334 return (long) 0;
335 }
336 return (long) x;
337 }
338
339 // Patch the 64 bit constant of a `load_const' sequence. This is a low
340 // level procedure. It neither flushes the instruction cache nor is it
341 // mt safe.
342 void MacroAssembler::patch_const(address a, long x) {
343 assert(is_load_const_at(a), "not a load of a constant");
344 int *p = (int*) a;
345 if (is_ori(*(p+1))) {
346 set_imm(0 + p, (x >> 48) & 0xffff);
347 set_imm(1 + p, (x >> 32) & 0xffff);
348 set_imm(3 + p, (x >> 16) & 0xffff);
349 set_imm(4 + p, x & 0xffff);
350 } else if (is_lis(*(p+1))) {
351 set_imm(0 + p, (x >> 48) & 0xffff);
352 set_imm(2 + p, (x >> 32) & 0xffff);
353 set_imm(1 + p, (x >> 16) & 0xffff);
354 set_imm(3 + p, x & 0xffff);
355 } else {
356 ShouldNotReachHere();
357 }
358 }
359
360 AddressLiteral MacroAssembler::allocate_metadata_address(Metadata* obj) {
361 assert(oop_recorder() != NULL, "this assembler needs a Recorder");
362 int index = oop_recorder()->allocate_metadata_index(obj);
363 RelocationHolder rspec = metadata_Relocation::spec(index);
364 return AddressLiteral((address)obj, rspec);
365 }
366
367 AddressLiteral MacroAssembler::constant_metadata_address(Metadata* obj) {
368 assert(oop_recorder() != NULL, "this assembler needs a Recorder");
369 int index = oop_recorder()->find_index(obj);
370 RelocationHolder rspec = metadata_Relocation::spec(index);
371 return AddressLiteral((address)obj, rspec);
372 }
373
374 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) {
375 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
376 int oop_index = oop_recorder()->allocate_oop_index(obj);
377 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
378 }
379
380 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) {
381 assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
382 int oop_index = oop_recorder()->find_index(obj);
383 return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
384 }
385
386 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
387 Register tmp, int offset) {
388 intptr_t value = *delayed_value_addr;
389 if (value != 0) {
390 return RegisterOrConstant(value + offset);
391 }
392
393 // Load indirectly to solve generation ordering problem.
394 // static address, no relocation
395 int simm16_offset = load_const_optimized(tmp, delayed_value_addr, noreg, true);
396 ld(tmp, simm16_offset, tmp); // must be aligned ((xa & 3) == 0)
397
398 if (offset != 0) {
399 addi(tmp, tmp, offset);
400 }
401
402 return RegisterOrConstant(tmp);
403 }
404
405 #ifndef PRODUCT
406 void MacroAssembler::pd_print_patched_instruction(address branch) {
407 Unimplemented(); // TODO: PPC port
408 }
409 #endif // ndef PRODUCT
410
411 // Conditional far branch for destinations encodable in 24+2 bits.
412 void MacroAssembler::bc_far(int boint, int biint, Label& dest, int optimize) {
413
414 // If requested by flag optimize, relocate the bc_far as a
415 // runtime_call and prepare for optimizing it when the code gets
416 // relocated.
417 if (optimize == bc_far_optimize_on_relocate) {
418 relocate(relocInfo::runtime_call_type);
419 }
420
421 // variant 2:
422 //
423 // b!cxx SKIP
424 // bxx DEST
425 // SKIP:
426 //
427
428 const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
429 opposite_bcond(inv_boint_bcond(boint)));
430
431 // We emit two branches.
432 // First, a conditional branch which jumps around the far branch.
433 const address not_taken_pc = pc() + 2 * BytesPerInstWord;
434 const address bc_pc = pc();
435 bc(opposite_boint, biint, not_taken_pc);
436
437 const int bc_instr = *(int*)bc_pc;
438 assert(not_taken_pc == (address)inv_bd_field(bc_instr, (intptr_t)bc_pc), "postcondition");
439 assert(opposite_boint == inv_bo_field(bc_instr), "postcondition");
440 assert(boint == add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(bc_instr))),
441 opposite_bcond(inv_boint_bcond(inv_bo_field(bc_instr)))),
442 "postcondition");
443 assert(biint == inv_bi_field(bc_instr), "postcondition");
444
445 // Second, an unconditional far branch which jumps to dest.
446 // Note: target(dest) remembers the current pc (see CodeSection::target)
447 // and returns the current pc if the label is not bound yet; when
448 // the label gets bound, the unconditional far branch will be patched.
449 const address target_pc = target(dest);
450 const address b_pc = pc();
451 b(target_pc);
452
453 assert(not_taken_pc == pc(), "postcondition");
454 assert(dest.is_bound() || target_pc == b_pc, "postcondition");
455 }
456
457 // 1 or 2 instructions
458 void MacroAssembler::bc_far_optimized(int boint, int biint, Label& dest) {
459 if (dest.is_bound() && is_within_range_of_bcxx(target(dest), pc())) {
460 bc(boint, biint, dest);
461 } else {
462 bc_far(boint, biint, dest, MacroAssembler::bc_far_optimize_on_relocate);
463 }
464 }
465
466 bool MacroAssembler::is_bc_far_at(address instruction_addr) {
467 return is_bc_far_variant1_at(instruction_addr) ||
468 is_bc_far_variant2_at(instruction_addr) ||
469 is_bc_far_variant3_at(instruction_addr);
470 }
471
472 address MacroAssembler::get_dest_of_bc_far_at(address instruction_addr) {
473 if (is_bc_far_variant1_at(instruction_addr)) {
474 const address instruction_1_addr = instruction_addr;
475 const int instruction_1 = *(int*)instruction_1_addr;
476 return (address)inv_bd_field(instruction_1, (intptr_t)instruction_1_addr);
477 } else if (is_bc_far_variant2_at(instruction_addr)) {
478 const address instruction_2_addr = instruction_addr + 4;
479 return bxx_destination(instruction_2_addr);
480 } else if (is_bc_far_variant3_at(instruction_addr)) {
481 return instruction_addr + 8;
482 }
483 // variant 4 ???
484 ShouldNotReachHere();
485 return NULL;
486 }
487 void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address dest) {
488
489 if (is_bc_far_variant3_at(instruction_addr)) {
490 // variant 3, far cond branch to the next instruction, already patched to nops:
491 //
492 // nop
493 // endgroup
494 // SKIP/DEST:
495 //
496 return;
497 }
498
499 // first, extract boint and biint from the current branch
500 int boint = 0;
501 int biint = 0;
502
503 ResourceMark rm;
504 const int code_size = 2 * BytesPerInstWord;
505 CodeBuffer buf(instruction_addr, code_size);
506 MacroAssembler masm(&buf);
507 if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) {
508 // Far branch to next instruction: Optimize it by patching nops (produce variant 3).
509 masm.nop();
510 masm.endgroup();
511 } else {
512 if (is_bc_far_variant1_at(instruction_addr)) {
513 // variant 1, the 1st instruction contains the destination address:
514 //
515 // bcxx DEST
516 // nop
517 //
518 const int instruction_1 = *(int*)(instruction_addr);
519 boint = inv_bo_field(instruction_1);
520 biint = inv_bi_field(instruction_1);
521 } else if (is_bc_far_variant2_at(instruction_addr)) {
522 // variant 2, the 2nd instruction contains the destination address:
523 //
524 // b!cxx SKIP
525 // bxx DEST
526 // SKIP:
527 //
528 const int instruction_1 = *(int*)(instruction_addr);
529 boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(instruction_1))),
530 opposite_bcond(inv_boint_bcond(inv_bo_field(instruction_1))));
531 biint = inv_bi_field(instruction_1);
532 } else {
533 // variant 4???
534 ShouldNotReachHere();
535 }
536
537 // second, set the new branch destination and optimize the code
538 if (dest != instruction_addr + 4 && // the bc_far is still unbound!
539 masm.is_within_range_of_bcxx(dest, instruction_addr)) {
540 // variant 1:
541 //
542 // bcxx DEST
543 // nop
544 //
545 masm.bc(boint, biint, dest);
546 masm.nop();
547 } else {
548 // variant 2:
549 //
550 // b!cxx SKIP
551 // bxx DEST
552 // SKIP:
553 //
554 const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
555 opposite_bcond(inv_boint_bcond(boint)));
556 const address not_taken_pc = masm.pc() + 2 * BytesPerInstWord;
557 masm.bc(opposite_boint, biint, not_taken_pc);
558 masm.b(dest);
559 }
560 }
561 ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
562 }
563
564 // Emit a NOT mt-safe patchable 64 bit absolute call/jump.
565 void MacroAssembler::bxx64_patchable(address dest, relocInfo::relocType rt, bool link) {
566 // get current pc
567 uint64_t start_pc = (uint64_t) pc();
568
569 const address pc_of_bl = (address) (start_pc + (6*BytesPerInstWord)); // bl is last
570 const address pc_of_b = (address) (start_pc + (0*BytesPerInstWord)); // b is first
571
572 // relocate here
573 if (rt != relocInfo::none) {
574 relocate(rt);
575 }
576
577 if ( ReoptimizeCallSequences &&
578 (( link && is_within_range_of_b(dest, pc_of_bl)) ||
579 (!link && is_within_range_of_b(dest, pc_of_b)))) {
580 // variant 2:
581 // Emit an optimized, pc-relative call/jump.
582
583 if (link) {
584 // some padding
585 nop();
586 nop();
587 nop();
588 nop();
589 nop();
590 nop();
591
592 // do the call
593 assert(pc() == pc_of_bl, "just checking");
594 bl(dest, relocInfo::none);
595 } else {
596 // do the jump
597 assert(pc() == pc_of_b, "just checking");
598 b(dest, relocInfo::none);
599
600 // some padding
601 nop();
602 nop();
603 nop();
604 nop();
605 nop();
606 nop();
607 }
608
609 // Assert that we can identify the emitted call/jump.
610 assert(is_bxx64_patchable_variant2_at((address)start_pc, link),
611 "can't identify emitted call");
612 } else {
613 // variant 1:
614 mr(R0, R11); // spill R11 -> R0.
615
616 // Load the destination address into CTR,
617 // calculate destination relative to global toc.
618 calculate_address_from_global_toc(R11, dest, true, true, false);
619
620 mtctr(R11);
621 mr(R11, R0); // spill R11 <- R0.
622 nop();
623
624 // do the call/jump
625 if (link) {
626 bctrl();
627 } else{
628 bctr();
629 }
630 // Assert that we can identify the emitted call/jump.
631 assert(is_bxx64_patchable_variant1b_at((address)start_pc, link),
632 "can't identify emitted call");
633 }
634
635 // Assert that we can identify the emitted call/jump.
636 assert(is_bxx64_patchable_at((address)start_pc, link),
637 "can't identify emitted call");
638 assert(get_dest_of_bxx64_patchable_at((address)start_pc, link) == dest,
639 "wrong encoding of dest address");
640 }
641
642 // Identify a bxx64_patchable instruction.
643 bool MacroAssembler::is_bxx64_patchable_at(address instruction_addr, bool link) {
644 return is_bxx64_patchable_variant1b_at(instruction_addr, link)
645 //|| is_bxx64_patchable_variant1_at(instruction_addr, link)
646 || is_bxx64_patchable_variant2_at(instruction_addr, link);
647 }
648
649 // Does the call64_patchable instruction use a pc-relative encoding of
650 // the call destination?
651 bool MacroAssembler::is_bxx64_patchable_pcrelative_at(address instruction_addr, bool link) {
652 // variant 2 is pc-relative
653 return is_bxx64_patchable_variant2_at(instruction_addr, link);
654 }
655
656 // Identify variant 1.
657 bool MacroAssembler::is_bxx64_patchable_variant1_at(address instruction_addr, bool link) {
658 unsigned int* instr = (unsigned int*) instruction_addr;
659 return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
660 && is_mtctr(instr[5]) // mtctr
661 && is_load_const_at(instruction_addr);
662 }
663
664 // Identify variant 1b: load destination relative to global toc.
665 bool MacroAssembler::is_bxx64_patchable_variant1b_at(address instruction_addr, bool link) {
666 unsigned int* instr = (unsigned int*) instruction_addr;
667 return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
668 && is_mtctr(instr[3]) // mtctr
669 && is_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord, instruction_addr);
670 }
671
672 // Identify variant 2.
673 bool MacroAssembler::is_bxx64_patchable_variant2_at(address instruction_addr, bool link) {
674 unsigned int* instr = (unsigned int*) instruction_addr;
675 if (link) {
676 return is_bl (instr[6]) // bl dest is last
677 && is_nop(instr[0]) // nop
678 && is_nop(instr[1]) // nop
679 && is_nop(instr[2]) // nop
680 && is_nop(instr[3]) // nop
681 && is_nop(instr[4]) // nop
682 && is_nop(instr[5]); // nop
683 } else {
684 return is_b (instr[0]) // b dest is first
685 && is_nop(instr[1]) // nop
686 && is_nop(instr[2]) // nop
687 && is_nop(instr[3]) // nop
688 && is_nop(instr[4]) // nop
689 && is_nop(instr[5]) // nop
690 && is_nop(instr[6]); // nop
691 }
692 }
693
694 // Set dest address of a bxx64_patchable instruction.
695 void MacroAssembler::set_dest_of_bxx64_patchable_at(address instruction_addr, address dest, bool link) {
696 ResourceMark rm;
697 int code_size = MacroAssembler::bxx64_patchable_size;
698 CodeBuffer buf(instruction_addr, code_size);
699 MacroAssembler masm(&buf);
700 masm.bxx64_patchable(dest, relocInfo::none, link);
701 ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
702 }
703
704 // Get dest address of a bxx64_patchable instruction.
705 address MacroAssembler::get_dest_of_bxx64_patchable_at(address instruction_addr, bool link) {
706 if (is_bxx64_patchable_variant1_at(instruction_addr, link)) {
707 return (address) (unsigned long) get_const(instruction_addr);
708 } else if (is_bxx64_patchable_variant2_at(instruction_addr, link)) {
709 unsigned int* instr = (unsigned int*) instruction_addr;
710 if (link) {
711 const int instr_idx = 6; // bl is last
712 int branchoffset = branch_destination(instr[instr_idx], 0);
713 return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
714 } else {
715 const int instr_idx = 0; // b is first
716 int branchoffset = branch_destination(instr[instr_idx], 0);
717 return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
718 }
719 // Load dest relative to global toc.
720 } else if (is_bxx64_patchable_variant1b_at(instruction_addr, link)) {
721 return get_address_of_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord,
722 instruction_addr);
723 } else {
724 ShouldNotReachHere();
725 return NULL;
726 }
727 }
728
729 // Uses ordering which corresponds to ABI:
730 // _savegpr0_14: std r14,-144(r1)
731 // _savegpr0_15: std r15,-136(r1)
732 // _savegpr0_16: std r16,-128(r1)
733 void MacroAssembler::save_nonvolatile_gprs(Register dst, int offset) {
734 std(R14, offset, dst); offset += 8;
735 std(R15, offset, dst); offset += 8;
736 std(R16, offset, dst); offset += 8;
737 std(R17, offset, dst); offset += 8;
738 std(R18, offset, dst); offset += 8;
739 std(R19, offset, dst); offset += 8;
740 std(R20, offset, dst); offset += 8;
741 std(R21, offset, dst); offset += 8;
742 std(R22, offset, dst); offset += 8;
743 std(R23, offset, dst); offset += 8;
744 std(R24, offset, dst); offset += 8;
745 std(R25, offset, dst); offset += 8;
746 std(R26, offset, dst); offset += 8;
747 std(R27, offset, dst); offset += 8;
748 std(R28, offset, dst); offset += 8;
749 std(R29, offset, dst); offset += 8;
750 std(R30, offset, dst); offset += 8;
751 std(R31, offset, dst); offset += 8;
752
753 stfd(F14, offset, dst); offset += 8;
754 stfd(F15, offset, dst); offset += 8;
755 stfd(F16, offset, dst); offset += 8;
756 stfd(F17, offset, dst); offset += 8;
757 stfd(F18, offset, dst); offset += 8;
758 stfd(F19, offset, dst); offset += 8;
759 stfd(F20, offset, dst); offset += 8;
760 stfd(F21, offset, dst); offset += 8;
761 stfd(F22, offset, dst); offset += 8;
762 stfd(F23, offset, dst); offset += 8;
763 stfd(F24, offset, dst); offset += 8;
764 stfd(F25, offset, dst); offset += 8;
765 stfd(F26, offset, dst); offset += 8;
766 stfd(F27, offset, dst); offset += 8;
767 stfd(F28, offset, dst); offset += 8;
768 stfd(F29, offset, dst); offset += 8;
769 stfd(F30, offset, dst); offset += 8;
770 stfd(F31, offset, dst);
771 }
772
773 // Uses ordering which corresponds to ABI:
774 // _restgpr0_14: ld r14,-144(r1)
775 // _restgpr0_15: ld r15,-136(r1)
776 // _restgpr0_16: ld r16,-128(r1)
777 void MacroAssembler::restore_nonvolatile_gprs(Register src, int offset) {
778 ld(R14, offset, src); offset += 8;
779 ld(R15, offset, src); offset += 8;
780 ld(R16, offset, src); offset += 8;
781 ld(R17, offset, src); offset += 8;
782 ld(R18, offset, src); offset += 8;
783 ld(R19, offset, src); offset += 8;
784 ld(R20, offset, src); offset += 8;
785 ld(R21, offset, src); offset += 8;
786 ld(R22, offset, src); offset += 8;
787 ld(R23, offset, src); offset += 8;
788 ld(R24, offset, src); offset += 8;
789 ld(R25, offset, src); offset += 8;
790 ld(R26, offset, src); offset += 8;
791 ld(R27, offset, src); offset += 8;
792 ld(R28, offset, src); offset += 8;
793 ld(R29, offset, src); offset += 8;
794 ld(R30, offset, src); offset += 8;
795 ld(R31, offset, src); offset += 8;
796
797 // FP registers
798 lfd(F14, offset, src); offset += 8;
799 lfd(F15, offset, src); offset += 8;
800 lfd(F16, offset, src); offset += 8;
801 lfd(F17, offset, src); offset += 8;
802 lfd(F18, offset, src); offset += 8;
803 lfd(F19, offset, src); offset += 8;
804 lfd(F20, offset, src); offset += 8;
805 lfd(F21, offset, src); offset += 8;
806 lfd(F22, offset, src); offset += 8;
807 lfd(F23, offset, src); offset += 8;
808 lfd(F24, offset, src); offset += 8;
809 lfd(F25, offset, src); offset += 8;
810 lfd(F26, offset, src); offset += 8;
811 lfd(F27, offset, src); offset += 8;
812 lfd(F28, offset, src); offset += 8;
813 lfd(F29, offset, src); offset += 8;
814 lfd(F30, offset, src); offset += 8;
815 lfd(F31, offset, src);
816 }
817
818 // For verify_oops.
819 void MacroAssembler::save_volatile_gprs(Register dst, int offset) {
820 std(R2, offset, dst); offset += 8;
821 std(R3, offset, dst); offset += 8;
822 std(R4, offset, dst); offset += 8;
823 std(R5, offset, dst); offset += 8;
824 std(R6, offset, dst); offset += 8;
825 std(R7, offset, dst); offset += 8;
826 std(R8, offset, dst); offset += 8;
827 std(R9, offset, dst); offset += 8;
828 std(R10, offset, dst); offset += 8;
829 std(R11, offset, dst); offset += 8;
830 std(R12, offset, dst); offset += 8;
831
832 stfd(F0, offset, dst); offset += 8;
833 stfd(F1, offset, dst); offset += 8;
834 stfd(F2, offset, dst); offset += 8;
835 stfd(F3, offset, dst); offset += 8;
836 stfd(F4, offset, dst); offset += 8;
837 stfd(F5, offset, dst); offset += 8;
838 stfd(F6, offset, dst); offset += 8;
839 stfd(F7, offset, dst); offset += 8;
840 stfd(F8, offset, dst); offset += 8;
841 stfd(F9, offset, dst); offset += 8;
842 stfd(F10, offset, dst); offset += 8;
843 stfd(F11, offset, dst); offset += 8;
844 stfd(F12, offset, dst); offset += 8;
845 stfd(F13, offset, dst);
846 }
847
848 // For verify_oops.
849 void MacroAssembler::restore_volatile_gprs(Register src, int offset) {
850 ld(R2, offset, src); offset += 8;
851 ld(R3, offset, src); offset += 8;
852 ld(R4, offset, src); offset += 8;
853 ld(R5, offset, src); offset += 8;
854 ld(R6, offset, src); offset += 8;
855 ld(R7, offset, src); offset += 8;
856 ld(R8, offset, src); offset += 8;
857 ld(R9, offset, src); offset += 8;
858 ld(R10, offset, src); offset += 8;
859 ld(R11, offset, src); offset += 8;
860 ld(R12, offset, src); offset += 8;
861
862 lfd(F0, offset, src); offset += 8;
863 lfd(F1, offset, src); offset += 8;
864 lfd(F2, offset, src); offset += 8;
865 lfd(F3, offset, src); offset += 8;
866 lfd(F4, offset, src); offset += 8;
867 lfd(F5, offset, src); offset += 8;
868 lfd(F6, offset, src); offset += 8;
869 lfd(F7, offset, src); offset += 8;
870 lfd(F8, offset, src); offset += 8;
871 lfd(F9, offset, src); offset += 8;
872 lfd(F10, offset, src); offset += 8;
873 lfd(F11, offset, src); offset += 8;
874 lfd(F12, offset, src); offset += 8;
875 lfd(F13, offset, src);
876 }
877
878 void MacroAssembler::save_LR_CR(Register tmp) {
879 mfcr(tmp);
880 std(tmp, _abi(cr), R1_SP);
881 mflr(tmp);
882 std(tmp, _abi(lr), R1_SP);
883 // Tmp must contain lr on exit! (see return_addr and prolog in ppc64.ad)
884 }
885
886 void MacroAssembler::restore_LR_CR(Register tmp) {
887 assert(tmp != R1_SP, "must be distinct");
888 ld(tmp, _abi(lr), R1_SP);
889 mtlr(tmp);
890 ld(tmp, _abi(cr), R1_SP);
891 mtcr(tmp);
892 }
893
894 address MacroAssembler::get_PC_trash_LR(Register result) {
895 Label L;
896 bl(L);
897 bind(L);
898 address lr_pc = pc();
899 mflr(result);
900 return lr_pc;
901 }
902
903 void MacroAssembler::resize_frame(Register offset, Register tmp) {
904 #ifdef ASSERT
905 assert_different_registers(offset, tmp, R1_SP);
906 andi_(tmp, offset, frame::alignment_in_bytes-1);
907 asm_assert_eq("resize_frame: unaligned", 0x204);
908 #endif
909
910 // tmp <- *(SP)
911 ld(tmp, _abi(callers_sp), R1_SP);
912 // addr <- SP + offset;
913 // *(addr) <- tmp;
914 // SP <- addr
915 stdux(tmp, R1_SP, offset);
916 }
917
918 void MacroAssembler::resize_frame(int offset, Register tmp) {
919 assert(is_simm(offset, 16), "too big an offset");
920 assert_different_registers(tmp, R1_SP);
921 assert((offset & (frame::alignment_in_bytes-1))==0, "resize_frame: unaligned");
922 // tmp <- *(SP)
923 ld(tmp, _abi(callers_sp), R1_SP);
924 // addr <- SP + offset;
925 // *(addr) <- tmp;
926 // SP <- addr
927 stdu(tmp, offset, R1_SP);
928 }
929
930 void MacroAssembler::resize_frame_absolute(Register addr, Register tmp1, Register tmp2) {
931 // (addr == tmp1) || (addr == tmp2) is allowed here!
932 assert(tmp1 != tmp2, "must be distinct");
933
934 // compute offset w.r.t. current stack pointer
935 // tmp_1 <- addr - SP (!)
936 subf(tmp1, R1_SP, addr);
937
938 // atomically update SP keeping back link.
939 resize_frame(tmp1/* offset */, tmp2/* tmp */);
940 }
941
942 void MacroAssembler::push_frame(Register bytes, Register tmp) {
943 #ifdef ASSERT
944 assert(bytes != R0, "r0 not allowed here");
945 andi_(R0, bytes, frame::alignment_in_bytes-1);
946 asm_assert_eq("push_frame(Reg, Reg): unaligned", 0x203);
947 #endif
948 neg(tmp, bytes);
949 stdux(R1_SP, R1_SP, tmp);
950 }
951
952 // Push a frame of size `bytes'.
953 void MacroAssembler::push_frame(unsigned int bytes, Register tmp) {
954 long offset = align_addr(bytes, frame::alignment_in_bytes);
955 if (is_simm(-offset, 16)) {
956 stdu(R1_SP, -offset, R1_SP);
957 } else {
958 load_const_optimized(tmp, -offset);
959 stdux(R1_SP, R1_SP, tmp);
960 }
961 }
962
963 // Push a frame of size `bytes' plus abi_reg_args on top.
964 void MacroAssembler::push_frame_reg_args(unsigned int bytes, Register tmp) {
965 push_frame(bytes + frame::abi_reg_args_size, tmp);
966 }
967
968 // Setup up a new C frame with a spill area for non-volatile GPRs and
969 // additional space for local variables.
970 void MacroAssembler::push_frame_reg_args_nonvolatiles(unsigned int bytes,
971 Register tmp) {
972 push_frame(bytes + frame::abi_reg_args_size + frame::spill_nonvolatiles_size, tmp);
973 }
974
975 // Pop current C frame.
976 void MacroAssembler::pop_frame() {
977 ld(R1_SP, _abi(callers_sp), R1_SP);
978 }
979
980 #if defined(ABI_ELFv2)
981 address MacroAssembler::branch_to(Register r_function_entry, bool and_link) {
982 // TODO(asmundak): make sure the caller uses R12 as function descriptor
983 // most of the times.
984 if (R12 != r_function_entry) {
985 mr(R12, r_function_entry);
986 }
987 mtctr(R12);
988 // Do a call or a branch.
989 if (and_link) {
990 bctrl();
991 } else {
992 bctr();
993 }
994 _last_calls_return_pc = pc();
995
996 return _last_calls_return_pc;
997 }
998
999 // Call a C function via a function descriptor and use full C
1000 // calling conventions. Updates and returns _last_calls_return_pc.
1001 address MacroAssembler::call_c(Register r_function_entry) {
1002 return branch_to(r_function_entry, /*and_link=*/true);
1003 }
1004
1005 // For tail calls: only branch, don't link, so callee returns to caller of this function.
1006 address MacroAssembler::call_c_and_return_to_caller(Register r_function_entry) {
1007 return branch_to(r_function_entry, /*and_link=*/false);
1008 }
1009
1010 address MacroAssembler::call_c(address function_entry, relocInfo::relocType rt) {
1011 load_const(R12, function_entry, R0);
1012 return branch_to(R12, /*and_link=*/true);
1013 }
1014
1015 #else
1016 // Generic version of a call to C function via a function descriptor
1017 // with variable support for C calling conventions (TOC, ENV, etc.).
1018 // Updates and returns _last_calls_return_pc.
1019 address MacroAssembler::branch_to(Register function_descriptor, bool and_link, bool save_toc_before_call,
1020 bool restore_toc_after_call, bool load_toc_of_callee, bool load_env_of_callee) {
1021 // we emit standard ptrgl glue code here
1022 assert((function_descriptor != R0), "function_descriptor cannot be R0");
1023
1024 // retrieve necessary entries from the function descriptor
1025 ld(R0, in_bytes(FunctionDescriptor::entry_offset()), function_descriptor);
1026 mtctr(R0);
1027
1028 if (load_toc_of_callee) {
1029 ld(R2_TOC, in_bytes(FunctionDescriptor::toc_offset()), function_descriptor);
1030 }
1031 if (load_env_of_callee) {
1032 ld(R11, in_bytes(FunctionDescriptor::env_offset()), function_descriptor);
1033 } else if (load_toc_of_callee) {
1034 li(R11, 0);
1035 }
1036
1037 // do a call or a branch
1038 if (and_link) {
1039 bctrl();
1040 } else {
1041 bctr();
1042 }
1043 _last_calls_return_pc = pc();
1044
1045 return _last_calls_return_pc;
1046 }
1047
1048 // Call a C function via a function descriptor and use full C calling
1049 // conventions.
1050 // We don't use the TOC in generated code, so there is no need to save
1051 // and restore its value.
1052 address MacroAssembler::call_c(Register fd) {
1053 return branch_to(fd, /*and_link=*/true,
1054 /*save toc=*/false,
1055 /*restore toc=*/false,
1056 /*load toc=*/true,
1057 /*load env=*/true);
1058 }
1059
1060 address MacroAssembler::call_c_and_return_to_caller(Register fd) {
1061 return branch_to(fd, /*and_link=*/false,
1062 /*save toc=*/false,
1063 /*restore toc=*/false,
1064 /*load toc=*/true,
1065 /*load env=*/true);
1066 }
1067
1068 address MacroAssembler::call_c(const FunctionDescriptor* fd, relocInfo::relocType rt) {
1069 if (rt != relocInfo::none) {
1070 // this call needs to be relocatable
1071 if (!ReoptimizeCallSequences
1072 || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1073 || fd == NULL // support code-size estimation
1074 || !fd->is_friend_function()
1075 || fd->entry() == NULL) {
1076 // it's not a friend function as defined by class FunctionDescriptor,
1077 // so do a full call-c here.
1078 load_const(R11, (address)fd, R0);
1079
1080 bool has_env = (fd != NULL && fd->env() != NULL);
1081 return branch_to(R11, /*and_link=*/true,
1082 /*save toc=*/false,
1083 /*restore toc=*/false,
1084 /*load toc=*/true,
1085 /*load env=*/has_env);
1086 } else {
1087 // It's a friend function. Load the entry point and don't care about
1088 // toc and env. Use an optimizable call instruction, but ensure the
1089 // same code-size as in the case of a non-friend function.
1090 nop();
1091 nop();
1092 nop();
1093 bl64_patchable(fd->entry(), rt);
1094 _last_calls_return_pc = pc();
1095 return _last_calls_return_pc;
1096 }
1097 } else {
1098 // This call does not need to be relocatable, do more aggressive
1099 // optimizations.
1100 if (!ReoptimizeCallSequences
1101 || !fd->is_friend_function()) {
1102 // It's not a friend function as defined by class FunctionDescriptor,
1103 // so do a full call-c here.
1104 load_const(R11, (address)fd, R0);
1105 return branch_to(R11, /*and_link=*/true,
1106 /*save toc=*/false,
1107 /*restore toc=*/false,
1108 /*load toc=*/true,
1109 /*load env=*/true);
1110 } else {
1111 // it's a friend function, load the entry point and don't care about
1112 // toc and env.
1113 address dest = fd->entry();
1114 if (is_within_range_of_b(dest, pc())) {
1115 bl(dest);
1116 } else {
1117 bl64_patchable(dest, rt);
1118 }
1119 _last_calls_return_pc = pc();
1120 return _last_calls_return_pc;
1121 }
1122 }
1123 }
1124
1125 // Call a C function. All constants needed reside in TOC.
1126 //
1127 // Read the address to call from the TOC.
1128 // Read env from TOC, if fd specifies an env.
1129 // Read new TOC from TOC.
1130 address MacroAssembler::call_c_using_toc(const FunctionDescriptor* fd,
1131 relocInfo::relocType rt, Register toc) {
1132 if (!ReoptimizeCallSequences
1133 || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1134 || !fd->is_friend_function()) {
1135 // It's not a friend function as defined by class FunctionDescriptor,
1136 // so do a full call-c here.
1137 assert(fd->entry() != NULL, "function must be linked");
1138
1139 AddressLiteral fd_entry(fd->entry());
1140 bool success = load_const_from_method_toc(R11, fd_entry, toc, /*fixed_size*/ true);
1141 mtctr(R11);
1142 if (fd->env() == NULL) {
1143 li(R11, 0);
1144 nop();
1145 } else {
1146 AddressLiteral fd_env(fd->env());
1147 success = success && load_const_from_method_toc(R11, fd_env, toc, /*fixed_size*/ true);
1148 }
1149 AddressLiteral fd_toc(fd->toc());
1150 // Set R2_TOC (load from toc)
1151 success = success && load_const_from_method_toc(R2_TOC, fd_toc, toc, /*fixed_size*/ true);
1152 bctrl();
1153 _last_calls_return_pc = pc();
1154 if (!success) { return NULL; }
1155 } else {
1156 // It's a friend function, load the entry point and don't care about
1157 // toc and env. Use an optimizable call instruction, but ensure the
1158 // same code-size as in the case of a non-friend function.
1159 nop();
1160 bl64_patchable(fd->entry(), rt);
1161 _last_calls_return_pc = pc();
1162 }
1163 return _last_calls_return_pc;
1164 }
1165 #endif // ABI_ELFv2
1166
1167 void MacroAssembler::call_VM_base(Register oop_result,
1168 Register last_java_sp,
1169 address entry_point,
1170 bool check_exceptions) {
1171 BLOCK_COMMENT("call_VM {");
1172 // Determine last_java_sp register.
1173 if (!last_java_sp->is_valid()) {
1174 last_java_sp = R1_SP;
1175 }
1176 set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1);
1177
1178 // ARG1 must hold thread address.
1179 mr(R3_ARG1, R16_thread);
1180 #if defined(ABI_ELFv2)
1181 address return_pc = call_c(entry_point, relocInfo::none);
1182 #else
1183 address return_pc = call_c((FunctionDescriptor*)entry_point, relocInfo::none);
1184 #endif
1185
1186 reset_last_Java_frame();
1187
1188 // Check for pending exceptions.
1189 if (check_exceptions) {
1190 // We don't check for exceptions here.
1191 ShouldNotReachHere();
1192 }
1193
1194 // Get oop result if there is one and reset the value in the thread.
1195 if (oop_result->is_valid()) {
1196 get_vm_result(oop_result);
1197 }
1198
1199 _last_calls_return_pc = return_pc;
1200 BLOCK_COMMENT("} call_VM");
1201 }
1202
1203 void MacroAssembler::call_VM_leaf_base(address entry_point) {
1204 BLOCK_COMMENT("call_VM_leaf {");
1205 #if defined(ABI_ELFv2)
1206 call_c(entry_point, relocInfo::none);
1207 #else
1208 call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, entry_point), relocInfo::none);
1209 #endif
1210 BLOCK_COMMENT("} call_VM_leaf");
1211 }
1212
1213 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
1214 call_VM_base(oop_result, noreg, entry_point, check_exceptions);
1215 }
1216
1217 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1,
1218 bool check_exceptions) {
1219 // R3_ARG1 is reserved for the thread.
1220 mr_if_needed(R4_ARG2, arg_1);
1221 call_VM(oop_result, entry_point, check_exceptions);
1222 }
1223
1224 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2,
1225 bool check_exceptions) {
1226 // R3_ARG1 is reserved for the thread
1227 mr_if_needed(R4_ARG2, arg_1);
1228 assert(arg_2 != R4_ARG2, "smashed argument");
1229 mr_if_needed(R5_ARG3, arg_2);
1230 call_VM(oop_result, entry_point, check_exceptions);
1231 }
1232
1233 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3,
1234 bool check_exceptions) {
1235 // R3_ARG1 is reserved for the thread
1236 mr_if_needed(R4_ARG2, arg_1);
1237 assert(arg_2 != R4_ARG2, "smashed argument");
1238 mr_if_needed(R5_ARG3, arg_2);
1239 mr_if_needed(R6_ARG4, arg_3);
1240 call_VM(oop_result, entry_point, check_exceptions);
1241 }
1242
1243 void MacroAssembler::call_VM_leaf(address entry_point) {
1244 call_VM_leaf_base(entry_point);
1245 }
1246
1247 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
1248 mr_if_needed(R3_ARG1, arg_1);
1249 call_VM_leaf(entry_point);
1250 }
1251
1252 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
1253 mr_if_needed(R3_ARG1, arg_1);
1254 assert(arg_2 != R3_ARG1, "smashed argument");
1255 mr_if_needed(R4_ARG2, arg_2);
1256 call_VM_leaf(entry_point);
1257 }
1258
1259 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
1260 mr_if_needed(R3_ARG1, arg_1);
1261 assert(arg_2 != R3_ARG1, "smashed argument");
1262 mr_if_needed(R4_ARG2, arg_2);
1263 assert(arg_3 != R3_ARG1 && arg_3 != R4_ARG2, "smashed argument");
1264 mr_if_needed(R5_ARG3, arg_3);
1265 call_VM_leaf(entry_point);
1266 }
1267
1268 // Check whether instruction is a read access to the polling page
1269 // which was emitted by load_from_polling_page(..).
1270 bool MacroAssembler::is_load_from_polling_page(int instruction, void* ucontext,
1271 address* polling_address_ptr) {
1272 if (!is_ld(instruction))
1273 return false; // It's not a ld. Fail.
1274
1275 int rt = inv_rt_field(instruction);
1276 int ra = inv_ra_field(instruction);
1277 int ds = inv_ds_field(instruction);
1278 if (!(ds == 0 && ra != 0 && rt == 0)) {
1279 return false; // It's not a ld(r0, X, ra). Fail.
1280 }
1281
1282 if (!ucontext) {
1283 // Set polling address.
1284 if (polling_address_ptr != NULL) {
1285 *polling_address_ptr = NULL;
1286 }
1287 return true; // No ucontext given. Can't check value of ra. Assume true.
1288 }
1289
1290 #ifdef LINUX
1291 // Ucontext given. Check that register ra contains the address of
1292 // the safepoing polling page.
1293 ucontext_t* uc = (ucontext_t*) ucontext;
1294 // Set polling address.
1295 address addr = (address)uc->uc_mcontext.regs->gpr[ra] + (ssize_t)ds;
1296 if (polling_address_ptr != NULL) {
1297 *polling_address_ptr = addr;
1298 }
1299 return os::is_poll_address(addr);
1300 #else
1301 // Not on Linux, ucontext must be NULL.
1302 ShouldNotReachHere();
1303 return false;
1304 #endif
1305 }
1306
1307 bool MacroAssembler::is_memory_serialization(int instruction, JavaThread* thread, void* ucontext) {
1308 #ifdef LINUX
1309 ucontext_t* uc = (ucontext_t*) ucontext;
1310
1311 if (is_stwx(instruction) || is_stwux(instruction)) {
1312 int ra = inv_ra_field(instruction);
1313 int rb = inv_rb_field(instruction);
1314
1315 // look up content of ra and rb in ucontext
1316 address ra_val=(address)uc->uc_mcontext.regs->gpr[ra];
1317 long rb_val=(long)uc->uc_mcontext.regs->gpr[rb];
1318 return os::is_memory_serialize_page(thread, ra_val+rb_val);
1319 } else if (is_stw(instruction) || is_stwu(instruction)) {
1320 int ra = inv_ra_field(instruction);
1321 int d1 = inv_d1_field(instruction);
1322
1323 // look up content of ra in ucontext
1324 address ra_val=(address)uc->uc_mcontext.regs->gpr[ra];
1325 return os::is_memory_serialize_page(thread, ra_val+d1);
1326 } else {
1327 return false;
1328 }
1329 #else
1330 // workaround not needed on !LINUX :-)
1331 ShouldNotCallThis();
1332 return false;
1333 #endif
1334 }
1335
1336 void MacroAssembler::bang_stack_with_offset(int offset) {
1337 // When increasing the stack, the old stack pointer will be written
1338 // to the new top of stack according to the PPC64 abi.
1339 // Therefore, stack banging is not necessary when increasing
1340 // the stack by <= os::vm_page_size() bytes.
1341 // When increasing the stack by a larger amount, this method is
1342 // called repeatedly to bang the intermediate pages.
1343
1344 // Stack grows down, caller passes positive offset.
1345 assert(offset > 0, "must bang with positive offset");
1346
1347 long stdoffset = -offset;
1348
1349 if (is_simm(stdoffset, 16)) {
1350 // Signed 16 bit offset, a simple std is ok.
1351 if (UseLoadInstructionsForStackBangingPPC64) {
1352 ld(R0, (int)(signed short)stdoffset, R1_SP);
1353 } else {
1354 std(R0,(int)(signed short)stdoffset, R1_SP);
1355 }
1356 } else if (is_simm(stdoffset, 31)) {
1357 const int hi = MacroAssembler::largeoffset_si16_si16_hi(stdoffset);
1358 const int lo = MacroAssembler::largeoffset_si16_si16_lo(stdoffset);
1359
1360 Register tmp = R11;
1361 addis(tmp, R1_SP, hi);
1362 if (UseLoadInstructionsForStackBangingPPC64) {
1363 ld(R0, lo, tmp);
1364 } else {
1365 std(R0, lo, tmp);
1366 }
1367 } else {
1368 ShouldNotReachHere();
1369 }
1370 }
1371
1372 // If instruction is a stack bang of the form
1373 // std R0, x(Ry), (see bang_stack_with_offset())
1374 // stdu R1_SP, x(R1_SP), (see push_frame(), resize_frame())
1375 // or stdux R1_SP, Rx, R1_SP (see push_frame(), resize_frame())
1376 // return the banged address. Otherwise, return 0.
1377 address MacroAssembler::get_stack_bang_address(int instruction, void *ucontext) {
1378 #ifdef LINUX
1379 ucontext_t* uc = (ucontext_t*) ucontext;
1380 int rs = inv_rs_field(instruction);
1381 int ra = inv_ra_field(instruction);
1382 if ( (is_ld(instruction) && rs == 0 && UseLoadInstructionsForStackBangingPPC64)
1383 || (is_std(instruction) && rs == 0 && !UseLoadInstructionsForStackBangingPPC64)
1384 || (is_stdu(instruction) && rs == 1)) {
1385 int ds = inv_ds_field(instruction);
1386 // return banged address
1387 return ds+(address)uc->uc_mcontext.regs->gpr[ra];
1388 } else if (is_stdux(instruction) && rs == 1) {
1389 int rb = inv_rb_field(instruction);
1390 address sp = (address)uc->uc_mcontext.regs->gpr[1];
1391 long rb_val = (long)uc->uc_mcontext.regs->gpr[rb];
1392 return ra != 1 || rb_val >= 0 ? NULL // not a stack bang
1393 : sp + rb_val; // banged address
1394 }
1395 return NULL; // not a stack bang
1396 #else
1397 // workaround not needed on !LINUX :-)
1398 ShouldNotCallThis();
1399 return NULL;
1400 #endif
1401 }
1402
1403 void MacroAssembler::reserved_stack_check(Register return_pc) {
1404 // Test if reserved zone needs to be enabled.
1405 Label no_reserved_zone_enabling;
1406
1407 ld_ptr(R0, JavaThread::reserved_stack_activation_offset(), R16_thread);
1408 cmpld(CCR0, R1_SP, R0);
1409 blt_predict_taken(CCR0, no_reserved_zone_enabling);
1410
1411 // Enable reserved zone again, throw stack overflow exception.
1412 push_frame_reg_args(0, R0);
1413 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), R16_thread);
1414 pop_frame();
1415 mtlr(return_pc);
1416 load_const_optimized(R0, StubRoutines::throw_delayed_StackOverflowError_entry());
1417 mtctr(R0);
1418 bctr();
1419
1420 should_not_reach_here();
1421
1422 bind(no_reserved_zone_enabling);
1423 }
1424
1425 void MacroAssembler::getandsetd(Register dest_current_value, Register exchange_value, Register addr_base,
1426 bool cmpxchgx_hint) {
1427 Label retry;
1428 bind(retry);
1429 ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1430 stdcx_(exchange_value, addr_base);
1431 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1432 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1433 } else {
1434 bne( CCR0, retry); // StXcx_ sets CCR0.
1435 }
1436 }
1437
1438 void MacroAssembler::getandaddd(Register dest_current_value, Register inc_value, Register addr_base,
1439 Register tmp, bool cmpxchgx_hint) {
1440 Label retry;
1441 bind(retry);
1442 ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1443 add(tmp, dest_current_value, inc_value);
1444 stdcx_(tmp, addr_base);
1445 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1446 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1447 } else {
1448 bne( CCR0, retry); // StXcx_ sets CCR0.
1449 }
1450 }
1451
1452 // Word/sub-word atomic helper functions
1453
1454 // Temps and addr_base are killed if size < 4 and processor does not support respective instructions.
1455 // Only signed types are supported with size < 4.
1456 // Atomic add always kills tmp1.
1457 void MacroAssembler::atomic_get_and_modify_generic(Register dest_current_value, Register exchange_value,
1458 Register addr_base, Register tmp1, Register tmp2, Register tmp3,
1459 bool cmpxchgx_hint, bool is_add, int size) {
1460 // Sub-word instructions are available since Power 8.
1461 // For older processors, instruction_type != size holds, and we
1462 // emulate the sub-word instructions by constructing a 4-byte value
1463 // that leaves the other bytes unchanged.
1464 const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1465
1466 Label retry;
1467 Register shift_amount = noreg,
1468 val32 = dest_current_value,
1469 modval = is_add ? tmp1 : exchange_value;
1470
1471 if (instruction_type != size) {
1472 assert_different_registers(tmp1, tmp2, tmp3, dest_current_value, exchange_value, addr_base);
1473 modval = tmp1;
1474 shift_amount = tmp2;
1475 val32 = tmp3;
1476 // Need some preperation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1477 #ifdef VM_LITTLE_ENDIAN
1478 rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1479 clrrdi(addr_base, addr_base, 2);
1480 #else
1481 xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1482 clrrdi(addr_base, addr_base, 2);
1483 rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1484 #endif
1485 }
1486
1487 // atomic emulation loop
1488 bind(retry);
1489
1490 switch (instruction_type) {
1491 case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1492 case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1493 case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1494 default: ShouldNotReachHere();
1495 }
1496
1497 if (instruction_type != size) {
1498 srw(dest_current_value, val32, shift_amount);
1499 }
1500
1501 if (is_add) { add(modval, dest_current_value, exchange_value); }
1502
1503 if (instruction_type != size) {
1504 // Transform exchange value such that the replacement can be done by one xor instruction.
1505 xorr(modval, dest_current_value, is_add ? modval : exchange_value);
1506 clrldi(modval, modval, (size == 1) ? 56 : 48);
1507 slw(modval, modval, shift_amount);
1508 xorr(modval, val32, modval);
1509 }
1510
1511 switch (instruction_type) {
1512 case 4: stwcx_(modval, addr_base); break;
1513 case 2: sthcx_(modval, addr_base); break;
1514 case 1: stbcx_(modval, addr_base); break;
1515 default: ShouldNotReachHere();
1516 }
1517
1518 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1519 bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1520 } else {
1521 bne( CCR0, retry); // StXcx_ sets CCR0.
1522 }
1523
1524 // l?arx zero-extends, but Java wants byte/short values sign-extended.
1525 if (size == 1) {
1526 extsb(dest_current_value, dest_current_value);
1527 } else if (size == 2) {
1528 extsh(dest_current_value, dest_current_value);
1529 };
1530 }
1531
1532 // Temps, addr_base and exchange_value are killed if size < 4 and processor does not support respective instructions.
1533 // Only signed types are supported with size < 4.
1534 void MacroAssembler::cmpxchg_loop_body(ConditionRegister flag, Register dest_current_value,
1535 Register compare_value, Register exchange_value,
1536 Register addr_base, Register tmp1, Register tmp2,
1537 Label &retry, Label &failed, bool cmpxchgx_hint, int size) {
1538 // Sub-word instructions are available since Power 8.
1539 // For older processors, instruction_type != size holds, and we
1540 // emulate the sub-word instructions by constructing a 4-byte value
1541 // that leaves the other bytes unchanged.
1542 const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1543
1544 Register shift_amount = noreg,
1545 val32 = dest_current_value,
1546 modval = exchange_value;
1547
1548 if (instruction_type != size) {
1549 assert_different_registers(tmp1, tmp2, dest_current_value, compare_value, exchange_value, addr_base);
1550 shift_amount = tmp1;
1551 val32 = tmp2;
1552 modval = tmp2;
1553 // Need some preperation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1554 #ifdef VM_LITTLE_ENDIAN
1555 rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1556 clrrdi(addr_base, addr_base, 2);
1557 #else
1558 xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1559 clrrdi(addr_base, addr_base, 2);
1560 rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1561 #endif
1562 // Transform exchange value such that the replacement can be done by one xor instruction.
1563 xorr(exchange_value, compare_value, exchange_value);
1564 clrldi(exchange_value, exchange_value, (size == 1) ? 56 : 48);
1565 slw(exchange_value, exchange_value, shift_amount);
1566 }
1567
1568 // atomic emulation loop
1569 bind(retry);
1570
1571 switch (instruction_type) {
1572 case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1573 case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1574 case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1575 default: ShouldNotReachHere();
1576 }
1577
1578 if (instruction_type != size) {
1579 srw(dest_current_value, val32, shift_amount);
1580 }
1581 if (size == 1) {
1582 extsb(dest_current_value, dest_current_value);
1583 } else if (size == 2) {
1584 extsh(dest_current_value, dest_current_value);
1585 };
1586
1587 cmpw(flag, dest_current_value, compare_value);
1588 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1589 bne_predict_not_taken(flag, failed);
1590 } else {
1591 bne( flag, failed);
1592 }
1593 // branch to done => (flag == ne), (dest_current_value != compare_value)
1594 // fall through => (flag == eq), (dest_current_value == compare_value)
1595
1596 if (instruction_type != size) {
1597 xorr(modval, val32, exchange_value);
1598 }
1599
1600 switch (instruction_type) {
1601 case 4: stwcx_(modval, addr_base); break;
1602 case 2: sthcx_(modval, addr_base); break;
1603 case 1: stbcx_(modval, addr_base); break;
1604 default: ShouldNotReachHere();
1605 }
1606 }
1607
1608 // CmpxchgX sets condition register to cmpX(current, compare).
1609 void MacroAssembler::cmpxchg_generic(ConditionRegister flag, Register dest_current_value,
1610 Register compare_value, Register exchange_value,
1611 Register addr_base, Register tmp1, Register tmp2,
1612 int semantics, bool cmpxchgx_hint,
1613 Register int_flag_success, bool contention_hint, bool weak, int size) {
1614 Label retry;
1615 Label failed;
1616 Label done;
1617
1618 // Save one branch if result is returned via register and
1619 // result register is different from the other ones.
1620 bool use_result_reg = (int_flag_success != noreg);
1621 bool preset_result_reg = (int_flag_success != dest_current_value && int_flag_success != compare_value &&
1622 int_flag_success != exchange_value && int_flag_success != addr_base &&
1623 int_flag_success != tmp1 && int_flag_success != tmp2);
1624 assert(!weak || flag == CCR0, "weak only supported with CCR0");
1625 assert(size == 1 || size == 2 || size == 4, "unsupported");
1626
1627 if (use_result_reg && preset_result_reg) {
1628 li(int_flag_success, 0); // preset (assume cas failed)
1629 }
1630
1631 // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1632 if (contention_hint) { // Don't try to reserve if cmp fails.
1633 switch (size) {
1634 case 1: lbz(dest_current_value, 0, addr_base); extsb(dest_current_value, dest_current_value); break;
1635 case 2: lha(dest_current_value, 0, addr_base); break;
1636 case 4: lwz(dest_current_value, 0, addr_base); break;
1637 default: ShouldNotReachHere();
1638 }
1639 cmpw(flag, dest_current_value, compare_value);
1640 bne(flag, failed);
1641 }
1642
1643 // release/fence semantics
1644 if (semantics & MemBarRel) {
1645 release();
1646 }
1647
1648 cmpxchg_loop_body(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2,
1649 retry, failed, cmpxchgx_hint, size);
1650 if (!weak || use_result_reg) {
1651 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1652 bne_predict_not_taken(CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1653 } else {
1654 bne( CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1655 }
1656 }
1657 // fall through => (flag == eq), (dest_current_value == compare_value), (swapped)
1658
1659 // Result in register (must do this at the end because int_flag_success can be the
1660 // same register as one above).
1661 if (use_result_reg) {
1662 li(int_flag_success, 1);
1663 }
1664
1665 if (semantics & MemBarFenceAfter) {
1666 fence();
1667 } else if (semantics & MemBarAcq) {
1668 isync();
1669 }
1670
1671 if (use_result_reg && !preset_result_reg) {
1672 b(done);
1673 }
1674
1675 bind(failed);
1676 if (use_result_reg && !preset_result_reg) {
1677 li(int_flag_success, 0);
1678 }
1679
1680 bind(done);
1681 // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1682 // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1683 }
1684
1685 // Preforms atomic compare exchange:
1686 // if (compare_value == *addr_base)
1687 // *addr_base = exchange_value
1688 // int_flag_success = 1;
1689 // else
1690 // int_flag_success = 0;
1691 //
1692 // ConditionRegister flag = cmp(compare_value, *addr_base)
1693 // Register dest_current_value = *addr_base
1694 // Register compare_value Used to compare with value in memory
1695 // Register exchange_value Written to memory if compare_value == *addr_base
1696 // Register addr_base The memory location to compareXChange
1697 // Register int_flag_success Set to 1 if exchange_value was written to *addr_base
1698 //
1699 // To avoid the costly compare exchange the value is tested beforehand.
1700 // Several special cases exist to avoid that unnecessary information is generated.
1701 //
1702 void MacroAssembler::cmpxchgd(ConditionRegister flag,
1703 Register dest_current_value, RegisterOrConstant compare_value, Register exchange_value,
1704 Register addr_base, int semantics, bool cmpxchgx_hint,
1705 Register int_flag_success, Label* failed_ext, bool contention_hint, bool weak) {
1706 Label retry;
1707 Label failed_int;
1708 Label& failed = (failed_ext != NULL) ? *failed_ext : failed_int;
1709 Label done;
1710
1711 // Save one branch if result is returned via register and result register is different from the other ones.
1712 bool use_result_reg = (int_flag_success!=noreg);
1713 bool preset_result_reg = (int_flag_success!=dest_current_value && int_flag_success!=compare_value.register_or_noreg() &&
1714 int_flag_success!=exchange_value && int_flag_success!=addr_base);
1715 assert(!weak || flag == CCR0, "weak only supported with CCR0");
1716 assert(int_flag_success == noreg || failed_ext == NULL, "cannot have both");
1717
1718 if (use_result_reg && preset_result_reg) {
1719 li(int_flag_success, 0); // preset (assume cas failed)
1720 }
1721
1722 // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1723 if (contention_hint) { // Don't try to reserve if cmp fails.
1724 ld(dest_current_value, 0, addr_base);
1725 cmpd(flag, compare_value, dest_current_value);
1726 bne(flag, failed);
1727 }
1728
1729 // release/fence semantics
1730 if (semantics & MemBarRel) {
1731 release();
1732 }
1733
1734 // atomic emulation loop
1735 bind(retry);
1736
1737 ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1738 cmpd(flag, compare_value, dest_current_value);
1739 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1740 bne_predict_not_taken(flag, failed);
1741 } else {
1742 bne( flag, failed);
1743 }
1744
1745 stdcx_(exchange_value, addr_base);
1746 if (!weak || use_result_reg || failed_ext) {
1747 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1748 bne_predict_not_taken(CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1749 } else {
1750 bne( CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1751 }
1752 }
1753
1754 // result in register (must do this at the end because int_flag_success can be the same register as one above)
1755 if (use_result_reg) {
1756 li(int_flag_success, 1);
1757 }
1758
1759 if (semantics & MemBarFenceAfter) {
1760 fence();
1761 } else if (semantics & MemBarAcq) {
1762 isync();
1763 }
1764
1765 if (use_result_reg && !preset_result_reg) {
1766 b(done);
1767 }
1768
1769 bind(failed_int);
1770 if (use_result_reg && !preset_result_reg) {
1771 li(int_flag_success, 0);
1772 }
1773
1774 bind(done);
1775 // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1776 // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1777 }
1778
1779 // Look up the method for a megamorphic invokeinterface call.
1780 // The target method is determined by <intf_klass, itable_index>.
1781 // The receiver klass is in recv_klass.
1782 // On success, the result will be in method_result, and execution falls through.
1783 // On failure, execution transfers to the given label.
1784 void MacroAssembler::lookup_interface_method(Register recv_klass,
1785 Register intf_klass,
1786 RegisterOrConstant itable_index,
1787 Register method_result,
1788 Register scan_temp,
1789 Register sethi_temp,
1790 Label& L_no_such_interface) {
1791 assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
1792 assert(itable_index.is_constant() || itable_index.as_register() == method_result,
1793 "caller must use same register for non-constant itable index as for method");
1794
1795 // Compute start of first itableOffsetEntry (which is at the end of the vtable).
1796 int vtable_base = in_bytes(Klass::vtable_start_offset());
1797 int itentry_off = itableMethodEntry::method_offset_in_bytes();
1798 int logMEsize = exact_log2(itableMethodEntry::size() * wordSize);
1799 int scan_step = itableOffsetEntry::size() * wordSize;
1800 int log_vte_size= exact_log2(vtableEntry::size_in_bytes());
1801
1802 lwz(scan_temp, in_bytes(Klass::vtable_length_offset()), recv_klass);
1803 // %%% We should store the aligned, prescaled offset in the klassoop.
1804 // Then the next several instructions would fold away.
1805
1806 sldi(scan_temp, scan_temp, log_vte_size);
1807 addi(scan_temp, scan_temp, vtable_base);
1808 add(scan_temp, recv_klass, scan_temp);
1809
1810 // Adjust recv_klass by scaled itable_index, so we can free itable_index.
1811 if (itable_index.is_register()) {
1812 Register itable_offset = itable_index.as_register();
1813 sldi(itable_offset, itable_offset, logMEsize);
1814 if (itentry_off) addi(itable_offset, itable_offset, itentry_off);
1815 add(recv_klass, itable_offset, recv_klass);
1816 } else {
1817 long itable_offset = (long)itable_index.as_constant();
1818 load_const_optimized(sethi_temp, (itable_offset<<logMEsize)+itentry_off); // static address, no relocation
1819 add(recv_klass, sethi_temp, recv_klass);
1820 }
1821
1822 // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
1823 // if (scan->interface() == intf) {
1824 // result = (klass + scan->offset() + itable_index);
1825 // }
1826 // }
1827 Label search, found_method;
1828
1829 for (int peel = 1; peel >= 0; peel--) {
1830 // %%%% Could load both offset and interface in one ldx, if they were
1831 // in the opposite order. This would save a load.
1832 ld(method_result, itableOffsetEntry::interface_offset_in_bytes(), scan_temp);
1833
1834 // Check that this entry is non-null. A null entry means that
1835 // the receiver class doesn't implement the interface, and wasn't the
1836 // same as when the caller was compiled.
1837 cmpd(CCR0, method_result, intf_klass);
1838
1839 if (peel) {
1840 beq(CCR0, found_method);
1841 } else {
1842 bne(CCR0, search);
1843 // (invert the test to fall through to found_method...)
1844 }
1845
1846 if (!peel) break;
1847
1848 bind(search);
1849
1850 cmpdi(CCR0, method_result, 0);
1851 beq(CCR0, L_no_such_interface);
1852 addi(scan_temp, scan_temp, scan_step);
1853 }
1854
1855 bind(found_method);
1856
1857 // Got a hit.
1858 int ito_offset = itableOffsetEntry::offset_offset_in_bytes();
1859 lwz(scan_temp, ito_offset, scan_temp);
1860 ldx(method_result, scan_temp, recv_klass);
1861 }
1862
1863 // virtual method calling
1864 void MacroAssembler::lookup_virtual_method(Register recv_klass,
1865 RegisterOrConstant vtable_index,
1866 Register method_result) {
1867
1868 assert_different_registers(recv_klass, method_result, vtable_index.register_or_noreg());
1869
1870 const int base = in_bytes(Klass::vtable_start_offset());
1871 assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
1872
1873 if (vtable_index.is_register()) {
1874 sldi(vtable_index.as_register(), vtable_index.as_register(), LogBytesPerWord);
1875 add(recv_klass, vtable_index.as_register(), recv_klass);
1876 } else {
1877 addi(recv_klass, recv_klass, vtable_index.as_constant() << LogBytesPerWord);
1878 }
1879 ld(R19_method, base + vtableEntry::method_offset_in_bytes(), recv_klass);
1880 }
1881
1882 /////////////////////////////////////////// subtype checking ////////////////////////////////////////////
1883 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
1884 Register super_klass,
1885 Register temp1_reg,
1886 Register temp2_reg,
1887 Label* L_success,
1888 Label* L_failure,
1889 Label* L_slow_path,
1890 RegisterOrConstant super_check_offset) {
1891
1892 const Register check_cache_offset = temp1_reg;
1893 const Register cached_super = temp2_reg;
1894
1895 assert_different_registers(sub_klass, super_klass, check_cache_offset, cached_super);
1896
1897 int sco_offset = in_bytes(Klass::super_check_offset_offset());
1898 int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1899
1900 bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1901 bool need_slow_path = (must_load_sco || super_check_offset.constant_or_zero() == sco_offset);
1902
1903 Label L_fallthrough;
1904 int label_nulls = 0;
1905 if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; }
1906 if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; }
1907 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
1908 assert(label_nulls <= 1 ||
1909 (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
1910 "at most one NULL in the batch, usually");
1911
1912 // If the pointers are equal, we are done (e.g., String[] elements).
1913 // This self-check enables sharing of secondary supertype arrays among
1914 // non-primary types such as array-of-interface. Otherwise, each such
1915 // type would need its own customized SSA.
1916 // We move this check to the front of the fast path because many
1917 // type checks are in fact trivially successful in this manner,
1918 // so we get a nicely predicted branch right at the start of the check.
1919 cmpd(CCR0, sub_klass, super_klass);
1920 beq(CCR0, *L_success);
1921
1922 // Check the supertype display:
1923 if (must_load_sco) {
1924 // The super check offset is always positive...
1925 lwz(check_cache_offset, sco_offset, super_klass);
1926 super_check_offset = RegisterOrConstant(check_cache_offset);
1927 // super_check_offset is register.
1928 assert_different_registers(sub_klass, super_klass, cached_super, super_check_offset.as_register());
1929 }
1930 // The loaded value is the offset from KlassOopDesc.
1931
1932 ld(cached_super, super_check_offset, sub_klass);
1933 cmpd(CCR0, cached_super, super_klass);
1934
1935 // This check has worked decisively for primary supers.
1936 // Secondary supers are sought in the super_cache ('super_cache_addr').
1937 // (Secondary supers are interfaces and very deeply nested subtypes.)
1938 // This works in the same check above because of a tricky aliasing
1939 // between the super_cache and the primary super display elements.
1940 // (The 'super_check_addr' can address either, as the case requires.)
1941 // Note that the cache is updated below if it does not help us find
1942 // what we need immediately.
1943 // So if it was a primary super, we can just fail immediately.
1944 // Otherwise, it's the slow path for us (no success at this point).
1945
1946 #define FINAL_JUMP(label) if (&(label) != &L_fallthrough) { b(label); }
1947
1948 if (super_check_offset.is_register()) {
1949 beq(CCR0, *L_success);
1950 cmpwi(CCR0, super_check_offset.as_register(), sc_offset);
1951 if (L_failure == &L_fallthrough) {
1952 beq(CCR0, *L_slow_path);
1953 } else {
1954 bne(CCR0, *L_failure);
1955 FINAL_JUMP(*L_slow_path);
1956 }
1957 } else {
1958 if (super_check_offset.as_constant() == sc_offset) {
1959 // Need a slow path; fast failure is impossible.
1960 if (L_slow_path == &L_fallthrough) {
1961 beq(CCR0, *L_success);
1962 } else {
1963 bne(CCR0, *L_slow_path);
1964 FINAL_JUMP(*L_success);
1965 }
1966 } else {
1967 // No slow path; it's a fast decision.
1968 if (L_failure == &L_fallthrough) {
1969 beq(CCR0, *L_success);
1970 } else {
1971 bne(CCR0, *L_failure);
1972 FINAL_JUMP(*L_success);
1973 }
1974 }
1975 }
1976
1977 bind(L_fallthrough);
1978 #undef FINAL_JUMP
1979 }
1980
1981 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
1982 Register super_klass,
1983 Register temp1_reg,
1984 Register temp2_reg,
1985 Label* L_success,
1986 Register result_reg) {
1987 const Register array_ptr = temp1_reg; // current value from cache array
1988 const Register temp = temp2_reg;
1989
1990 assert_different_registers(sub_klass, super_klass, array_ptr, temp);
1991
1992 int source_offset = in_bytes(Klass::secondary_supers_offset());
1993 int target_offset = in_bytes(Klass::secondary_super_cache_offset());
1994
1995 int length_offset = Array<Klass*>::length_offset_in_bytes();
1996 int base_offset = Array<Klass*>::base_offset_in_bytes();
1997
1998 Label hit, loop, failure, fallthru;
1999
2000 ld(array_ptr, source_offset, sub_klass);
2001
2002 // TODO: PPC port: assert(4 == arrayOopDesc::length_length_in_bytes(), "precondition violated.");
2003 lwz(temp, length_offset, array_ptr);
2004 cmpwi(CCR0, temp, 0);
2005 beq(CCR0, result_reg!=noreg ? failure : fallthru); // length 0
2006
2007 mtctr(temp); // load ctr
2008
2009 bind(loop);
2010 // Oops in table are NO MORE compressed.
2011 ld(temp, base_offset, array_ptr);
2012 cmpd(CCR0, temp, super_klass);
2013 beq(CCR0, hit);
2014 addi(array_ptr, array_ptr, BytesPerWord);
2015 bdnz(loop);
2016
2017 bind(failure);
2018 if (result_reg!=noreg) li(result_reg, 1); // load non-zero result (indicates a miss)
2019 b(fallthru);
2020
2021 bind(hit);
2022 std(super_klass, target_offset, sub_klass); // save result to cache
2023 if (result_reg != noreg) { li(result_reg, 0); } // load zero result (indicates a hit)
2024 if (L_success != NULL) { b(*L_success); }
2025 else if (result_reg == noreg) { blr(); } // return with CR0.eq if neither label nor result reg provided
2026
2027 bind(fallthru);
2028 }
2029
2030 // Try fast path, then go to slow one if not successful
2031 void MacroAssembler::check_klass_subtype(Register sub_klass,
2032 Register super_klass,
2033 Register temp1_reg,
2034 Register temp2_reg,
2035 Label& L_success) {
2036 Label L_failure;
2037 check_klass_subtype_fast_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success, &L_failure);
2038 check_klass_subtype_slow_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success);
2039 bind(L_failure); // Fallthru if not successful.
2040 }
2041
2042 void MacroAssembler::check_method_handle_type(Register mtype_reg, Register mh_reg,
2043 Register temp_reg,
2044 Label& wrong_method_type) {
2045 assert_different_registers(mtype_reg, mh_reg, temp_reg);
2046 // Compare method type against that of the receiver.
2047 load_heap_oop_not_null(temp_reg, delayed_value(java_lang_invoke_MethodHandle::type_offset_in_bytes, temp_reg), mh_reg);
2048 cmpd(CCR0, temp_reg, mtype_reg);
2049 bne(CCR0, wrong_method_type);
2050 }
2051
2052 RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot,
2053 Register temp_reg,
2054 int extra_slot_offset) {
2055 // cf. TemplateTable::prepare_invoke(), if (load_receiver).
2056 int stackElementSize = Interpreter::stackElementSize;
2057 int offset = extra_slot_offset * stackElementSize;
2058 if (arg_slot.is_constant()) {
2059 offset += arg_slot.as_constant() * stackElementSize;
2060 return offset;
2061 } else {
2062 assert(temp_reg != noreg, "must specify");
2063 sldi(temp_reg, arg_slot.as_register(), exact_log2(stackElementSize));
2064 if (offset != 0)
2065 addi(temp_reg, temp_reg, offset);
2066 return temp_reg;
2067 }
2068 }
2069
2070 // Supports temp2_reg = R0.
2071 void MacroAssembler::biased_locking_enter(ConditionRegister cr_reg, Register obj_reg,
2072 Register mark_reg, Register temp_reg,
2073 Register temp2_reg, Label& done, Label* slow_case) {
2074 assert(UseBiasedLocking, "why call this otherwise?");
2075
2076 #ifdef ASSERT
2077 assert_different_registers(obj_reg, mark_reg, temp_reg, temp2_reg);
2078 #endif
2079
2080 Label cas_label;
2081
2082 // Branch to done if fast path fails and no slow_case provided.
2083 Label *slow_case_int = (slow_case != NULL) ? slow_case : &done;
2084
2085 // Biased locking
2086 // See whether the lock is currently biased toward our thread and
2087 // whether the epoch is still valid
2088 // Note that the runtime guarantees sufficient alignment of JavaThread
2089 // pointers to allow age to be placed into low bits
2090 assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits,
2091 "biased locking makes assumptions about bit layout");
2092
2093 if (PrintBiasedLockingStatistics) {
2094 load_const(temp2_reg, (address) BiasedLocking::total_entry_count_addr(), temp_reg);
2095 lwzx(temp_reg, temp2_reg);
2096 addi(temp_reg, temp_reg, 1);
2097 stwx(temp_reg, temp2_reg);
2098 }
2099
2100 andi(temp_reg, mark_reg, markOopDesc::biased_lock_mask_in_place);
2101 cmpwi(cr_reg, temp_reg, markOopDesc::biased_lock_pattern);
2102 bne(cr_reg, cas_label);
2103
2104 load_klass(temp_reg, obj_reg);
2105
2106 load_const_optimized(temp2_reg, ~((int) markOopDesc::age_mask_in_place));
2107 ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg);
2108 orr(temp_reg, R16_thread, temp_reg);
2109 xorr(temp_reg, mark_reg, temp_reg);
2110 andr(temp_reg, temp_reg, temp2_reg);
2111 cmpdi(cr_reg, temp_reg, 0);
2112 if (PrintBiasedLockingStatistics) {
2113 Label l;
2114 bne(cr_reg, l);
2115 load_const(temp2_reg, (address) BiasedLocking::biased_lock_entry_count_addr());
2116 lwzx(mark_reg, temp2_reg);
2117 addi(mark_reg, mark_reg, 1);
2118 stwx(mark_reg, temp2_reg);
2119 // restore mark_reg
2120 ld(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg);
2121 bind(l);
2122 }
2123 beq(cr_reg, done);
2124
2125 Label try_revoke_bias;
2126 Label try_rebias;
2127
2128 // At this point we know that the header has the bias pattern and
2129 // that we are not the bias owner in the current epoch. We need to
2130 // figure out more details about the state of the header in order to
2131 // know what operations can be legally performed on the object's
2132 // header.
2133
2134 // If the low three bits in the xor result aren't clear, that means
2135 // the prototype header is no longer biased and we have to revoke
2136 // the bias on this object.
2137 andi(temp2_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);
2138 cmpwi(cr_reg, temp2_reg, 0);
2139 bne(cr_reg, try_revoke_bias);
2140
2141 // Biasing is still enabled for this data type. See whether the
2142 // epoch of the current bias is still valid, meaning that the epoch
2143 // bits of the mark word are equal to the epoch bits of the
2144 // prototype header. (Note that the prototype header's epoch bits
2145 // only change at a safepoint.) If not, attempt to rebias the object
2146 // toward the current thread. Note that we must be absolutely sure
2147 // that the current epoch is invalid in order to do this because
2148 // otherwise the manipulations it performs on the mark word are
2149 // illegal.
2150
2151 int shift_amount = 64 - markOopDesc::epoch_shift;
2152 // rotate epoch bits to right (little) end and set other bits to 0
2153 // [ big part | epoch | little part ] -> [ 0..0 | epoch ]
2154 rldicl_(temp2_reg, temp_reg, shift_amount, 64 - markOopDesc::epoch_bits);
2155 // branch if epoch bits are != 0, i.e. they differ, because the epoch has been incremented
2156 bne(CCR0, try_rebias);
2157
2158 // The epoch of the current bias is still valid but we know nothing
2159 // about the owner; it might be set or it might be clear. Try to
2160 // acquire the bias of the object using an atomic operation. If this
2161 // fails we will go in to the runtime to revoke the object's bias.
2162 // Note that we first construct the presumed unbiased header so we
2163 // don't accidentally blow away another thread's valid bias.
2164 andi(mark_reg, mark_reg, (markOopDesc::biased_lock_mask_in_place |
2165 markOopDesc::age_mask_in_place |
2166 markOopDesc::epoch_mask_in_place));
2167 orr(temp_reg, R16_thread, mark_reg);
2168
2169 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2170
2171 // CmpxchgX sets cr_reg to cmpX(temp2_reg, mark_reg).
2172 cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg,
2173 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg,
2174 /*where=*/obj_reg,
2175 MacroAssembler::MemBarAcq,
2176 MacroAssembler::cmpxchgx_hint_acquire_lock(),
2177 noreg, slow_case_int); // bail out if failed
2178
2179 // If the biasing toward our thread failed, this means that
2180 // another thread succeeded in biasing it toward itself and we
2181 // need to revoke that bias. The revocation will occur in the
2182 // interpreter runtime in the slow case.
2183 if (PrintBiasedLockingStatistics) {
2184 load_const(temp2_reg, (address) BiasedLocking::anonymously_biased_lock_entry_count_addr(), temp_reg);
2185 lwzx(temp_reg, temp2_reg);
2186 addi(temp_reg, temp_reg, 1);
2187 stwx(temp_reg, temp2_reg);
2188 }
2189 b(done);
2190
2191 bind(try_rebias);
2192 // At this point we know the epoch has expired, meaning that the
2193 // current "bias owner", if any, is actually invalid. Under these
2194 // circumstances _only_, we are allowed to use the current header's
2195 // value as the comparison value when doing the cas to acquire the
2196 // bias in the current epoch. In other words, we allow transfer of
2197 // the bias from one thread to another directly in this situation.
2198 load_klass(temp_reg, obj_reg);
2199 andi(temp2_reg, mark_reg, markOopDesc::age_mask_in_place);
2200 orr(temp2_reg, R16_thread, temp2_reg);
2201 ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg);
2202 orr(temp_reg, temp2_reg, temp_reg);
2203
2204 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2205
2206 cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg,
2207 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg,
2208 /*where=*/obj_reg,
2209 MacroAssembler::MemBarAcq,
2210 MacroAssembler::cmpxchgx_hint_acquire_lock(),
2211 noreg, slow_case_int); // bail out if failed
2212
2213 // If the biasing toward our thread failed, this means that
2214 // another thread succeeded in biasing it toward itself and we
2215 // need to revoke that bias. The revocation will occur in the
2216 // interpreter runtime in the slow case.
2217 if (PrintBiasedLockingStatistics) {
2218 load_const(temp2_reg, (address) BiasedLocking::rebiased_lock_entry_count_addr(), temp_reg);
2219 lwzx(temp_reg, temp2_reg);
2220 addi(temp_reg, temp_reg, 1);
2221 stwx(temp_reg, temp2_reg);
2222 }
2223 b(done);
2224
2225 bind(try_revoke_bias);
2226 // The prototype mark in the klass doesn't have the bias bit set any
2227 // more, indicating that objects of this data type are not supposed
2228 // to be biased any more. We are going to try to reset the mark of
2229 // this object to the prototype value and fall through to the
2230 // CAS-based locking scheme. Note that if our CAS fails, it means
2231 // that another thread raced us for the privilege of revoking the
2232 // bias of this particular object, so it's okay to continue in the
2233 // normal locking code.
2234 load_klass(temp_reg, obj_reg);
2235 ld(temp_reg, in_bytes(Klass::prototype_header_offset()), temp_reg);
2236 andi(temp2_reg, mark_reg, markOopDesc::age_mask_in_place);
2237 orr(temp_reg, temp_reg, temp2_reg);
2238
2239 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2240
2241 // CmpxchgX sets cr_reg to cmpX(temp2_reg, mark_reg).
2242 cmpxchgd(/*flag=*/cr_reg, /*current_value=*/temp2_reg,
2243 /*compare_value=*/mark_reg, /*exchange_value=*/temp_reg,
2244 /*where=*/obj_reg,
2245 MacroAssembler::MemBarAcq,
2246 MacroAssembler::cmpxchgx_hint_acquire_lock());
2247
2248 // reload markOop in mark_reg before continuing with lightweight locking
2249 ld(mark_reg, oopDesc::mark_offset_in_bytes(), obj_reg);
2250
2251 // Fall through to the normal CAS-based lock, because no matter what
2252 // the result of the above CAS, some thread must have succeeded in
2253 // removing the bias bit from the object's header.
2254 if (PrintBiasedLockingStatistics) {
2255 Label l;
2256 bne(cr_reg, l);
2257 load_const(temp2_reg, (address) BiasedLocking::revoked_lock_entry_count_addr(), temp_reg);
2258 lwzx(temp_reg, temp2_reg);
2259 addi(temp_reg, temp_reg, 1);
2260 stwx(temp_reg, temp2_reg);
2261 bind(l);
2262 }
2263
2264 bind(cas_label);
2265 }
2266
2267 void MacroAssembler::biased_locking_exit (ConditionRegister cr_reg, Register mark_addr, Register temp_reg, Label& done) {
2268 // Check for biased locking unlock case, which is a no-op
2269 // Note: we do not have to check the thread ID for two reasons.
2270 // First, the interpreter checks for IllegalMonitorStateException at
2271 // a higher level. Second, if the bias was revoked while we held the
2272 // lock, the object could not be rebiased toward another thread, so
2273 // the bias bit would be clear.
2274
2275 ld(temp_reg, 0, mark_addr);
2276 andi(temp_reg, temp_reg, markOopDesc::biased_lock_mask_in_place);
2277
2278 cmpwi(cr_reg, temp_reg, markOopDesc::biased_lock_pattern);
2279 beq(cr_reg, done);
2280 }
2281
2282 // allocation (for C1)
2283 void MacroAssembler::eden_allocate(
2284 Register obj, // result: pointer to object after successful allocation
2285 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
2286 int con_size_in_bytes, // object size in bytes if known at compile time
2287 Register t1, // temp register
2288 Register t2, // temp register
2289 Label& slow_case // continuation point if fast allocation fails
2290 ) {
2291 b(slow_case);
2292 }
2293
2294 void MacroAssembler::tlab_allocate(
2295 Register obj, // result: pointer to object after successful allocation
2296 Register var_size_in_bytes, // object size in bytes if unknown at compile time; invalid otherwise
2297 int con_size_in_bytes, // object size in bytes if known at compile time
2298 Register t1, // temp register
2299 Label& slow_case // continuation point if fast allocation fails
2300 ) {
2301 // make sure arguments make sense
2302 assert_different_registers(obj, var_size_in_bytes, t1);
2303 assert(0 <= con_size_in_bytes && is_simm13(con_size_in_bytes), "illegal object size");
2304 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
2305
2306 const Register new_top = t1;
2307 //verify_tlab(); not implemented
2308
2309 ld(obj, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2310 ld(R0, in_bytes(JavaThread::tlab_end_offset()), R16_thread);
2311 if (var_size_in_bytes == noreg) {
2312 addi(new_top, obj, con_size_in_bytes);
2313 } else {
2314 add(new_top, obj, var_size_in_bytes);
2315 }
2316 cmpld(CCR0, new_top, R0);
2317 bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
2318
2319 #ifdef ASSERT
2320 // make sure new free pointer is properly aligned
2321 {
2322 Label L;
2323 andi_(R0, new_top, MinObjAlignmentInBytesMask);
2324 beq(CCR0, L);
2325 stop("updated TLAB free is not properly aligned", 0x934);
2326 bind(L);
2327 }
2328 #endif // ASSERT
2329
2330 // update the tlab top pointer
2331 std(new_top, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2332 //verify_tlab(); not implemented
2333 }
2334 void MacroAssembler::tlab_refill(Label& retry_tlab, Label& try_eden, Label& slow_case) {
2335 unimplemented("tlab_refill");
2336 }
2337 void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, Register t1, Register t2) {
2338 unimplemented("incr_allocated_bytes");
2339 }
2340
2341 address MacroAssembler::emit_trampoline_stub(int destination_toc_offset,
2342 int insts_call_instruction_offset, Register Rtoc) {
2343 // Start the stub.
2344 address stub = start_a_stub(64);
2345 if (stub == NULL) { return NULL; } // CodeCache full: bail out
2346
2347 // Create a trampoline stub relocation which relates this trampoline stub
2348 // with the call instruction at insts_call_instruction_offset in the
2349 // instructions code-section.
2350 relocate(trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset));
2351 const int stub_start_offset = offset();
2352
2353 // For java_to_interp stubs we use R11_scratch1 as scratch register
2354 // and in call trampoline stubs we use R12_scratch2. This way we
2355 // can distinguish them (see is_NativeCallTrampolineStub_at()).
2356 Register reg_scratch = R12_scratch2;
2357
2358 // Now, create the trampoline stub's code:
2359 // - load the TOC
2360 // - load the call target from the constant pool
2361 // - call
2362 if (Rtoc == noreg) {
2363 calculate_address_from_global_toc(reg_scratch, method_toc());
2364 Rtoc = reg_scratch;
2365 }
2366
2367 ld_largeoffset_unchecked(reg_scratch, destination_toc_offset, Rtoc, false);
2368 mtctr(reg_scratch);
2369 bctr();
2370
2371 const address stub_start_addr = addr_at(stub_start_offset);
2372
2373 // Assert that the encoded destination_toc_offset can be identified and that it is correct.
2374 assert(destination_toc_offset == NativeCallTrampolineStub_at(stub_start_addr)->destination_toc_offset(),
2375 "encoded offset into the constant pool must match");
2376 // Trampoline_stub_size should be good.
2377 assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size");
2378 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
2379
2380 // End the stub.
2381 end_a_stub();
2382 return stub;
2383 }
2384
2385 // TM on PPC64.
2386 void MacroAssembler::atomic_inc_ptr(Register addr, Register result, int simm16) {
2387 Label retry;
2388 bind(retry);
2389 ldarx(result, addr, /*hint*/ false);
2390 addi(result, result, simm16);
2391 stdcx_(result, addr);
2392 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2393 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2394 } else {
2395 bne( CCR0, retry); // stXcx_ sets CCR0
2396 }
2397 }
2398
2399 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) {
2400 Label retry;
2401 bind(retry);
2402 lwarx(result, addr, /*hint*/ false);
2403 ori(result, result, uimm16);
2404 stwcx_(result, addr);
2405 if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2406 bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2407 } else {
2408 bne( CCR0, retry); // stXcx_ sets CCR0
2409 }
2410 }
2411
2412 #if INCLUDE_RTM_OPT
2413
2414 // Update rtm_counters based on abort status
2415 // input: abort_status
2416 // rtm_counters (RTMLockingCounters*)
2417 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) {
2418 // Mapping to keep PreciseRTMLockingStatistics similar to x86.
2419 // x86 ppc (! means inverted, ? means not the same)
2420 // 0 31 Set if abort caused by XABORT instruction.
2421 // 1 ! 7 If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set.
2422 // 2 13 Set if another logical processor conflicted with a memory address that was part of the transaction that aborted.
2423 // 3 10 Set if an internal buffer overflowed.
2424 // 4 ?12 Set if a debug breakpoint was hit.
2425 // 5 ?32 Set if an abort occurred during execution of a nested transaction.
2426 const int tm_failure_bit[] = {Assembler::tm_tabort, // Note: Seems like signal handler sets this, too.
2427 Assembler::tm_failure_persistent, // inverted: transient
2428 Assembler::tm_trans_cf,
2429 Assembler::tm_footprint_of,
2430 Assembler::tm_non_trans_cf,
2431 Assembler::tm_suspended};
2432 const bool tm_failure_inv[] = {false, true, false, false, false, false};
2433 assert(sizeof(tm_failure_bit)/sizeof(int) == RTMLockingCounters::ABORT_STATUS_LIMIT, "adapt mapping!");
2434
2435 const Register addr_Reg = R0;
2436 // Keep track of offset to where rtm_counters_Reg had pointed to.
2437 int counters_offs = RTMLockingCounters::abort_count_offset();
2438 addi(addr_Reg, rtm_counters_Reg, counters_offs);
2439 const Register temp_Reg = rtm_counters_Reg;
2440
2441 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically
2442 ldx(temp_Reg, addr_Reg);
2443 addi(temp_Reg, temp_Reg, 1);
2444 stdx(temp_Reg, addr_Reg);
2445
2446 if (PrintPreciseRTMLockingStatistics) {
2447 int counters_offs_delta = RTMLockingCounters::abortX_count_offset() - counters_offs;
2448
2449 //mftexasr(abort_status); done by caller
2450 for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
2451 counters_offs += counters_offs_delta;
2452 li(temp_Reg, counters_offs_delta); // can't use addi with R0
2453 add(addr_Reg, addr_Reg, temp_Reg); // point to next counter
2454 counters_offs_delta = sizeof(uintx);
2455
2456 Label check_abort;
2457 rldicr_(temp_Reg, abort_status, tm_failure_bit[i], 0);
2458 if (tm_failure_inv[i]) {
2459 bne(CCR0, check_abort);
2460 } else {
2461 beq(CCR0, check_abort);
2462 }
2463 //atomic_inc_ptr(addr_Reg, temp_Reg); We don't increment atomically
2464 ldx(temp_Reg, addr_Reg);
2465 addi(temp_Reg, temp_Reg, 1);
2466 stdx(temp_Reg, addr_Reg);
2467 bind(check_abort);
2468 }
2469 }
2470 li(temp_Reg, -counters_offs); // can't use addi with R0
2471 add(rtm_counters_Reg, addr_Reg, temp_Reg); // restore
2472 }
2473
2474 // Branch if (random & (count-1) != 0), count is 2^n
2475 // tmp and CR0 are killed
2476 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) {
2477 mftb(tmp);
2478 andi_(tmp, tmp, count-1);
2479 bne(CCR0, brLabel);
2480 }
2481
2482 // Perform abort ratio calculation, set no_rtm bit if high ratio.
2483 // input: rtm_counters_Reg (RTMLockingCounters* address) - KILLED
2484 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg,
2485 RTMLockingCounters* rtm_counters,
2486 Metadata* method_data) {
2487 Label L_done, L_check_always_rtm1, L_check_always_rtm2;
2488
2489 if (RTMLockingCalculationDelay > 0) {
2490 // Delay calculation.
2491 ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr());
2492 cmpdi(CCR0, rtm_counters_Reg, 0);
2493 beq(CCR0, L_done);
2494 load_const_optimized(rtm_counters_Reg, (address)rtm_counters, R0); // reload
2495 }
2496 // Abort ratio calculation only if abort_count > RTMAbortThreshold.
2497 // Aborted transactions = abort_count * 100
2498 // All transactions = total_count * RTMTotalCountIncrRate
2499 // Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
2500 ld(R0, RTMLockingCounters::abort_count_offset(), rtm_counters_Reg);
2501 cmpdi(CCR0, R0, RTMAbortThreshold);
2502 blt(CCR0, L_check_always_rtm2);
2503 mulli(R0, R0, 100);
2504
2505 const Register tmpReg = rtm_counters_Reg;
2506 ld(tmpReg, RTMLockingCounters::total_count_offset(), rtm_counters_Reg);
2507 mulli(tmpReg, tmpReg, RTMTotalCountIncrRate);
2508 mulli(tmpReg, tmpReg, RTMAbortRatio);
2509 cmpd(CCR0, R0, tmpReg);
2510 blt(CCR0, L_check_always_rtm1); // jump to reload
2511 if (method_data != NULL) {
2512 // Set rtm_state to "no rtm" in MDO.
2513 // Not using a metadata relocation. Method and Class Loader are kept alive anyway.
2514 // (See nmethod::metadata_do and CodeBuffer::finalize_oop_references.)
2515 load_const(R0, (address)method_data + MethodData::rtm_state_offset_in_bytes(), tmpReg);
2516 atomic_ori_int(R0, tmpReg, NoRTM);
2517 }
2518 b(L_done);
2519
2520 bind(L_check_always_rtm1);
2521 load_const_optimized(rtm_counters_Reg, (address)rtm_counters, R0); // reload
2522 bind(L_check_always_rtm2);
2523 ld(tmpReg, RTMLockingCounters::total_count_offset(), rtm_counters_Reg);
2524 int64_t thresholdValue = RTMLockingThreshold / RTMTotalCountIncrRate;
2525 if (is_simm(thresholdValue, 16)) { // cmpdi can handle 16bit immediate only.
2526 cmpdi(CCR0, tmpReg, thresholdValue);
2527 } else {
2528 load_const_optimized(R0, thresholdValue);
2529 cmpd(CCR0, tmpReg, R0);
2530 }
2531 blt(CCR0, L_done);
2532 if (method_data != NULL) {
2533 // Set rtm_state to "always rtm" in MDO.
2534 // Not using a metadata relocation. See above.
2535 load_const(R0, (address)method_data + MethodData::rtm_state_offset_in_bytes(), tmpReg);
2536 atomic_ori_int(R0, tmpReg, UseRTM);
2537 }
2538 bind(L_done);
2539 }
2540
2541 // Update counters and perform abort ratio calculation.
2542 // input: abort_status_Reg
2543 void MacroAssembler::rtm_profiling(Register abort_status_Reg, Register temp_Reg,
2544 RTMLockingCounters* rtm_counters,
2545 Metadata* method_data,
2546 bool profile_rtm) {
2547
2548 assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
2549 // Update rtm counters based on state at abort.
2550 // Reads abort_status_Reg, updates flags.
2551 assert_different_registers(abort_status_Reg, temp_Reg);
2552 load_const_optimized(temp_Reg, (address)rtm_counters, R0);
2553 rtm_counters_update(abort_status_Reg, temp_Reg);
2554 if (profile_rtm) {
2555 assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
2556 rtm_abort_ratio_calculation(temp_Reg, rtm_counters, method_data);
2557 }
2558 }
2559
2560 // Retry on abort if abort's status indicates non-persistent failure.
2561 // inputs: retry_count_Reg
2562 // : abort_status_Reg
2563 // output: retry_count_Reg decremented by 1
2564 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg,
2565 Label& retryLabel, Label* checkRetry) {
2566 Label doneRetry;
2567 rldicr_(R0, abort_status_Reg, tm_failure_persistent, 0);
2568 bne(CCR0, doneRetry);
2569 if (checkRetry) { bind(*checkRetry); }
2570 addic_(retry_count_Reg, retry_count_Reg, -1);
2571 blt(CCR0, doneRetry);
2572 smt_yield(); // Can't use wait(). No permission (SIGILL).
2573 b(retryLabel);
2574 bind(doneRetry);
2575 }
2576
2577 // Spin and retry if lock is busy.
2578 // inputs: owner_addr_Reg (monitor address)
2579 // : retry_count_Reg
2580 // output: retry_count_Reg decremented by 1
2581 // CTR is killed
2582 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register owner_addr_Reg, Label& retryLabel) {
2583 Label SpinLoop, doneRetry;
2584 addic_(retry_count_Reg, retry_count_Reg, -1);
2585 blt(CCR0, doneRetry);
2586
2587 if (RTMSpinLoopCount > 1) {
2588 li(R0, RTMSpinLoopCount);
2589 mtctr(R0);
2590 }
2591
2592 bind(SpinLoop);
2593 smt_yield(); // Can't use waitrsv(). No permission (SIGILL).
2594
2595 if (RTMSpinLoopCount > 1) {
2596 bdz(retryLabel);
2597 ld(R0, 0, owner_addr_Reg);
2598 cmpdi(CCR0, R0, 0);
2599 bne(CCR0, SpinLoop);
2600 }
2601
2602 b(retryLabel);
2603
2604 bind(doneRetry);
2605 }
2606
2607 // Use RTM for normal stack locks.
2608 // Input: objReg (object to lock)
2609 void MacroAssembler::rtm_stack_locking(ConditionRegister flag,
2610 Register obj, Register mark_word, Register tmp,
2611 Register retry_on_abort_count_Reg,
2612 RTMLockingCounters* stack_rtm_counters,
2613 Metadata* method_data, bool profile_rtm,
2614 Label& DONE_LABEL, Label& IsInflated) {
2615 assert(UseRTMForStackLocks, "why call this otherwise?");
2616 assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
2617 Label L_rtm_retry, L_decrement_retry, L_on_abort;
2618
2619 if (RTMRetryCount > 0) {
2620 load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
2621 bind(L_rtm_retry);
2622 }
2623 andi_(R0, mark_word, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
2624 bne(CCR0, IsInflated);
2625
2626 if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2627 Label L_noincrement;
2628 if (RTMTotalCountIncrRate > 1) {
2629 branch_on_random_using_tb(tmp, (int)RTMTotalCountIncrRate, L_noincrement);
2630 }
2631 assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
2632 load_const_optimized(tmp, (address)stack_rtm_counters->total_count_addr(), R0);
2633 //atomic_inc_ptr(tmp, /*temp, will be reloaded*/mark_word); We don't increment atomically
2634 ldx(mark_word, tmp);
2635 addi(mark_word, mark_word, 1);
2636 stdx(mark_word, tmp);
2637 bind(L_noincrement);
2638 }
2639 tbegin_();
2640 beq(CCR0, L_on_abort);
2641 ld(mark_word, oopDesc::mark_offset_in_bytes(), obj); // Reload in transaction, conflicts need to be tracked.
2642 andi(R0, mark_word, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
2643 cmpwi(flag, R0, markOopDesc::unlocked_value); // bits = 001 unlocked
2644 beq(flag, DONE_LABEL); // all done if unlocked
2645
2646 if (UseRTMXendForLockBusy) {
2647 tend_();
2648 b(L_decrement_retry);
2649 } else {
2650 tabort_();
2651 }
2652 bind(L_on_abort);
2653 const Register abort_status_Reg = tmp;
2654 mftexasr(abort_status_Reg);
2655 if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2656 rtm_profiling(abort_status_Reg, /*temp*/mark_word, stack_rtm_counters, method_data, profile_rtm);
2657 }
2658 ld(mark_word, oopDesc::mark_offset_in_bytes(), obj); // reload
2659 if (RTMRetryCount > 0) {
2660 // Retry on lock abort if abort status is not permanent.
2661 rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry, &L_decrement_retry);
2662 } else {
2663 bind(L_decrement_retry);
2664 }
2665 }
2666
2667 // Use RTM for inflating locks
2668 // inputs: obj (object to lock)
2669 // mark_word (current header - KILLED)
2670 // boxReg (on-stack box address (displaced header location) - KILLED)
2671 void MacroAssembler::rtm_inflated_locking(ConditionRegister flag,
2672 Register obj, Register mark_word, Register boxReg,
2673 Register retry_on_busy_count_Reg, Register retry_on_abort_count_Reg,
2674 RTMLockingCounters* rtm_counters,
2675 Metadata* method_data, bool profile_rtm,
2676 Label& DONE_LABEL) {
2677 assert(UseRTMLocking, "why call this otherwise?");
2678 Label L_rtm_retry, L_decrement_retry, L_on_abort;
2679 // Clean monitor_value bit to get valid pointer.
2680 int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
2681
2682 // Store non-null, using boxReg instead of (intptr_t)markOopDesc::unused_mark().
2683 std(boxReg, BasicLock::displaced_header_offset_in_bytes(), boxReg);
2684 const Register tmpReg = boxReg;
2685 const Register owner_addr_Reg = mark_word;
2686 addi(owner_addr_Reg, mark_word, owner_offset);
2687
2688 if (RTMRetryCount > 0) {
2689 load_const_optimized(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy.
2690 load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort.
2691 bind(L_rtm_retry);
2692 }
2693 if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2694 Label L_noincrement;
2695 if (RTMTotalCountIncrRate > 1) {
2696 branch_on_random_using_tb(R0, (int)RTMTotalCountIncrRate, L_noincrement);
2697 }
2698 assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
2699 load_const(R0, (address)rtm_counters->total_count_addr(), tmpReg);
2700 //atomic_inc_ptr(R0, tmpReg); We don't increment atomically
2701 ldx(tmpReg, R0);
2702 addi(tmpReg, tmpReg, 1);
2703 stdx(tmpReg, R0);
2704 bind(L_noincrement);
2705 }
2706 tbegin_();
2707 beq(CCR0, L_on_abort);
2708 // We don't reload mark word. Will only be reset at safepoint.
2709 ld(R0, 0, owner_addr_Reg); // Load in transaction, conflicts need to be tracked.
2710 cmpdi(flag, R0, 0);
2711 beq(flag, DONE_LABEL);
2712
2713 if (UseRTMXendForLockBusy) {
2714 tend_();
2715 b(L_decrement_retry);
2716 } else {
2717 tabort_();
2718 }
2719 bind(L_on_abort);
2720 const Register abort_status_Reg = tmpReg;
2721 mftexasr(abort_status_Reg);
2722 if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2723 rtm_profiling(abort_status_Reg, /*temp*/ owner_addr_Reg, rtm_counters, method_data, profile_rtm);
2724 // Restore owner_addr_Reg
2725 ld(mark_word, oopDesc::mark_offset_in_bytes(), obj);
2726 #ifdef ASSERT
2727 andi_(R0, mark_word, markOopDesc::monitor_value);
2728 asm_assert_ne("must be inflated", 0xa754); // Deflating only allowed at safepoint.
2729 #endif
2730 addi(owner_addr_Reg, mark_word, owner_offset);
2731 }
2732 if (RTMRetryCount > 0) {
2733 // Retry on lock abort if abort status is not permanent.
2734 rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
2735 }
2736
2737 // Appears unlocked - try to swing _owner from null to non-null.
2738 cmpxchgd(flag, /*current val*/ R0, (intptr_t)0, /*new val*/ R16_thread, owner_addr_Reg,
2739 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2740 MacroAssembler::cmpxchgx_hint_acquire_lock(), noreg, &L_decrement_retry, true);
2741
2742 if (RTMRetryCount > 0) {
2743 // success done else retry
2744 b(DONE_LABEL);
2745 bind(L_decrement_retry);
2746 // Spin and retry if lock is busy.
2747 rtm_retry_lock_on_busy(retry_on_busy_count_Reg, owner_addr_Reg, L_rtm_retry);
2748 } else {
2749 bind(L_decrement_retry);
2750 }
2751 }
2752
2753 #endif // INCLUDE_RTM_OPT
2754
2755 // "The box" is the space on the stack where we copy the object mark.
2756 void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
2757 Register temp, Register displaced_header, Register current_header,
2758 bool try_bias,
2759 RTMLockingCounters* rtm_counters,
2760 RTMLockingCounters* stack_rtm_counters,
2761 Metadata* method_data,
2762 bool use_rtm, bool profile_rtm) {
2763 assert_different_registers(oop, box, temp, displaced_header, current_header);
2764 assert(flag != CCR0, "bad condition register");
2765 Label cont;
2766 Label object_has_monitor;
2767 Label cas_failed;
2768
2769 // Load markOop from object into displaced_header.
2770 ld(displaced_header, oopDesc::mark_offset_in_bytes(), oop);
2771
2772
2773 // Always do locking in runtime.
2774 if (EmitSync & 0x01) {
2775 cmpdi(flag, oop, 0); // Oop can't be 0 here => always false.
2776 return;
2777 }
2778
2779 if (try_bias) {
2780 biased_locking_enter(flag, oop, displaced_header, temp, current_header, cont);
2781 }
2782
2783 #if INCLUDE_RTM_OPT
2784 if (UseRTMForStackLocks && use_rtm) {
2785 rtm_stack_locking(flag, oop, displaced_header, temp, /*temp*/ current_header,
2786 stack_rtm_counters, method_data, profile_rtm,
2787 cont, object_has_monitor);
2788 }
2789 #endif // INCLUDE_RTM_OPT
2790
2791 // Handle existing monitor.
2792 if ((EmitSync & 0x02) == 0) {
2793 // The object has an existing monitor iff (mark & monitor_value) != 0.
2794 andi_(temp, displaced_header, markOopDesc::monitor_value);
2795 bne(CCR0, object_has_monitor);
2796 }
2797
2798 // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
2799 ori(displaced_header, displaced_header, markOopDesc::unlocked_value);
2800
2801 // Load Compare Value application register.
2802
2803 // Initialize the box. (Must happen before we update the object mark!)
2804 std(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2805
2806 // Must fence, otherwise, preceding store(s) may float below cmpxchg.
2807 // Compare object markOop with mark and if equal exchange scratch1 with object markOop.
2808 cmpxchgd(/*flag=*/flag,
2809 /*current_value=*/current_header,
2810 /*compare_value=*/displaced_header,
2811 /*exchange_value=*/box,
2812 /*where=*/oop,
2813 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2814 MacroAssembler::cmpxchgx_hint_acquire_lock(),
2815 noreg,
2816 &cas_failed,
2817 /*check without membar and ldarx first*/true);
2818 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2819
2820 // If the compare-and-exchange succeeded, then we found an unlocked
2821 // object and we have now locked it.
2822 b(cont);
2823
2824 bind(cas_failed);
2825 // We did not see an unlocked object so try the fast recursive case.
2826
2827 // Check if the owner is self by comparing the value in the markOop of object
2828 // (current_header) with the stack pointer.
2829 sub(current_header, current_header, R1_SP);
2830 load_const_optimized(temp, ~(os::vm_page_size()-1) | markOopDesc::lock_mask_in_place);
2831
2832 and_(R0/*==0?*/, current_header, temp);
2833 // If condition is true we are cont and hence we can store 0 as the
2834 // displaced header in the box, which indicates that it is a recursive lock.
2835 mcrf(flag,CCR0);
2836 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2837
2838 // Handle existing monitor.
2839 if ((EmitSync & 0x02) == 0) {
2840 b(cont);
2841
2842 bind(object_has_monitor);
2843 // The object's monitor m is unlocked iff m->owner == NULL,
2844 // otherwise m->owner may contain a thread or a stack address.
2845
2846 #if INCLUDE_RTM_OPT
2847 // Use the same RTM locking code in 32- and 64-bit VM.
2848 if (use_rtm) {
2849 rtm_inflated_locking(flag, oop, displaced_header, box, temp, /*temp*/ current_header,
2850 rtm_counters, method_data, profile_rtm, cont);
2851 } else {
2852 #endif // INCLUDE_RTM_OPT
2853
2854 // Try to CAS m->owner from NULL to current thread.
2855 addi(temp, displaced_header, ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value);
2856 cmpxchgd(/*flag=*/flag,
2857 /*current_value=*/current_header,
2858 /*compare_value=*/(intptr_t)0,
2859 /*exchange_value=*/R16_thread,
2860 /*where=*/temp,
2861 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2862 MacroAssembler::cmpxchgx_hint_acquire_lock());
2863
2864 // Store a non-null value into the box.
2865 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2866
2867 # ifdef ASSERT
2868 bne(flag, cont);
2869 // We have acquired the monitor, check some invariants.
2870 addi(/*monitor=*/temp, temp, -ObjectMonitor::owner_offset_in_bytes());
2871 // Invariant 1: _recursions should be 0.
2872 //assert(ObjectMonitor::recursions_size_in_bytes() == 8, "unexpected size");
2873 asm_assert_mem8_is_zero(ObjectMonitor::recursions_offset_in_bytes(), temp,
2874 "monitor->_recursions should be 0", -1);
2875 // Invariant 2: OwnerIsThread shouldn't be 0.
2876 //assert(ObjectMonitor::OwnerIsThread_size_in_bytes() == 4, "unexpected size");
2877 //asm_assert_mem4_isnot_zero(ObjectMonitor::OwnerIsThread_offset_in_bytes(), temp,
2878 // "monitor->OwnerIsThread shouldn't be 0", -1);
2879 # endif
2880
2881 #if INCLUDE_RTM_OPT
2882 } // use_rtm()
2883 #endif
2884 }
2885
2886 bind(cont);
2887 // flag == EQ indicates success
2888 // flag == NE indicates failure
2889 }
2890
2891 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2892 Register temp, Register displaced_header, Register current_header,
2893 bool try_bias, bool use_rtm) {
2894 assert_different_registers(oop, box, temp, displaced_header, current_header);
2895 assert(flag != CCR0, "bad condition register");
2896 Label cont;
2897 Label object_has_monitor;
2898
2899 // Always do locking in runtime.
2900 if (EmitSync & 0x01) {
2901 cmpdi(flag, oop, 0); // Oop can't be 0 here => always false.
2902 return;
2903 }
2904
2905 if (try_bias) {
2906 biased_locking_exit(flag, oop, current_header, cont);
2907 }
2908
2909 #if INCLUDE_RTM_OPT
2910 if (UseRTMForStackLocks && use_rtm) {
2911 assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
2912 Label L_regular_unlock;
2913 ld(current_header, oopDesc::mark_offset_in_bytes(), oop); // fetch markword
2914 andi(R0, current_header, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
2915 cmpwi(flag, R0, markOopDesc::unlocked_value); // bits = 001 unlocked
2916 bne(flag, L_regular_unlock); // else RegularLock
2917 tend_(); // otherwise end...
2918 b(cont); // ... and we're done
2919 bind(L_regular_unlock);
2920 }
2921 #endif
2922
2923 // Find the lock address and load the displaced header from the stack.
2924 ld(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2925
2926 // If the displaced header is 0, we have a recursive unlock.
2927 cmpdi(flag, displaced_header, 0);
2928 beq(flag, cont);
2929
2930 // Handle existing monitor.
2931 if ((EmitSync & 0x02) == 0) {
2932 // The object has an existing monitor iff (mark & monitor_value) != 0.
2933 RTM_OPT_ONLY( if (!(UseRTMForStackLocks && use_rtm)) ) // skip load if already done
2934 ld(current_header, oopDesc::mark_offset_in_bytes(), oop);
2935 andi_(R0, current_header, markOopDesc::monitor_value);
2936 bne(CCR0, object_has_monitor);
2937 }
2938
2939 // Check if it is still a light weight lock, this is is true if we see
2940 // the stack address of the basicLock in the markOop of the object.
2941 // Cmpxchg sets flag to cmpd(current_header, box).
2942 cmpxchgd(/*flag=*/flag,
2943 /*current_value=*/current_header,
2944 /*compare_value=*/box,
2945 /*exchange_value=*/displaced_header,
2946 /*where=*/oop,
2947 MacroAssembler::MemBarRel,
2948 MacroAssembler::cmpxchgx_hint_release_lock(),
2949 noreg,
2950 &cont);
2951
2952 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2953
2954 // Handle existing monitor.
2955 if ((EmitSync & 0x02) == 0) {
2956 b(cont);
2957
2958 bind(object_has_monitor);
2959 addi(current_header, current_header, -markOopDesc::monitor_value); // monitor
2960 ld(temp, ObjectMonitor::owner_offset_in_bytes(), current_header);
2961
2962 // It's inflated.
2963 #if INCLUDE_RTM_OPT
2964 if (use_rtm) {
2965 Label L_regular_inflated_unlock;
2966 // Clean monitor_value bit to get valid pointer
2967 cmpdi(flag, temp, 0);
2968 bne(flag, L_regular_inflated_unlock);
2969 tend_();
2970 b(cont);
2971 bind(L_regular_inflated_unlock);
2972 }
2973 #endif
2974
2975 ld(displaced_header, ObjectMonitor::recursions_offset_in_bytes(), current_header);
2976 xorr(temp, R16_thread, temp); // Will be 0 if we are the owner.
2977 orr(temp, temp, displaced_header); // Will be 0 if there are 0 recursions.
2978 cmpdi(flag, temp, 0);
2979 bne(flag, cont);
2980
2981 ld(temp, ObjectMonitor::EntryList_offset_in_bytes(), current_header);
2982 ld(displaced_header, ObjectMonitor::cxq_offset_in_bytes(), current_header);
2983 orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2984 cmpdi(flag, temp, 0);
2985 bne(flag, cont);
2986 release();
2987 std(temp, ObjectMonitor::owner_offset_in_bytes(), current_header);
2988 }
2989
2990 bind(cont);
2991 // flag == EQ indicates success
2992 // flag == NE indicates failure
2993 }
2994
2995 // Write serialization page so VM thread can do a pseudo remote membar.
2996 // We use the current thread pointer to calculate a thread specific
2997 // offset to write to within the page. This minimizes bus traffic
2998 // due to cache line collision.
2999 void MacroAssembler::serialize_memory(Register thread, Register tmp1, Register tmp2) {
3000 srdi(tmp2, thread, os::get_serialize_page_shift_count());
3001
3002 int mask = os::vm_page_size() - sizeof(int);
3003 if (Assembler::is_simm(mask, 16)) {
3004 andi(tmp2, tmp2, mask);
3005 } else {
3006 lis(tmp1, (int)((signed short) (mask >> 16)));
3007 ori(tmp1, tmp1, mask & 0x0000ffff);
3008 andr(tmp2, tmp2, tmp1);
3009 }
3010
3011 load_const(tmp1, (long) os::get_memory_serialize_page());
3012 release();
3013 stwx(R0, tmp1, tmp2);
3014 }
3015
3016
3017 // GC barrier helper macros
3018
3019 // Write the card table byte if needed.
3020 void MacroAssembler::card_write_barrier_post(Register Rstore_addr, Register Rnew_val, Register Rtmp) {
3021 CardTableModRefBS* bs =
3022 barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
3023 assert(bs->kind() == BarrierSet::CardTableForRS ||
3024 bs->kind() == BarrierSet::CardTableExtension, "wrong barrier");
3025 #ifdef ASSERT
3026 cmpdi(CCR0, Rnew_val, 0);
3027 asm_assert_ne("null oop not allowed", 0x321);
3028 #endif
3029 card_table_write(bs->byte_map_base, Rtmp, Rstore_addr);
3030 }
3031
3032 // Write the card table byte.
3033 void MacroAssembler::card_table_write(jbyte* byte_map_base, Register Rtmp, Register Robj) {
3034 assert_different_registers(Robj, Rtmp, R0);
3035 load_const_optimized(Rtmp, (address)byte_map_base, R0);
3036 srdi(Robj, Robj, CardTableModRefBS::card_shift);
3037 li(R0, 0); // dirty
3038 if (UseConcMarkSweepGC) membar(Assembler::StoreStore);
3039 stbx(R0, Rtmp, Robj);
3040 }
3041
3042 // Kills R31 if value is a volatile register.
3043 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2, bool needs_frame) {
3044 Label done;
3045 cmpdi(CCR0, value, 0);
3046 beq(CCR0, done); // Use NULL as-is.
3047
3048 clrrdi(tmp1, value, JNIHandles::weak_tag_size);
3049 #if INCLUDE_ALL_GCS
3050 if (UseG1GC) { andi_(tmp2, value, JNIHandles::weak_tag_mask); }
3051 #endif
3052 ld(value, 0, tmp1); // Resolve (untagged) jobject.
3053
3054 #if INCLUDE_ALL_GCS
3055 if (UseG1GC) {
3056 Label not_weak;
3057 beq(CCR0, not_weak); // Test for jweak tag.
3058 verify_oop(value);
3059 g1_write_barrier_pre(noreg, // obj
3060 noreg, // offset
3061 value, // pre_val
3062 tmp1, tmp2, needs_frame);
3063 bind(not_weak);
3064 }
3065 #endif // INCLUDE_ALL_GCS
3066 verify_oop(value);
3067 bind(done);
3068 }
3069
3070 #if INCLUDE_ALL_GCS
3071 // General G1 pre-barrier generator.
3072 // Goal: record the previous value if it is not null.
3073 void MacroAssembler::g1_write_barrier_pre(Register Robj, RegisterOrConstant offset, Register Rpre_val,
3074 Register Rtmp1, Register Rtmp2, bool needs_frame) {
3075 Label runtime, filtered;
3076
3077 // Is marking active?
3078 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
3079 lwz(Rtmp1, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active()), R16_thread);
3080 } else {
3081 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
3082 lbz(Rtmp1, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active()), R16_thread);
3083 }
3084 cmpdi(CCR0, Rtmp1, 0);
3085 beq(CCR0, filtered);
3086
3087 // Do we need to load the previous value?
3088 if (Robj != noreg) {
3089 // Load the previous value...
3090 if (UseCompressedOops) {
3091 lwz(Rpre_val, offset, Robj);
3092 } else {
3093 ld(Rpre_val, offset, Robj);
3094 }
3095 // Previous value has been loaded into Rpre_val.
3096 }
3097 assert(Rpre_val != noreg, "must have a real register");
3098
3099 // Is the previous value null?
3100 cmpdi(CCR0, Rpre_val, 0);
3101 beq(CCR0, filtered);
3102
3103 if (Robj != noreg && UseCompressedOops) {
3104 decode_heap_oop_not_null(Rpre_val);
3105 }
3106
3107 // OK, it's not filtered, so we'll need to call enqueue. In the normal
3108 // case, pre_val will be a scratch G-reg, but there are some cases in
3109 // which it's an O-reg. In the first case, do a normal call. In the
3110 // latter, do a save here and call the frameless version.
3111
3112 // Can we store original value in the thread's buffer?
3113 // Is index == 0?
3114 // (The index field is typed as size_t.)
3115 const Register Rbuffer = Rtmp1, Rindex = Rtmp2;
3116
3117 ld(Rindex, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index()), R16_thread);
3118 cmpdi(CCR0, Rindex, 0);
3119 beq(CCR0, runtime); // If index == 0, goto runtime.
3120 ld(Rbuffer, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf()), R16_thread);
3121
3122 addi(Rindex, Rindex, -wordSize); // Decrement index.
3123 std(Rindex, in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index()), R16_thread);
3124
3125 // Record the previous value.
3126 stdx(Rpre_val, Rbuffer, Rindex);
3127 b(filtered);
3128
3129 bind(runtime);
3130
3131 // May need to preserve LR. Also needed if current frame is not compatible with C calling convention.
3132 if (needs_frame) {
3133 save_LR_CR(Rtmp1);
3134 push_frame_reg_args(0, Rtmp2);
3135 }
3136
3137 if (Rpre_val->is_volatile() && Robj == noreg) mr(R31, Rpre_val); // Save pre_val across C call if it was preloaded.
3138 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), Rpre_val, R16_thread);
3139 if (Rpre_val->is_volatile() && Robj == noreg) mr(Rpre_val, R31); // restore
3140
3141 if (needs_frame) {
3142 pop_frame();
3143 restore_LR_CR(Rtmp1);
3144 }
3145
3146 bind(filtered);
3147 }
3148
3149 // General G1 post-barrier generator
3150 // Store cross-region card.
3151 void MacroAssembler::g1_write_barrier_post(Register Rstore_addr, Register Rnew_val, Register Rtmp1, Register Rtmp2, Register Rtmp3, Label *filtered_ext) {
3152 Label runtime, filtered_int;
3153 Label& filtered = (filtered_ext != NULL) ? *filtered_ext : filtered_int;
3154 assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2);
3155
3156 G1SATBCardTableLoggingModRefBS* bs =
3157 barrier_set_cast<G1SATBCardTableLoggingModRefBS>(Universe::heap()->barrier_set());
3158
3159 // Does store cross heap regions?
3160 if (G1RSBarrierRegionFilter) {
3161 xorr(Rtmp1, Rstore_addr, Rnew_val);
3162 srdi_(Rtmp1, Rtmp1, HeapRegion::LogOfHRGrainBytes);
3163 beq(CCR0, filtered);
3164 }
3165
3166 // Crosses regions, storing NULL?
3167 #ifdef ASSERT
3168 cmpdi(CCR0, Rnew_val, 0);
3169 asm_assert_ne("null oop not allowed (G1)", 0x322); // Checked by caller on PPC64, so following branch is obsolete:
3170 //beq(CCR0, filtered);
3171 #endif
3172
3173 // Storing region crossing non-NULL, is card already dirty?
3174 assert(sizeof(*bs->byte_map_base) == sizeof(jbyte), "adjust this code");
3175 const Register Rcard_addr = Rtmp1;
3176 Register Rbase = Rtmp2;
3177 load_const_optimized(Rbase, (address)bs->byte_map_base, /*temp*/ Rtmp3);
3178
3179 srdi(Rcard_addr, Rstore_addr, CardTableModRefBS::card_shift);
3180
3181 // Get the address of the card.
3182 lbzx(/*card value*/ Rtmp3, Rbase, Rcard_addr);
3183 cmpwi(CCR0, Rtmp3, (int)G1SATBCardTableModRefBS::g1_young_card_val());
3184 beq(CCR0, filtered);
3185
3186 membar(Assembler::StoreLoad);
3187 lbzx(/*card value*/ Rtmp3, Rbase, Rcard_addr); // Reload after membar.
3188 cmpwi(CCR0, Rtmp3 /* card value */, CardTableModRefBS::dirty_card_val());
3189 beq(CCR0, filtered);
3190
3191 // Storing a region crossing, non-NULL oop, card is clean.
3192 // Dirty card and log.
3193 li(Rtmp3, CardTableModRefBS::dirty_card_val());
3194 //release(); // G1: oops are allowed to get visible after dirty marking.
3195 stbx(Rtmp3, Rbase, Rcard_addr);
3196
3197 add(Rcard_addr, Rbase, Rcard_addr); // This is the address which needs to get enqueued.
3198 Rbase = noreg; // end of lifetime
3199
3200 const Register Rqueue_index = Rtmp2,
3201 Rqueue_buf = Rtmp3;
3202 ld(Rqueue_index, in_bytes(JavaThread::dirty_card_queue_offset() + DirtyCardQueue::byte_offset_of_index()), R16_thread);
3203 cmpdi(CCR0, Rqueue_index, 0);
3204 beq(CCR0, runtime); // index == 0 then jump to runtime
3205 ld(Rqueue_buf, in_bytes(JavaThread::dirty_card_queue_offset() + DirtyCardQueue::byte_offset_of_buf()), R16_thread);
3206
3207 addi(Rqueue_index, Rqueue_index, -wordSize); // decrement index
3208 std(Rqueue_index, in_bytes(JavaThread::dirty_card_queue_offset() + DirtyCardQueue::byte_offset_of_index()), R16_thread);
3209
3210 stdx(Rcard_addr, Rqueue_buf, Rqueue_index); // store card
3211 b(filtered);
3212
3213 bind(runtime);
3214
3215 // Save the live input values.
3216 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), Rcard_addr, R16_thread);
3217
3218 bind(filtered_int);
3219 }
3220 #endif // INCLUDE_ALL_GCS
3221
3222 // Values for last_Java_pc, and last_Java_sp must comply to the rules
3223 // in frame_ppc.hpp.
3224 void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Java_pc) {
3225 // Always set last_Java_pc and flags first because once last_Java_sp
3226 // is visible has_last_Java_frame is true and users will look at the
3227 // rest of the fields. (Note: flags should always be zero before we
3228 // get here so doesn't need to be set.)
3229
3230 // Verify that last_Java_pc was zeroed on return to Java
3231 asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_pc_offset()), R16_thread,
3232 "last_Java_pc not zeroed before leaving Java", 0x200);
3233
3234 // When returning from calling out from Java mode the frame anchor's
3235 // last_Java_pc will always be set to NULL. It is set here so that
3236 // if we are doing a call to native (not VM) that we capture the
3237 // known pc and don't have to rely on the native call having a
3238 // standard frame linkage where we can find the pc.
3239 if (last_Java_pc != noreg)
3240 std(last_Java_pc, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
3241
3242 // Set last_Java_sp last.
3243 std(last_Java_sp, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
3244 }
3245
3246 void MacroAssembler::reset_last_Java_frame(void) {
3247 asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()),
3248 R16_thread, "SP was not set, still zero", 0x202);
3249
3250 BLOCK_COMMENT("reset_last_Java_frame {");
3251 li(R0, 0);
3252
3253 // _last_Java_sp = 0
3254 std(R0, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
3255
3256 // _last_Java_pc = 0
3257 std(R0, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
3258 BLOCK_COMMENT("} reset_last_Java_frame");
3259 }
3260
3261 void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) {
3262 assert_different_registers(sp, tmp1);
3263
3264 // sp points to a TOP_IJAVA_FRAME, retrieve frame's PC via
3265 // TOP_IJAVA_FRAME_ABI.
3266 // FIXME: assert that we really have a TOP_IJAVA_FRAME here!
3267 address entry = pc();
3268 load_const_optimized(tmp1, entry);
3269
3270 set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1);
3271 }
3272
3273 void MacroAssembler::get_vm_result(Register oop_result) {
3274 // Read:
3275 // R16_thread
3276 // R16_thread->in_bytes(JavaThread::vm_result_offset())
3277 //
3278 // Updated:
3279 // oop_result
3280 // R16_thread->in_bytes(JavaThread::vm_result_offset())
3281
3282 verify_thread();
3283
3284 ld(oop_result, in_bytes(JavaThread::vm_result_offset()), R16_thread);
3285 li(R0, 0);
3286 std(R0, in_bytes(JavaThread::vm_result_offset()), R16_thread);
3287
3288 verify_oop(oop_result);
3289 }
3290
3291 void MacroAssembler::get_vm_result_2(Register metadata_result) {
3292 // Read:
3293 // R16_thread
3294 // R16_thread->in_bytes(JavaThread::vm_result_2_offset())
3295 //
3296 // Updated:
3297 // metadata_result
3298 // R16_thread->in_bytes(JavaThread::vm_result_2_offset())
3299
3300 ld(metadata_result, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
3301 li(R0, 0);
3302 std(R0, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
3303 }
3304
3305 Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
3306 Register current = (src != noreg) ? src : dst; // Klass is in dst if no src provided.
3307 if (Universe::narrow_klass_base() != 0) {
3308 // Use dst as temp if it is free.
3309 sub_const_optimized(dst, current, Universe::narrow_klass_base(), R0);
3310 current = dst;
3311 }
3312 if (Universe::narrow_klass_shift() != 0) {
3313 srdi(dst, current, Universe::narrow_klass_shift());
3314 current = dst;
3315 }
3316 return current;
3317 }
3318
3319 void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
3320 if (UseCompressedClassPointers) {
3321 Register compressedKlass = encode_klass_not_null(ck, klass);
3322 stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
3323 } else {
3324 std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
3325 }
3326 }
3327
3328 void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
3329 if (UseCompressedClassPointers) {
3330 if (val == noreg) {
3331 val = R0;
3332 li(val, 0);
3333 }
3334 stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed
3335 }
3336 }
3337
3338 int MacroAssembler::instr_size_for_decode_klass_not_null() {
3339 if (!UseCompressedClassPointers) return 0;
3340 int num_instrs = 1; // shift or move
3341 if (Universe::narrow_klass_base() != 0) num_instrs = 7; // shift + load const + add
3342 return num_instrs * BytesPerInstWord;
3343 }
3344
3345 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3346 assert(dst != R0, "Dst reg may not be R0, as R0 is used here.");
3347 if (src == noreg) src = dst;
3348 Register shifted_src = src;
3349 if (Universe::narrow_klass_shift() != 0 ||
3350 Universe::narrow_klass_base() == 0 && src != dst) { // Move required.
3351 shifted_src = dst;
3352 sldi(shifted_src, src, Universe::narrow_klass_shift());
3353 }
3354 if (Universe::narrow_klass_base() != 0) {
3355 add_const_optimized(dst, shifted_src, Universe::narrow_klass_base(), R0);
3356 }
3357 }
3358
3359 void MacroAssembler::load_klass(Register dst, Register src) {
3360 if (UseCompressedClassPointers) {
3361 lwz(dst, oopDesc::klass_offset_in_bytes(), src);
3362 // Attention: no null check here!
3363 decode_klass_not_null(dst, dst);
3364 } else {
3365 ld(dst, oopDesc::klass_offset_in_bytes(), src);
3366 }
3367 }
3368
3369 void MacroAssembler::load_mirror_from_const_method(Register mirror, Register const_method) {
3370 ld(mirror, in_bytes(ConstMethod::constants_offset()), const_method);
3371 ld(mirror, ConstantPool::pool_holder_offset_in_bytes(), mirror);
3372 ld(mirror, in_bytes(Klass::java_mirror_offset()), mirror);
3373 }
3374
3375 // Clear Array
3376 // For very short arrays. tmp == R0 is allowed.
3377 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3378 if (cnt_dwords > 0) { li(tmp, 0); }
3379 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3380 }
3381
3382 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3383 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3384 if (cnt_dwords < 8) {
3385 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3386 return;
3387 }
3388
3389 Label loop;
3390 const long loopcnt = cnt_dwords >> 1,
3391 remainder = cnt_dwords & 1;
3392
3393 li(tmp, loopcnt);
3394 mtctr(tmp);
3395 li(tmp, 0);
3396 bind(loop);
3397 std(tmp, 0, base_ptr);
3398 std(tmp, 8, base_ptr);
3399 addi(base_ptr, base_ptr, 16);
3400 bdnz(loop);
3401 if (remainder) { std(tmp, 0, base_ptr); }
3402 }
3403
3404 // Kills both input registers. tmp == R0 is allowed.
3405 void MacroAssembler::clear_memory_doubleword(Register base_ptr, Register cnt_dwords, Register tmp, long const_cnt) {
3406 // Procedure for large arrays (uses data cache block zero instruction).
3407 Label startloop, fast, fastloop, small_rest, restloop, done;
3408 const int cl_size = VM_Version::L1_data_cache_line_size(),
3409 cl_dwords = cl_size >> 3,
3410 cl_dw_addr_bits = exact_log2(cl_dwords),
3411 dcbz_min = 1, // Min count of dcbz executions, needs to be >0.
3412 min_cnt = ((dcbz_min + 1) << cl_dw_addr_bits) - 1;
3413
3414 if (const_cnt >= 0) {
3415 // Constant case.
3416 if (const_cnt < min_cnt) {
3417 clear_memory_constlen(base_ptr, const_cnt, tmp);
3418 return;
3419 }
3420 load_const_optimized(cnt_dwords, const_cnt, tmp);
3421 } else {
3422 // cnt_dwords already loaded in register. Need to check size.
3423 cmpdi(CCR1, cnt_dwords, min_cnt); // Big enough? (ensure >= dcbz_min lines included).
3424 blt(CCR1, small_rest);
3425 }
3426 rldicl_(tmp, base_ptr, 64-3, 64-cl_dw_addr_bits); // Extract dword offset within first cache line.
3427 beq(CCR0, fast); // Already 128byte aligned.
3428
3429 subfic(tmp, tmp, cl_dwords);
3430 mtctr(tmp); // Set ctr to hit 128byte boundary (0<ctr<cl_dwords).
3431 subf(cnt_dwords, tmp, cnt_dwords); // rest.
3432 li(tmp, 0);
3433
3434 bind(startloop); // Clear at the beginning to reach 128byte boundary.
3435 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3436 addi(base_ptr, base_ptr, 8);
3437 bdnz(startloop);
3438
3439 bind(fast); // Clear 128byte blocks.
3440 srdi(tmp, cnt_dwords, cl_dw_addr_bits); // Loop count for 128byte loop (>0).
3441 andi(cnt_dwords, cnt_dwords, cl_dwords-1); // Rest in dwords.
3442 mtctr(tmp); // Load counter.
3443
3444 bind(fastloop);
3445 dcbz(base_ptr); // Clear 128byte aligned block.
3446 addi(base_ptr, base_ptr, cl_size);
3447 bdnz(fastloop);
3448
3449 bind(small_rest);
3450 cmpdi(CCR0, cnt_dwords, 0); // size 0?
3451 beq(CCR0, done); // rest == 0
3452 li(tmp, 0);
3453 mtctr(cnt_dwords); // Load counter.
3454
3455 bind(restloop); // Clear rest.
3456 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3457 addi(base_ptr, base_ptr, 8);
3458 bdnz(restloop);
3459
3460 bind(done);
3461 }
3462
3463 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3464
3465 #ifdef COMPILER2
3466 // Intrinsics for CompactStrings
3467
3468 // Compress char[] to byte[] by compressing 16 bytes at once.
3469 void MacroAssembler::string_compress_16(Register src, Register dst, Register cnt,
3470 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5,
3471 Label& Lfailure) {
3472
3473 const Register tmp0 = R0;
3474 assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5);
3475 Label Lloop, Lslow;
3476
3477 // Check if cnt >= 8 (= 16 bytes)
3478 lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF00FF00FF
3479 srwi_(tmp2, cnt, 3);
3480 beq(CCR0, Lslow);
3481 ori(tmp1, tmp1, 0xFF);
3482 rldimi(tmp1, tmp1, 32, 0);
3483 mtctr(tmp2);
3484
3485 // 2x unrolled loop
3486 bind(Lloop);
3487 ld(tmp2, 0, src); // _0_1_2_3 (Big Endian)
3488 ld(tmp4, 8, src); // _4_5_6_7
3489
3490 orr(tmp0, tmp2, tmp4);
3491 rldicl(tmp3, tmp2, 6*8, 64-24); // _____1_2
3492 rldimi(tmp2, tmp2, 2*8, 2*8); // _0_2_3_3
3493 rldicl(tmp5, tmp4, 6*8, 64-24); // _____5_6
3494 rldimi(tmp4, tmp4, 2*8, 2*8); // _4_6_7_7
3495
3496 andc_(tmp0, tmp0, tmp1);
3497 bne(CCR0, Lfailure); // Not latin1.
3498 addi(src, src, 16);
3499
3500 rlwimi(tmp3, tmp2, 0*8, 24, 31);// _____1_3
3501 srdi(tmp2, tmp2, 3*8); // ____0_2_
3502 rlwimi(tmp5, tmp4, 0*8, 24, 31);// _____5_7
3503 srdi(tmp4, tmp4, 3*8); // ____4_6_
3504
3505 orr(tmp2, tmp2, tmp3); // ____0123
3506 orr(tmp4, tmp4, tmp5); // ____4567
3507
3508 stw(tmp2, 0, dst);
3509 stw(tmp4, 4, dst);
3510 addi(dst, dst, 8);
3511 bdnz(Lloop);
3512
3513 bind(Lslow); // Fallback to slow version
3514 }
3515
3516 // Compress char[] to byte[]. cnt must be positive int.
3517 void MacroAssembler::string_compress(Register src, Register dst, Register cnt, Register tmp, Label& Lfailure) {
3518 Label Lloop;
3519 mtctr(cnt);
3520
3521 bind(Lloop);
3522 lhz(tmp, 0, src);
3523 cmplwi(CCR0, tmp, 0xff);
3524 bgt(CCR0, Lfailure); // Not latin1.
3525 addi(src, src, 2);
3526 stb(tmp, 0, dst);
3527 addi(dst, dst, 1);
3528 bdnz(Lloop);
3529 }
3530
3531 // Inflate byte[] to char[] by inflating 16 bytes at once.
3532 void MacroAssembler::string_inflate_16(Register src, Register dst, Register cnt,
3533 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
3534 const Register tmp0 = R0;
3535 assert_different_registers(src, dst, cnt, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5);
3536 Label Lloop, Lslow;
3537
3538 // Check if cnt >= 8
3539 srwi_(tmp2, cnt, 3);
3540 beq(CCR0, Lslow);
3541 lis(tmp1, 0xFF); // tmp1 = 0x00FF00FF
3542 ori(tmp1, tmp1, 0xFF);
3543 mtctr(tmp2);
3544
3545 // 2x unrolled loop
3546 bind(Lloop);
3547 lwz(tmp2, 0, src); // ____0123 (Big Endian)
3548 lwz(tmp4, 4, src); // ____4567
3549 addi(src, src, 8);
3550
3551 rldicl(tmp3, tmp2, 7*8, 64-8); // _______2
3552 rlwimi(tmp2, tmp2, 3*8, 16, 23);// ____0113
3553 rldicl(tmp5, tmp4, 7*8, 64-8); // _______6
3554 rlwimi(tmp4, tmp4, 3*8, 16, 23);// ____4557
3555
3556 andc(tmp0, tmp2, tmp1); // ____0_1_
3557 rlwimi(tmp2, tmp3, 2*8, 0, 23); // _____2_3
3558 andc(tmp3, tmp4, tmp1); // ____4_5_
3559 rlwimi(tmp4, tmp5, 2*8, 0, 23); // _____6_7
3560
3561 rldimi(tmp2, tmp0, 3*8, 0*8); // _0_1_2_3
3562 rldimi(tmp4, tmp3, 3*8, 0*8); // _4_5_6_7
3563
3564 std(tmp2, 0, dst);
3565 std(tmp4, 8, dst);
3566 addi(dst, dst, 16);
3567 bdnz(Lloop);
3568
3569 bind(Lslow); // Fallback to slow version
3570 }
3571
3572 // Inflate byte[] to char[]. cnt must be positive int.
3573 void MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) {
3574 Label Lloop;
3575 mtctr(cnt);
3576
3577 bind(Lloop);
3578 lbz(tmp, 0, src);
3579 addi(src, src, 1);
3580 sth(tmp, 0, dst);
3581 addi(dst, dst, 2);
3582 bdnz(Lloop);
3583 }
3584
3585 void MacroAssembler::string_compare(Register str1, Register str2,
3586 Register cnt1, Register cnt2,
3587 Register tmp1, Register result, int ae) {
3588 const Register tmp0 = R0,
3589 diff = tmp1;
3590
3591 assert_different_registers(str1, str2, cnt1, cnt2, tmp0, tmp1, result);
3592 Label Ldone, Lslow, Lloop, Lreturn_diff;
3593
3594 // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a)
3595 // we interchange str1 and str2 in the UL case and negate the result.
3596 // Like this, str1 is always latin1 encoded, except for the UU case.
3597 // In addition, we need 0 (or sign which is 0) extend.
3598
3599 if (ae == StrIntrinsicNode::UU) {
3600 srwi(cnt1, cnt1, 1);
3601 } else {
3602 clrldi(cnt1, cnt1, 32);
3603 }
3604
3605 if (ae != StrIntrinsicNode::LL) {
3606 srwi(cnt2, cnt2, 1);
3607 } else {
3608 clrldi(cnt2, cnt2, 32);
3609 }
3610
3611 // See if the lengths are different, and calculate min in cnt1.
3612 // Save diff in case we need it for a tie-breaker.
3613 subf_(diff, cnt2, cnt1); // diff = cnt1 - cnt2
3614 // if (diff > 0) { cnt1 = cnt2; }
3615 if (VM_Version::has_isel()) {
3616 isel(cnt1, CCR0, Assembler::greater, /*invert*/ false, cnt2);
3617 } else {
3618 Label Lskip;
3619 blt(CCR0, Lskip);
3620 mr(cnt1, cnt2);
3621 bind(Lskip);
3622 }
3623
3624 // Rename registers
3625 Register chr1 = result;
3626 Register chr2 = tmp0;
3627
3628 // Compare multiple characters in fast loop (only implemented for same encoding).
3629 int stride1 = 8, stride2 = 8;
3630 if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
3631 int log2_chars_per_iter = (ae == StrIntrinsicNode::LL) ? 3 : 2;
3632 Label Lfastloop, Lskipfast;
3633
3634 srwi_(tmp0, cnt1, log2_chars_per_iter);
3635 beq(CCR0, Lskipfast);
3636 rldicl(cnt2, cnt1, 0, 64 - log2_chars_per_iter); // Remaining characters.
3637 li(cnt1, 1 << log2_chars_per_iter); // Initialize for failure case: Rescan characters from current iteration.
3638 mtctr(tmp0);
3639
3640 bind(Lfastloop);
3641 ld(chr1, 0, str1);
3642 ld(chr2, 0, str2);
3643 cmpd(CCR0, chr1, chr2);
3644 bne(CCR0, Lslow);
3645 addi(str1, str1, stride1);
3646 addi(str2, str2, stride2);
3647 bdnz(Lfastloop);
3648 mr(cnt1, cnt2); // Remaining characters.
3649 bind(Lskipfast);
3650 }
3651
3652 // Loop which searches the first difference character by character.
3653 cmpwi(CCR0, cnt1, 0);
3654 beq(CCR0, Lreturn_diff);
3655 bind(Lslow);
3656 mtctr(cnt1);
3657
3658 switch (ae) {
3659 case StrIntrinsicNode::LL: stride1 = 1; stride2 = 1; break;
3660 case StrIntrinsicNode::UL: // fallthru (see comment above)
3661 case StrIntrinsicNode::LU: stride1 = 1; stride2 = 2; break;
3662 case StrIntrinsicNode::UU: stride1 = 2; stride2 = 2; break;
3663 default: ShouldNotReachHere(); break;
3664 }
3665
3666 bind(Lloop);
3667 if (stride1 == 1) { lbz(chr1, 0, str1); } else { lhz(chr1, 0, str1); }
3668 if (stride2 == 1) { lbz(chr2, 0, str2); } else { lhz(chr2, 0, str2); }
3669 subf_(result, chr2, chr1); // result = chr1 - chr2
3670 bne(CCR0, Ldone);
3671 addi(str1, str1, stride1);
3672 addi(str2, str2, stride2);
3673 bdnz(Lloop);
3674
3675 // If strings are equal up to min length, return the length difference.
3676 bind(Lreturn_diff);
3677 mr(result, diff);
3678
3679 // Otherwise, return the difference between the first mismatched chars.
3680 bind(Ldone);
3681 if (ae == StrIntrinsicNode::UL) {
3682 neg(result, result); // Negate result (see note above).
3683 }
3684 }
3685
3686 void MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2,
3687 Register limit, Register tmp1, Register result, bool is_byte) {
3688 const Register tmp0 = R0;
3689 assert_different_registers(ary1, ary2, limit, tmp0, tmp1, result);
3690 Label Ldone, Lskiploop, Lloop, Lfastloop, Lskipfast;
3691 bool limit_needs_shift = false;
3692
3693 if (is_array_equ) {
3694 const int length_offset = arrayOopDesc::length_offset_in_bytes();
3695 const int base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR);
3696
3697 // Return true if the same array.
3698 cmpd(CCR0, ary1, ary2);
3699 beq(CCR0, Lskiploop);
3700
3701 // Return false if one of them is NULL.
3702 cmpdi(CCR0, ary1, 0);
3703 cmpdi(CCR1, ary2, 0);
3704 li(result, 0);
3705 cror(CCR0, Assembler::equal, CCR1, Assembler::equal);
3706 beq(CCR0, Ldone);
3707
3708 // Load the lengths of arrays.
3709 lwz(limit, length_offset, ary1);
3710 lwz(tmp0, length_offset, ary2);
3711
3712 // Return false if the two arrays are not equal length.
3713 cmpw(CCR0, limit, tmp0);
3714 bne(CCR0, Ldone);
3715
3716 // Load array addresses.
3717 addi(ary1, ary1, base_offset);
3718 addi(ary2, ary2, base_offset);
3719 } else {
3720 limit_needs_shift = !is_byte;
3721 li(result, 0); // Assume not equal.
3722 }
3723
3724 // Rename registers
3725 Register chr1 = tmp0;
3726 Register chr2 = tmp1;
3727
3728 // Compare 8 bytes per iteration in fast loop.
3729 const int log2_chars_per_iter = is_byte ? 3 : 2;
3730
3731 srwi_(tmp0, limit, log2_chars_per_iter + (limit_needs_shift ? 1 : 0));
3732 beq(CCR0, Lskipfast);
3733 mtctr(tmp0);
3734
3735 bind(Lfastloop);
3736 ld(chr1, 0, ary1);
3737 ld(chr2, 0, ary2);
3738 addi(ary1, ary1, 8);
3739 addi(ary2, ary2, 8);
3740 cmpd(CCR0, chr1, chr2);
3741 bne(CCR0, Ldone);
3742 bdnz(Lfastloop);
3743
3744 bind(Lskipfast);
3745 rldicl_(limit, limit, limit_needs_shift ? 64 - 1 : 0, 64 - log2_chars_per_iter); // Remaining characters.
3746 beq(CCR0, Lskiploop);
3747 mtctr(limit);
3748
3749 // Character by character.
3750 bind(Lloop);
3751 if (is_byte) {
3752 lbz(chr1, 0, ary1);
3753 lbz(chr2, 0, ary2);
3754 addi(ary1, ary1, 1);
3755 addi(ary2, ary2, 1);
3756 } else {
3757 lhz(chr1, 0, ary1);
3758 lhz(chr2, 0, ary2);
3759 addi(ary1, ary1, 2);
3760 addi(ary2, ary2, 2);
3761 }
3762 cmpw(CCR0, chr1, chr2);
3763 bne(CCR0, Ldone);
3764 bdnz(Lloop);
3765
3766 bind(Lskiploop);
3767 li(result, 1); // All characters are equal.
3768 bind(Ldone);
3769 }
3770
3771 void MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt,
3772 Register needle, ciTypeArray* needle_values, Register needlecnt, int needlecntval,
3773 Register tmp1, Register tmp2, Register tmp3, Register tmp4, int ae) {
3774
3775 // Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite!
3776 Label L_TooShort, L_Found, L_NotFound, L_End;
3777 Register last_addr = haycnt, // Kill haycnt at the beginning.
3778 addr = tmp1,
3779 n_start = tmp2,
3780 ch1 = tmp3,
3781 ch2 = R0;
3782
3783 assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
3784 const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2;
3785 const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1;
3786
3787 // **************************************************************************************************
3788 // Prepare for main loop: optimized for needle count >=2, bail out otherwise.
3789 // **************************************************************************************************
3790
3791 // Compute last haystack addr to use if no match gets found.
3792 clrldi(haycnt, haycnt, 32); // Ensure positive int is valid as 64 bit value.
3793 addi(addr, haystack, -h_csize); // Accesses use pre-increment.
3794 if (needlecntval == 0) { // variable needlecnt
3795 cmpwi(CCR6, needlecnt, 2);
3796 clrldi(needlecnt, needlecnt, 32); // Ensure positive int is valid as 64 bit value.
3797 blt(CCR6, L_TooShort); // Variable needlecnt: handle short needle separately.
3798 }
3799
3800 if (n_csize == 2) { lwz(n_start, 0, needle); } else { lhz(n_start, 0, needle); } // Load first 2 characters of needle.
3801
3802 if (needlecntval == 0) { // variable needlecnt
3803 subf(ch1, needlecnt, haycnt); // Last character index to compare is haycnt-needlecnt.
3804 addi(needlecnt, needlecnt, -2); // Rest of needle.
3805 } else { // constant needlecnt
3806 guarantee(needlecntval != 1, "IndexOf with single-character needle must be handled separately");
3807 assert((needlecntval & 0x7fff) == needlecntval, "wrong immediate");
3808 addi(ch1, haycnt, -needlecntval); // Last character index to compare is haycnt-needlecnt.
3809 if (needlecntval > 3) { li(needlecnt, needlecntval - 2); } // Rest of needle.
3810 }
3811
3812 if (h_csize == 2) { slwi(ch1, ch1, 1); } // Scale to number of bytes.
3813
3814 if (ae ==StrIntrinsicNode::UL) {
3815 srwi(tmp4, n_start, 1*8); // ___0
3816 rlwimi(n_start, tmp4, 2*8, 0, 23); // _0_1
3817 }
3818
3819 add(last_addr, haystack, ch1); // Point to last address to compare (haystack+2*(haycnt-needlecnt)).
3820
3821 // Main Loop (now we have at least 2 characters).
3822 Label L_OuterLoop, L_InnerLoop, L_FinalCheck, L_Comp1, L_Comp2;
3823 bind(L_OuterLoop); // Search for 1st 2 characters.
3824 Register addr_diff = tmp4;
3825 subf(addr_diff, addr, last_addr); // Difference between already checked address and last address to check.
3826 addi(addr, addr, h_csize); // This is the new address we want to use for comparing.
3827 srdi_(ch2, addr_diff, h_csize);
3828 beq(CCR0, L_FinalCheck); // 2 characters left?
3829 mtctr(ch2); // num of characters / 2
3830 bind(L_InnerLoop); // Main work horse (2x unrolled search loop)
3831 if (h_csize == 2) { // Load 2 characters of haystack (ignore alignment).
3832 lwz(ch1, 0, addr);
3833 lwz(ch2, 2, addr);
3834 } else {
3835 lhz(ch1, 0, addr);
3836 lhz(ch2, 1, addr);
3837 }
3838 cmpw(CCR0, ch1, n_start); // Compare 2 characters (1 would be sufficient but try to reduce branches to CompLoop).
3839 cmpw(CCR1, ch2, n_start);
3840 beq(CCR0, L_Comp1); // Did we find the needle start?
3841 beq(CCR1, L_Comp2);
3842 addi(addr, addr, 2 * h_csize);
3843 bdnz(L_InnerLoop);
3844 bind(L_FinalCheck);
3845 andi_(addr_diff, addr_diff, h_csize); // Remaining characters not covered by InnerLoop: (num of characters) & 1.
3846 beq(CCR0, L_NotFound);
3847 if (h_csize == 2) { lwz(ch1, 0, addr); } else { lhz(ch1, 0, addr); } // One position left at which we have to compare.
3848 cmpw(CCR1, ch1, n_start);
3849 beq(CCR1, L_Comp1);
3850 bind(L_NotFound);
3851 li(result, -1); // not found
3852 b(L_End);
3853
3854 // **************************************************************************************************
3855 // Special Case: unfortunately, the variable needle case can be called with needlecnt<2
3856 // **************************************************************************************************
3857 if (needlecntval == 0) { // We have to handle these cases separately.
3858 Label L_OneCharLoop;
3859 bind(L_TooShort);
3860 mtctr(haycnt);
3861 if (n_csize == 2) { lhz(n_start, 0, needle); } else { lbz(n_start, 0, needle); } // First character of needle
3862 bind(L_OneCharLoop);
3863 if (h_csize == 2) { lhzu(ch1, 2, addr); } else { lbzu(ch1, 1, addr); }
3864 cmpw(CCR1, ch1, n_start);
3865 beq(CCR1, L_Found); // Did we find the one character needle?
3866 bdnz(L_OneCharLoop);
3867 li(result, -1); // Not found.
3868 b(L_End);
3869 }
3870
3871 // **************************************************************************************************
3872 // Regular Case Part II: compare rest of needle (first 2 characters have been compared already)
3873 // **************************************************************************************************
3874
3875 // Compare the rest
3876 bind(L_Comp2);
3877 addi(addr, addr, h_csize); // First comparison has failed, 2nd one hit.
3878 bind(L_Comp1); // Addr points to possible needle start.
3879 if (needlecntval != 2) { // Const needlecnt==2?
3880 if (needlecntval != 3) {
3881 if (needlecntval == 0) { beq(CCR6, L_Found); } // Variable needlecnt==2?
3882 Register n_ind = tmp4,
3883 h_ind = n_ind;
3884 li(n_ind, 2 * n_csize); // First 2 characters are already compared, use index 2.
3885 mtctr(needlecnt); // Decremented by 2, still > 0.
3886 Label L_CompLoop;
3887 bind(L_CompLoop);
3888 if (ae ==StrIntrinsicNode::UL) {
3889 h_ind = ch1;
3890 sldi(h_ind, n_ind, 1);
3891 }
3892 if (n_csize == 2) { lhzx(ch2, needle, n_ind); } else { lbzx(ch2, needle, n_ind); }
3893 if (h_csize == 2) { lhzx(ch1, addr, h_ind); } else { lbzx(ch1, addr, h_ind); }
3894 cmpw(CCR1, ch1, ch2);
3895 bne(CCR1, L_OuterLoop);
3896 addi(n_ind, n_ind, n_csize);
3897 bdnz(L_CompLoop);
3898 } else { // No loop required if there's only one needle character left.
3899 if (n_csize == 2) { lhz(ch2, 2 * 2, needle); } else { lbz(ch2, 2 * 1, needle); }
3900 if (h_csize == 2) { lhz(ch1, 2 * 2, addr); } else { lbz(ch1, 2 * 1, addr); }
3901 cmpw(CCR1, ch1, ch2);
3902 bne(CCR1, L_OuterLoop);
3903 }
3904 }
3905 // Return index ...
3906 bind(L_Found);
3907 subf(result, haystack, addr); // relative to haystack, ...
3908 if (h_csize == 2) { srdi(result, result, 1); } // in characters.
3909 bind(L_End);
3910 } // string_indexof
3911
3912 void MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt,
3913 Register needle, jchar needleChar, Register tmp1, Register tmp2, bool is_byte) {
3914 assert_different_registers(haystack, haycnt, needle, tmp1, tmp2);
3915
3916 Label L_InnerLoop, L_FinalCheck, L_Found1, L_Found2, L_NotFound, L_End;
3917 Register addr = tmp1,
3918 ch1 = tmp2,
3919 ch2 = R0;
3920
3921 const int h_csize = is_byte ? 1 : 2;
3922
3923 //4:
3924 srwi_(tmp2, haycnt, 1); // Shift right by exact_log2(UNROLL_FACTOR).
3925 mr(addr, haystack);
3926 beq(CCR0, L_FinalCheck);
3927 mtctr(tmp2); // Move to count register.
3928 //8:
3929 bind(L_InnerLoop); // Main work horse (2x unrolled search loop).
3930 if (!is_byte) {
3931 lhz(ch1, 0, addr);
3932 lhz(ch2, 2, addr);
3933 } else {
3934 lbz(ch1, 0, addr);
3935 lbz(ch2, 1, addr);
3936 }
3937 (needle != R0) ? cmpw(CCR0, ch1, needle) : cmplwi(CCR0, ch1, (unsigned int)needleChar);
3938 (needle != R0) ? cmpw(CCR1, ch2, needle) : cmplwi(CCR1, ch2, (unsigned int)needleChar);
3939 beq(CCR0, L_Found1); // Did we find the needle?
3940 beq(CCR1, L_Found2);
3941 addi(addr, addr, 2 * h_csize);
3942 bdnz(L_InnerLoop);
3943 //16:
3944 bind(L_FinalCheck);
3945 andi_(R0, haycnt, 1);
3946 beq(CCR0, L_NotFound);
3947 if (!is_byte) { lhz(ch1, 0, addr); } else { lbz(ch1, 0, addr); } // One position left at which we have to compare.
3948 (needle != R0) ? cmpw(CCR1, ch1, needle) : cmplwi(CCR1, ch1, (unsigned int)needleChar);
3949 beq(CCR1, L_Found1);
3950 //21:
3951 bind(L_NotFound);
3952 li(result, -1); // Not found.
3953 b(L_End);
3954
3955 bind(L_Found2);
3956 addi(addr, addr, h_csize);
3957 //24:
3958 bind(L_Found1); // Return index ...
3959 subf(result, haystack, addr); // relative to haystack, ...
3960 if (!is_byte) { srdi(result, result, 1); } // in characters.
3961 bind(L_End);
3962 } // string_indexof_char
3963
3964
3965 void MacroAssembler::has_negatives(Register src, Register cnt, Register result,
3966 Register tmp1, Register tmp2) {
3967 const Register tmp0 = R0;
3968 assert_different_registers(src, result, cnt, tmp0, tmp1, tmp2);
3969 Label Lfastloop, Lslow, Lloop, Lnoneg, Ldone;
3970
3971 // Check if cnt >= 8 (= 16 bytes)
3972 lis(tmp1, (int)(short)0x8080); // tmp1 = 0x8080808080808080
3973 srwi_(tmp2, cnt, 4);
3974 li(result, 1); // Assume there's a negative byte.
3975 beq(CCR0, Lslow);
3976 ori(tmp1, tmp1, 0x8080);
3977 rldimi(tmp1, tmp1, 32, 0);
3978 mtctr(tmp2);
3979
3980 // 2x unrolled loop
3981 bind(Lfastloop);
3982 ld(tmp2, 0, src);
3983 ld(tmp0, 8, src);
3984
3985 orr(tmp0, tmp2, tmp0);
3986
3987 and_(tmp0, tmp0, tmp1);
3988 bne(CCR0, Ldone); // Found negative byte.
3989 addi(src, src, 16);
3990
3991 bdnz(Lfastloop);
3992
3993 bind(Lslow); // Fallback to slow version
3994 rldicl_(tmp0, cnt, 0, 64-4);
3995 beq(CCR0, Lnoneg);
3996 mtctr(tmp0);
3997 bind(Lloop);
3998 lbz(tmp0, 0, src);
3999 addi(src, src, 1);
4000 andi_(tmp0, tmp0, 0x80);
4001 bne(CCR0, Ldone); // Found negative byte.
4002 bdnz(Lloop);
4003 bind(Lnoneg);
4004 li(result, 0);
4005
4006 bind(Ldone);
4007 }
4008
4009 #endif // Compiler2
4010
4011 // Helpers for Intrinsic Emitters
4012 //
4013 // Revert the byte order of a 32bit value in a register
4014 // src: 0x44556677
4015 // dst: 0x77665544
4016 // Three steps to obtain the result:
4017 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
4018 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
4019 // This value initializes dst.
4020 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
4021 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
4022 // This value is mask inserted into dst with a [0..23] mask of 1s.
4023 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
4024 // This value is mask inserted into dst with a [8..15] mask of 1s.
4025 void MacroAssembler::load_reverse_32(Register dst, Register src) {
4026 assert_different_registers(dst, src);
4027
4028 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4029 rlwimi(dst, src, 3*8, 0, 23); // Insert byte 5 into position 6, 7 into 4, leave pos 7 alone.
4030 rlwimi(dst, src, 1*8, 8, 15); // Insert byte 6 into position 5, leave the rest alone.
4031 }
4032
4033 // Calculate the column addresses of the crc32 lookup table into distinct registers.
4034 // This loop-invariant calculation is moved out of the loop body, reducing the loop
4035 // body size from 20 to 16 instructions.
4036 // Returns the offset that was used to calculate the address of column tc3.
4037 // Due to register shortage, setting tc3 may overwrite table. With the return offset
4038 // at hand, the original table address can be easily reconstructed.
4039 int MacroAssembler::crc32_table_columns(Register table, Register tc0, Register tc1, Register tc2, Register tc3) {
4040
4041 #ifdef VM_LITTLE_ENDIAN
4042 // This is what we implement (the DOLIT4 part):
4043 // ========================================================================= */
4044 // #define DOLIT4 c ^= *buf4++; \
4045 // c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \
4046 // crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24]
4047 // #define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4
4048 // ========================================================================= */
4049 const int ix0 = 3*(4*CRC32_COLUMN_SIZE);
4050 const int ix1 = 2*(4*CRC32_COLUMN_SIZE);
4051 const int ix2 = 1*(4*CRC32_COLUMN_SIZE);
4052 const int ix3 = 0*(4*CRC32_COLUMN_SIZE);
4053 #else
4054 // This is what we implement (the DOBIG4 part):
4055 // =========================================================================
4056 // #define DOBIG4 c ^= *++buf4; \
4057 // c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \
4058 // crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24]
4059 // #define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4
4060 // =========================================================================
4061 const int ix0 = 4*(4*CRC32_COLUMN_SIZE);
4062 const int ix1 = 5*(4*CRC32_COLUMN_SIZE);
4063 const int ix2 = 6*(4*CRC32_COLUMN_SIZE);
4064 const int ix3 = 7*(4*CRC32_COLUMN_SIZE);
4065 #endif
4066 assert_different_registers(table, tc0, tc1, tc2);
4067 assert(table == tc3, "must be!");
4068
4069 addi(tc0, table, ix0);
4070 addi(tc1, table, ix1);
4071 addi(tc2, table, ix2);
4072 if (ix3 != 0) addi(tc3, table, ix3);
4073
4074 return ix3;
4075 }
4076
4077 /**
4078 * uint32_t crc;
4079 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
4080 */
4081 void MacroAssembler::fold_byte_crc32(Register crc, Register val, Register table, Register tmp) {
4082 assert_different_registers(crc, table, tmp);
4083 assert_different_registers(val, table);
4084
4085 if (crc == val) { // Must rotate first to use the unmodified value.
4086 rlwinm(tmp, val, 2, 24-2, 31-2); // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
4087 // As we use a word (4-byte) instruction, we have to adapt the mask bit positions.
4088 srwi(crc, crc, 8); // Unsigned shift, clear leftmost 8 bits.
4089 } else {
4090 srwi(crc, crc, 8); // Unsigned shift, clear leftmost 8 bits.
4091 rlwinm(tmp, val, 2, 24-2, 31-2); // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
4092 }
4093 lwzx(tmp, table, tmp);
4094 xorr(crc, crc, tmp);
4095 }
4096
4097 /**
4098 * uint32_t crc;
4099 * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
4100 */
4101 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
4102 fold_byte_crc32(crc, crc, table, tmp);
4103 }
4104
4105 /**
4106 * Emits code to update CRC-32 with a byte value according to constants in table.
4107 *
4108 * @param [in,out]crc Register containing the crc.
4109 * @param [in]val Register containing the byte to fold into the CRC.
4110 * @param [in]table Register containing the table of crc constants.
4111 *
4112 * uint32_t crc;
4113 * val = crc_table[(val ^ crc) & 0xFF];
4114 * crc = val ^ (crc >> 8);
4115 */
4116 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
4117 BLOCK_COMMENT("update_byte_crc32:");
4118 xorr(val, val, crc);
4119 fold_byte_crc32(crc, val, table, val);
4120 }
4121
4122 /**
4123 * @param crc register containing existing CRC (32-bit)
4124 * @param buf register pointing to input byte buffer (byte*)
4125 * @param len register containing number of bytes
4126 * @param table register pointing to CRC table
4127 */
4128 void MacroAssembler::update_byteLoop_crc32(Register crc, Register buf, Register len, Register table,
4129 Register data, bool loopAlignment) {
4130 assert_different_registers(crc, buf, len, table, data);
4131
4132 Label L_mainLoop, L_done;
4133 const int mainLoop_stepping = 1;
4134 const int mainLoop_alignment = loopAlignment ? 32 : 4; // (InputForNewCode > 4 ? InputForNewCode : 32) : 4;
4135
4136 // Process all bytes in a single-byte loop.
4137 clrldi_(len, len, 32); // Enforce 32 bit. Anything to do?
4138 beq(CCR0, L_done);
4139
4140 mtctr(len);
4141 align(mainLoop_alignment);
4142 BIND(L_mainLoop);
4143 lbz(data, 0, buf); // Byte from buffer, zero-extended.
4144 addi(buf, buf, mainLoop_stepping); // Advance buffer position.
4145 update_byte_crc32(crc, data, table);
4146 bdnz(L_mainLoop); // Iterate.
4147
4148 bind(L_done);
4149 }
4150
4151 /**
4152 * Emits code to update CRC-32 with a 4-byte value according to constants in table
4153 * Implementation according to jdk/src/share/native/java/util/zip/zlib-1.2.8/crc32.c
4154 */
4155 // A not on the lookup table address(es):
4156 // The lookup table consists of two sets of four columns each.
4157 // The columns {0..3} are used for little-endian machines.
4158 // The columns {4..7} are used for big-endian machines.
4159 // To save the effort of adding the column offset to the table address each time
4160 // a table element is looked up, it is possible to pass the pre-calculated
4161 // column addresses.
4162 // Uses R9..R12 as work register. Must be saved/restored by caller, if necessary.
4163 void MacroAssembler::update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc,
4164 Register t0, Register t1, Register t2, Register t3,
4165 Register tc0, Register tc1, Register tc2, Register tc3) {
4166 assert_different_registers(crc, t3);
4167
4168 // XOR crc with next four bytes of buffer.
4169 lwz(t3, bufDisp, buf);
4170 if (bufInc != 0) {
4171 addi(buf, buf, bufInc);
4172 }
4173 xorr(t3, t3, crc);
4174
4175 // Chop crc into 4 single-byte pieces, shifted left 2 bits, to form the table indices.
4176 rlwinm(t0, t3, 2, 24-2, 31-2); // ((t1 >> 0) & 0xff) << 2
4177 rlwinm(t1, t3, 32+(2- 8), 24-2, 31-2); // ((t1 >> 8) & 0xff) << 2
4178 rlwinm(t2, t3, 32+(2-16), 24-2, 31-2); // ((t1 >> 16) & 0xff) << 2
4179 rlwinm(t3, t3, 32+(2-24), 24-2, 31-2); // ((t1 >> 24) & 0xff) << 2
4180
4181 // Use the pre-calculated column addresses.
4182 // Load pre-calculated table values.
4183 lwzx(t0, tc0, t0);
4184 lwzx(t1, tc1, t1);
4185 lwzx(t2, tc2, t2);
4186 lwzx(t3, tc3, t3);
4187
4188 // Calculate new crc from table values.
4189 xorr(t0, t0, t1);
4190 xorr(t2, t2, t3);
4191 xorr(crc, t0, t2); // Now crc contains the final checksum value.
4192 }
4193
4194 /**
4195 * @param crc register containing existing CRC (32-bit)
4196 * @param buf register pointing to input byte buffer (byte*)
4197 * @param len register containing number of bytes
4198 * @param table register pointing to CRC table
4199 *
4200 * Uses R9..R12 as work register. Must be saved/restored by caller!
4201 */
4202 void MacroAssembler::kernel_crc32_2word(Register crc, Register buf, Register len, Register table,
4203 Register t0, Register t1, Register t2, Register t3,
4204 Register tc0, Register tc1, Register tc2, Register tc3,
4205 bool invertCRC) {
4206 assert_different_registers(crc, buf, len, table);
4207
4208 Label L_mainLoop, L_tail;
4209 Register tmp = t0;
4210 Register data = t0;
4211 Register tmp2 = t1;
4212 const int mainLoop_stepping = 8;
4213 const int tailLoop_stepping = 1;
4214 const int log_stepping = exact_log2(mainLoop_stepping);
4215 const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32;
4216 const int complexThreshold = 2*mainLoop_stepping;
4217
4218 // Don't test for len <= 0 here. This pathological case should not occur anyway.
4219 // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles
4220 // for all well-behaved cases. The situation itself is detected and handled correctly
4221 // within update_byteLoop_crc32.
4222 assert(tailLoop_stepping == 1, "check tailLoop_stepping!");
4223
4224 BLOCK_COMMENT("kernel_crc32_2word {");
4225
4226 if (invertCRC) {
4227 nand(crc, crc, crc); // 1s complement of crc
4228 }
4229
4230 // Check for short (<mainLoop_stepping) buffer.
4231 cmpdi(CCR0, len, complexThreshold);
4232 blt(CCR0, L_tail);
4233
4234 // Pre-mainLoop alignment did show a slight (1%) positive effect on performance.
4235 // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions.
4236 {
4237 // Align buf addr to mainLoop_stepping boundary.
4238 neg(tmp2, buf); // Calculate # preLoop iterations for alignment.
4239 rldicl(tmp2, tmp2, 0, 64-log_stepping); // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63.
4240
4241 if (complexThreshold > mainLoop_stepping) {
4242 sub(len, len, tmp2); // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
4243 } else {
4244 sub(tmp, len, tmp2); // Remaining bytes for main loop.
4245 cmpdi(CCR0, tmp, mainLoop_stepping);
4246 blt(CCR0, L_tail); // For less than one mainloop_stepping left, do only tail processing
4247 mr(len, tmp); // remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
4248 }
4249 update_byteLoop_crc32(crc, buf, tmp2, table, data, false);
4250 }
4251
4252 srdi(tmp2, len, log_stepping); // #iterations for mainLoop
4253 andi(len, len, mainLoop_stepping-1); // remaining bytes for tailLoop
4254 mtctr(tmp2);
4255
4256 #ifdef VM_LITTLE_ENDIAN
4257 Register crc_rv = crc;
4258 #else
4259 Register crc_rv = tmp; // Load_reverse needs separate registers to work on.
4260 // Occupies tmp, but frees up crc.
4261 load_reverse_32(crc_rv, crc); // Revert byte order because we are dealing with big-endian data.
4262 tmp = crc;
4263 #endif
4264
4265 int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3);
4266
4267 align(mainLoop_alignment); // Octoword-aligned loop address. Shows 2% improvement.
4268 BIND(L_mainLoop);
4269 update_1word_crc32(crc_rv, buf, table, 0, 0, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3);
4270 update_1word_crc32(crc_rv, buf, table, 4, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3);
4271 bdnz(L_mainLoop);
4272
4273 #ifndef VM_LITTLE_ENDIAN
4274 load_reverse_32(crc, crc_rv); // Revert byte order because we are dealing with big-endian data.
4275 tmp = crc_rv; // Tmp uses it's original register again.
4276 #endif
4277
4278 // Restore original table address for tailLoop.
4279 if (reconstructTableOffset != 0) {
4280 addi(table, table, -reconstructTableOffset);
4281 }
4282
4283 // Process last few (<complexThreshold) bytes of buffer.
4284 BIND(L_tail);
4285 update_byteLoop_crc32(crc, buf, len, table, data, false);
4286
4287 if (invertCRC) {
4288 nand(crc, crc, crc); // 1s complement of crc
4289 }
4290 BLOCK_COMMENT("} kernel_crc32_2word");
4291 }
4292
4293 /**
4294 * @param crc register containing existing CRC (32-bit)
4295 * @param buf register pointing to input byte buffer (byte*)
4296 * @param len register containing number of bytes
4297 * @param table register pointing to CRC table
4298 *
4299 * uses R9..R12 as work register. Must be saved/restored by caller!
4300 */
4301 void MacroAssembler::kernel_crc32_1word(Register crc, Register buf, Register len, Register table,
4302 Register t0, Register t1, Register t2, Register t3,
4303 Register tc0, Register tc1, Register tc2, Register tc3,
4304 bool invertCRC) {
4305 assert_different_registers(crc, buf, len, table);
4306
4307 Label L_mainLoop, L_tail;
4308 Register tmp = t0;
4309 Register data = t0;
4310 Register tmp2 = t1;
4311 const int mainLoop_stepping = 4;
4312 const int tailLoop_stepping = 1;
4313 const int log_stepping = exact_log2(mainLoop_stepping);
4314 const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32;
4315 const int complexThreshold = 2*mainLoop_stepping;
4316
4317 // Don't test for len <= 0 here. This pathological case should not occur anyway.
4318 // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles
4319 // for all well-behaved cases. The situation itself is detected and handled correctly
4320 // within update_byteLoop_crc32.
4321 assert(tailLoop_stepping == 1, "check tailLoop_stepping!");
4322
4323 BLOCK_COMMENT("kernel_crc32_1word {");
4324
4325 if (invertCRC) {
4326 nand(crc, crc, crc); // 1s complement of crc
4327 }
4328
4329 // Check for short (<mainLoop_stepping) buffer.
4330 cmpdi(CCR0, len, complexThreshold);
4331 blt(CCR0, L_tail);
4332
4333 // Pre-mainLoop alignment did show a slight (1%) positive effect on performance.
4334 // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions.
4335 {
4336 // Align buf addr to mainLoop_stepping boundary.
4337 neg(tmp2, buf); // Calculate # preLoop iterations for alignment.
4338 rldicl(tmp2, tmp2, 0, 64-log_stepping); // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63.
4339
4340 if (complexThreshold > mainLoop_stepping) {
4341 sub(len, len, tmp2); // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
4342 } else {
4343 sub(tmp, len, tmp2); // Remaining bytes for main loop.
4344 cmpdi(CCR0, tmp, mainLoop_stepping);
4345 blt(CCR0, L_tail); // For less than one mainloop_stepping left, do only tail processing
4346 mr(len, tmp); // remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
4347 }
4348 update_byteLoop_crc32(crc, buf, tmp2, table, data, false);
4349 }
4350
4351 srdi(tmp2, len, log_stepping); // #iterations for mainLoop
4352 andi(len, len, mainLoop_stepping-1); // remaining bytes for tailLoop
4353 mtctr(tmp2);
4354
4355 #ifdef VM_LITTLE_ENDIAN
4356 Register crc_rv = crc;
4357 #else
4358 Register crc_rv = tmp; // Load_reverse needs separate registers to work on.
4359 // Occupies tmp, but frees up crc.
4360 load_reverse_32(crc_rv, crc); // Revert byte order because we are dealing with big-endian data.
4361 tmp = crc;
4362 #endif
4363
4364 int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3);
4365
4366 align(mainLoop_alignment); // Octoword-aligned loop address. Shows 2% improvement.
4367 BIND(L_mainLoop);
4368 update_1word_crc32(crc_rv, buf, table, 0, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3);
4369 bdnz(L_mainLoop);
4370
4371 #ifndef VM_LITTLE_ENDIAN
4372 load_reverse_32(crc, crc_rv); // Revert byte order because we are dealing with big-endian data.
4373 tmp = crc_rv; // Tmp uses it's original register again.
4374 #endif
4375
4376 // Restore original table address for tailLoop.
4377 if (reconstructTableOffset != 0) {
4378 addi(table, table, -reconstructTableOffset);
4379 }
4380
4381 // Process last few (<complexThreshold) bytes of buffer.
4382 BIND(L_tail);
4383 update_byteLoop_crc32(crc, buf, len, table, data, false);
4384
4385 if (invertCRC) {
4386 nand(crc, crc, crc); // 1s complement of crc
4387 }
4388 BLOCK_COMMENT("} kernel_crc32_1word");
4389 }
4390
4391 /**
4392 * @param crc register containing existing CRC (32-bit)
4393 * @param buf register pointing to input byte buffer (byte*)
4394 * @param len register containing number of bytes
4395 * @param table register pointing to CRC table
4396 *
4397 * Uses R7_ARG5, R8_ARG6 as work registers.
4398 */
4399 void MacroAssembler::kernel_crc32_1byte(Register crc, Register buf, Register len, Register table,
4400 Register t0, Register t1, Register t2, Register t3,
4401 bool invertCRC) {
4402 assert_different_registers(crc, buf, len, table);
4403
4404 Register data = t0; // Holds the current byte to be folded into crc.
4405
4406 BLOCK_COMMENT("kernel_crc32_1byte {");
4407
4408 if (invertCRC) {
4409 nand(crc, crc, crc); // 1s complement of crc
4410 }
4411
4412 // Process all bytes in a single-byte loop.
4413 update_byteLoop_crc32(crc, buf, len, table, data, true);
4414
4415 if (invertCRC) {
4416 nand(crc, crc, crc); // 1s complement of crc
4417 }
4418 BLOCK_COMMENT("} kernel_crc32_1byte");
4419 }
4420
4421 /**
4422 * @param crc register containing existing CRC (32-bit)
4423 * @param buf register pointing to input byte buffer (byte*)
4424 * @param len register containing number of bytes
4425 * @param table register pointing to CRC table
4426 * @param constants register pointing to CRC table for 128-bit aligned memory
4427 * @param barretConstants register pointing to table for barrett reduction
4428 * @param t0 volatile register
4429 * @param t1 volatile register
4430 * @param t2 volatile register
4431 * @param t3 volatile register
4432 */
4433 void MacroAssembler::kernel_crc32_1word_vpmsumd(Register crc, Register buf, Register len, Register table,
4434 Register constants, Register barretConstants,
4435 Register t0, Register t1, Register t2, Register t3, Register t4,
4436 bool invertCRC) {
4437 assert_different_registers(crc, buf, len, table);
4438
4439 Label L_alignedHead, L_tail, L_alignTail, L_start, L_end;
4440
4441 Register prealign = t0;
4442 Register postalign = t0;
4443
4444 BLOCK_COMMENT("kernel_crc32_1word_vpmsumb {");
4445
4446 // 1. use kernel_crc32_1word for shorter than 384bit
4447 clrldi(len, len, 32);
4448 cmpdi(CCR0, len, 384);
4449 bge(CCR0, L_start);
4450
4451 Register tc0 = t4;
4452 Register tc1 = constants;
4453 Register tc2 = barretConstants;
4454 kernel_crc32_1word(crc, buf, len, table,t0, t1, t2, t3, tc0, tc1, tc2, table, invertCRC);
4455 b(L_end);
4456
4457 BIND(L_start);
4458
4459 // 2. ~c
4460 if (invertCRC) {
4461 nand(crc, crc, crc); // 1s complement of crc
4462 }
4463
4464 // 3. calculate from 0 to first 128bit-aligned address
4465 clrldi_(prealign, buf, 57);
4466 beq(CCR0, L_alignedHead);
4467
4468 subfic(prealign, prealign, 128);
4469
4470 subf(len, prealign, len);
4471 update_byteLoop_crc32(crc, buf, prealign, table, t2, false);
4472
4473 // 4. calculate from first 128bit-aligned address to last 128bit-aligned address
4474 BIND(L_alignedHead);
4475
4476 clrldi(postalign, len, 57);
4477 subf(len, postalign, len);
4478
4479 // len must be more than 256bit
4480 kernel_crc32_1word_aligned(crc, buf, len, constants, barretConstants, t1, t2, t3);
4481
4482 // 5. calculate remaining
4483 cmpdi(CCR0, postalign, 0);
4484 beq(CCR0, L_tail);
4485
4486 update_byteLoop_crc32(crc, buf, postalign, table, t2, false);
4487
4488 BIND(L_tail);
4489
4490 // 6. ~c
4491 if (invertCRC) {
4492 nand(crc, crc, crc); // 1s complement of crc
4493 }
4494
4495 BIND(L_end);
4496
4497 BLOCK_COMMENT("} kernel_crc32_1word_vpmsumb");
4498 }
4499
4500 /**
4501 * @param crc register containing existing CRC (32-bit)
4502 * @param buf register pointing to input byte buffer (byte*)
4503 * @param len register containing number of bytes
4504 * @param constants register pointing to CRC table for 128-bit aligned memory
4505 * @param barretConstants register pointing to table for barrett reduction
4506 * @param t0 volatile register
4507 * @param t1 volatile register
4508 * @param t2 volatile register
4509 */
4510 void MacroAssembler::kernel_crc32_1word_aligned(Register crc, Register buf, Register len,
4511 Register constants, Register barretConstants, Register t0, Register t1, Register t2) {
4512 Label L_mainLoop, L_tail, L_alignTail, L_barrett_reduction, L_end, L_first_warm_up_done, L_first_cool_down, L_second_cool_down, L_XOR, L_test;
4513 Label L_lv0, L_lv1, L_lv2, L_lv3, L_lv4, L_lv5, L_lv6, L_lv7, L_lv8, L_lv9, L_lv10, L_lv11, L_lv12, L_lv13, L_lv14, L_lv15;
4514 Label L_1, L_2, L_3, L_4;
4515
4516 Register rLoaded = t0;
4517 Register rTmp1 = t1;
4518 Register rTmp2 = t2;
4519 Register off16 = R22;
4520 Register off32 = R23;
4521 Register off48 = R24;
4522 Register off64 = R25;
4523 Register off80 = R26;
4524 Register off96 = R27;
4525 Register off112 = R28;
4526 Register rIdx = R29;
4527 Register rMax = R30;
4528 Register constantsPos = R31;
4529
4530 VectorRegister mask_32bit = VR24;
4531 VectorRegister mask_64bit = VR25;
4532 VectorRegister zeroes = VR26;
4533 VectorRegister const1 = VR27;
4534 VectorRegister const2 = VR28;
4535
4536 // Save non-volatile vector registers (frameless).
4537 Register offset = t1; int offsetInt = 0;
4538 offsetInt -= 16; li(offset, -16); stvx(VR20, offset, R1_SP);
4539 offsetInt -= 16; addi(offset, offset, -16); stvx(VR21, offset, R1_SP);
4540 offsetInt -= 16; addi(offset, offset, -16); stvx(VR22, offset, R1_SP);
4541 offsetInt -= 16; addi(offset, offset, -16); stvx(VR23, offset, R1_SP);
4542 offsetInt -= 16; addi(offset, offset, -16); stvx(VR24, offset, R1_SP);
4543 offsetInt -= 16; addi(offset, offset, -16); stvx(VR25, offset, R1_SP);
4544 offsetInt -= 16; addi(offset, offset, -16); stvx(VR26, offset, R1_SP);
4545 offsetInt -= 16; addi(offset, offset, -16); stvx(VR27, offset, R1_SP);
4546 offsetInt -= 16; addi(offset, offset, -16); stvx(VR28, offset, R1_SP);
4547 offsetInt -= 8; std(R22, offsetInt, R1_SP);
4548 offsetInt -= 8; std(R23, offsetInt, R1_SP);
4549 offsetInt -= 8; std(R24, offsetInt, R1_SP);
4550 offsetInt -= 8; std(R25, offsetInt, R1_SP);
4551 offsetInt -= 8; std(R26, offsetInt, R1_SP);
4552 offsetInt -= 8; std(R27, offsetInt, R1_SP);
4553 offsetInt -= 8; std(R28, offsetInt, R1_SP);
4554 offsetInt -= 8; std(R29, offsetInt, R1_SP);
4555 offsetInt -= 8; std(R30, offsetInt, R1_SP);
4556 offsetInt -= 8; std(R31, offsetInt, R1_SP);
4557
4558 // Set constants
4559 li(off16, 16);
4560 li(off32, 32);
4561 li(off48, 48);
4562 li(off64, 64);
4563 li(off80, 80);
4564 li(off96, 96);
4565 li(off112, 112);
4566
4567 clrldi(crc, crc, 32);
4568
4569 vxor(zeroes, zeroes, zeroes);
4570 vspltisw(VR0, -1);
4571
4572 vsldoi(mask_32bit, zeroes, VR0, 4);
4573 vsldoi(mask_64bit, zeroes, VR0, -8);
4574
4575 // Get the initial value into v8
4576 vxor(VR8, VR8, VR8);
4577 mtvrd(VR8, crc);
4578 vsldoi(VR8, zeroes, VR8, -8); // shift into bottom 32 bits
4579
4580 li (rLoaded, 0);
4581
4582 rldicr(rIdx, len, 0, 56);
4583
4584 {
4585 BIND(L_1);
4586 // Checksum in blocks of MAX_SIZE (32768)
4587 lis(rMax, 0);
4588 ori(rMax, rMax, 32768);
4589 mr(rTmp2, rMax);
4590 cmpd(CCR0, rIdx, rMax);
4591 bgt(CCR0, L_2);
4592 mr(rMax, rIdx);
4593
4594 BIND(L_2);
4595 subf(rIdx, rMax, rIdx);
4596
4597 // our main loop does 128 bytes at a time
4598 srdi(rMax, rMax, 7);
4599
4600 /*
4601 * Work out the offset into the constants table to start at. Each
4602 * constant is 16 bytes, and it is used against 128 bytes of input
4603 * data - 128 / 16 = 8
4604 */
4605 sldi(rTmp1, rMax, 4);
4606 srdi(rTmp2, rTmp2, 3);
4607 subf(rTmp1, rTmp1, rTmp2);
4608
4609 // We reduce our final 128 bytes in a separate step
4610 addi(rMax, rMax, -1);
4611 mtctr(rMax);
4612
4613 // Find the start of our constants
4614 add(constantsPos, constants, rTmp1);
4615
4616 // zero VR0-v7 which will contain our checksums
4617 vxor(VR0, VR0, VR0);
4618 vxor(VR1, VR1, VR1);
4619 vxor(VR2, VR2, VR2);
4620 vxor(VR3, VR3, VR3);
4621 vxor(VR4, VR4, VR4);
4622 vxor(VR5, VR5, VR5);
4623 vxor(VR6, VR6, VR6);
4624 vxor(VR7, VR7, VR7);
4625
4626 lvx(const1, constantsPos);
4627
4628 /*
4629 * If we are looping back to consume more data we use the values
4630 * already in VR16-v23.
4631 */
4632 cmpdi(CCR0, rLoaded, 1);
4633 beq(CCR0, L_3);
4634 {
4635
4636 // First warm up pass
4637 lvx(VR16, buf);
4638 lvx(VR17, off16, buf);
4639 lvx(VR18, off32, buf);
4640 lvx(VR19, off48, buf);
4641 lvx(VR20, off64, buf);
4642 lvx(VR21, off80, buf);
4643 lvx(VR22, off96, buf);
4644 lvx(VR23, off112, buf);
4645 addi(buf, buf, 8*16);
4646
4647 // xor in initial value
4648 vxor(VR16, VR16, VR8);
4649 }
4650
4651 BIND(L_3);
4652 bdz(L_first_warm_up_done);
4653
4654 addi(constantsPos, constantsPos, 16);
4655 lvx(const2, constantsPos);
4656
4657 // Second warm up pass
4658 vpmsumd(VR8, VR16, const1);
4659 lvx(VR16, buf);
4660
4661 vpmsumd(VR9, VR17, const1);
4662 lvx(VR17, off16, buf);
4663
4664 vpmsumd(VR10, VR18, const1);
4665 lvx(VR18, off32, buf);
4666
4667 vpmsumd(VR11, VR19, const1);
4668 lvx(VR19, off48, buf);
4669
4670 vpmsumd(VR12, VR20, const1);
4671 lvx(VR20, off64, buf);
4672
4673 vpmsumd(VR13, VR21, const1);
4674 lvx(VR21, off80, buf);
4675
4676 vpmsumd(VR14, VR22, const1);
4677 lvx(VR22, off96, buf);
4678
4679 vpmsumd(VR15, VR23, const1);
4680 lvx(VR23, off112, buf);
4681
4682 addi(buf, buf, 8 * 16);
4683
4684 bdz(L_first_cool_down);
4685
4686 /*
4687 * main loop. We modulo schedule it such that it takes three iterations
4688 * to complete - first iteration load, second iteration vpmsum, third
4689 * iteration xor.
4690 */
4691 {
4692 BIND(L_4);
4693 lvx(const1, constantsPos); addi(constantsPos, constantsPos, 16);
4694
4695 vxor(VR0, VR0, VR8);
4696 vpmsumd(VR8, VR16, const2);
4697 lvx(VR16, buf);
4698
4699 vxor(VR1, VR1, VR9);
4700 vpmsumd(VR9, VR17, const2);
4701 lvx(VR17, off16, buf);
4702
4703 vxor(VR2, VR2, VR10);
4704 vpmsumd(VR10, VR18, const2);
4705 lvx(VR18, off32, buf);
4706
4707 vxor(VR3, VR3, VR11);
4708 vpmsumd(VR11, VR19, const2);
4709 lvx(VR19, off48, buf);
4710 lvx(const2, constantsPos);
4711
4712 vxor(VR4, VR4, VR12);
4713 vpmsumd(VR12, VR20, const1);
4714 lvx(VR20, off64, buf);
4715
4716 vxor(VR5, VR5, VR13);
4717 vpmsumd(VR13, VR21, const1);
4718 lvx(VR21, off80, buf);
4719
4720 vxor(VR6, VR6, VR14);
4721 vpmsumd(VR14, VR22, const1);
4722 lvx(VR22, off96, buf);
4723
4724 vxor(VR7, VR7, VR15);
4725 vpmsumd(VR15, VR23, const1);
4726 lvx(VR23, off112, buf);
4727
4728 addi(buf, buf, 8 * 16);
4729
4730 bdnz(L_4);
4731 }
4732
4733 BIND(L_first_cool_down);
4734
4735 // First cool down pass
4736 lvx(const1, constantsPos);
4737 addi(constantsPos, constantsPos, 16);
4738
4739 vxor(VR0, VR0, VR8);
4740 vpmsumd(VR8, VR16, const1);
4741
4742 vxor(VR1, VR1, VR9);
4743 vpmsumd(VR9, VR17, const1);
4744
4745 vxor(VR2, VR2, VR10);
4746 vpmsumd(VR10, VR18, const1);
4747
4748 vxor(VR3, VR3, VR11);
4749 vpmsumd(VR11, VR19, const1);
4750
4751 vxor(VR4, VR4, VR12);
4752 vpmsumd(VR12, VR20, const1);
4753
4754 vxor(VR5, VR5, VR13);
4755 vpmsumd(VR13, VR21, const1);
4756
4757 vxor(VR6, VR6, VR14);
4758 vpmsumd(VR14, VR22, const1);
4759
4760 vxor(VR7, VR7, VR15);
4761 vpmsumd(VR15, VR23, const1);
4762
4763 BIND(L_second_cool_down);
4764 // Second cool down pass
4765 vxor(VR0, VR0, VR8);
4766 vxor(VR1, VR1, VR9);
4767 vxor(VR2, VR2, VR10);
4768 vxor(VR3, VR3, VR11);
4769 vxor(VR4, VR4, VR12);
4770 vxor(VR5, VR5, VR13);
4771 vxor(VR6, VR6, VR14);
4772 vxor(VR7, VR7, VR15);
4773
4774 /*
4775 * vpmsumd produces a 96 bit result in the least significant bits
4776 * of the register. Since we are bit reflected we have to shift it
4777 * left 32 bits so it occupies the least significant bits in the
4778 * bit reflected domain.
4779 */
4780 vsldoi(VR0, VR0, zeroes, 4);
4781 vsldoi(VR1, VR1, zeroes, 4);
4782 vsldoi(VR2, VR2, zeroes, 4);
4783 vsldoi(VR3, VR3, zeroes, 4);
4784 vsldoi(VR4, VR4, zeroes, 4);
4785 vsldoi(VR5, VR5, zeroes, 4);
4786 vsldoi(VR6, VR6, zeroes, 4);
4787 vsldoi(VR7, VR7, zeroes, 4);
4788
4789 // xor with last 1024 bits
4790 lvx(VR8, buf);
4791 lvx(VR9, off16, buf);
4792 lvx(VR10, off32, buf);
4793 lvx(VR11, off48, buf);
4794 lvx(VR12, off64, buf);
4795 lvx(VR13, off80, buf);
4796 lvx(VR14, off96, buf);
4797 lvx(VR15, off112, buf);
4798 addi(buf, buf, 8 * 16);
4799
4800 vxor(VR16, VR0, VR8);
4801 vxor(VR17, VR1, VR9);
4802 vxor(VR18, VR2, VR10);
4803 vxor(VR19, VR3, VR11);
4804 vxor(VR20, VR4, VR12);
4805 vxor(VR21, VR5, VR13);
4806 vxor(VR22, VR6, VR14);
4807 vxor(VR23, VR7, VR15);
4808
4809 li(rLoaded, 1);
4810 cmpdi(CCR0, rIdx, 0);
4811 addi(rIdx, rIdx, 128);
4812 bne(CCR0, L_1);
4813 }
4814
4815 // Work out how many bytes we have left
4816 andi_(len, len, 127);
4817
4818 // Calculate where in the constant table we need to start
4819 subfic(rTmp1, len, 128);
4820 add(constantsPos, constantsPos, rTmp1);
4821
4822 // How many 16 byte chunks are in the tail
4823 srdi(rIdx, len, 4);
4824 mtctr(rIdx);
4825
4826 /*
4827 * Reduce the previously calculated 1024 bits to 64 bits, shifting
4828 * 32 bits to include the trailing 32 bits of zeros
4829 */
4830 lvx(VR0, constantsPos);
4831 lvx(VR1, off16, constantsPos);
4832 lvx(VR2, off32, constantsPos);
4833 lvx(VR3, off48, constantsPos);
4834 lvx(VR4, off64, constantsPos);
4835 lvx(VR5, off80, constantsPos);
4836 lvx(VR6, off96, constantsPos);
4837 lvx(VR7, off112, constantsPos);
4838 addi(constantsPos, constantsPos, 8 * 16);
4839
4840 vpmsumw(VR0, VR16, VR0);
4841 vpmsumw(VR1, VR17, VR1);
4842 vpmsumw(VR2, VR18, VR2);
4843 vpmsumw(VR3, VR19, VR3);
4844 vpmsumw(VR4, VR20, VR4);
4845 vpmsumw(VR5, VR21, VR5);
4846 vpmsumw(VR6, VR22, VR6);
4847 vpmsumw(VR7, VR23, VR7);
4848
4849 // Now reduce the tail (0 - 112 bytes)
4850 cmpdi(CCR0, rIdx, 0);
4851 beq(CCR0, L_XOR);
4852
4853 lvx(VR16, buf); addi(buf, buf, 16);
4854 lvx(VR17, constantsPos);
4855 vpmsumw(VR16, VR16, VR17);
4856 vxor(VR0, VR0, VR16);
4857 beq(CCR0, L_XOR);
4858
4859 lvx(VR16, buf); addi(buf, buf, 16);
4860 lvx(VR17, off16, constantsPos);
4861 vpmsumw(VR16, VR16, VR17);
4862 vxor(VR0, VR0, VR16);
4863 beq(CCR0, L_XOR);
4864
4865 lvx(VR16, buf); addi(buf, buf, 16);
4866 lvx(VR17, off32, constantsPos);
4867 vpmsumw(VR16, VR16, VR17);
4868 vxor(VR0, VR0, VR16);
4869 beq(CCR0, L_XOR);
4870
4871 lvx(VR16, buf); addi(buf, buf, 16);
4872 lvx(VR17, off48,constantsPos);
4873 vpmsumw(VR16, VR16, VR17);
4874 vxor(VR0, VR0, VR16);
4875 beq(CCR0, L_XOR);
4876
4877 lvx(VR16, buf); addi(buf, buf, 16);
4878 lvx(VR17, off64, constantsPos);
4879 vpmsumw(VR16, VR16, VR17);
4880 vxor(VR0, VR0, VR16);
4881 beq(CCR0, L_XOR);
4882
4883 lvx(VR16, buf); addi(buf, buf, 16);
4884 lvx(VR17, off80, constantsPos);
4885 vpmsumw(VR16, VR16, VR17);
4886 vxor(VR0, VR0, VR16);
4887 beq(CCR0, L_XOR);
4888
4889 lvx(VR16, buf); addi(buf, buf, 16);
4890 lvx(VR17, off96, constantsPos);
4891 vpmsumw(VR16, VR16, VR17);
4892 vxor(VR0, VR0, VR16);
4893
4894 // Now xor all the parallel chunks together
4895 BIND(L_XOR);
4896 vxor(VR0, VR0, VR1);
4897 vxor(VR2, VR2, VR3);
4898 vxor(VR4, VR4, VR5);
4899 vxor(VR6, VR6, VR7);
4900
4901 vxor(VR0, VR0, VR2);
4902 vxor(VR4, VR4, VR6);
4903
4904 vxor(VR0, VR0, VR4);
4905
4906 b(L_barrett_reduction);
4907
4908 BIND(L_first_warm_up_done);
4909 lvx(const1, constantsPos);
4910 addi(constantsPos, constantsPos, 16);
4911 vpmsumd(VR8, VR16, const1);
4912 vpmsumd(VR9, VR17, const1);
4913 vpmsumd(VR10, VR18, const1);
4914 vpmsumd(VR11, VR19, const1);
4915 vpmsumd(VR12, VR20, const1);
4916 vpmsumd(VR13, VR21, const1);
4917 vpmsumd(VR14, VR22, const1);
4918 vpmsumd(VR15, VR23, const1);
4919 b(L_second_cool_down);
4920
4921 BIND(L_barrett_reduction);
4922
4923 lvx(const1, barretConstants);
4924 addi(barretConstants, barretConstants, 16);
4925 lvx(const2, barretConstants);
4926
4927 vsldoi(VR1, VR0, VR0, -8);
4928 vxor(VR0, VR0, VR1); // xor two 64 bit results together
4929
4930 // shift left one bit
4931 vspltisb(VR1, 1);
4932 vsl(VR0, VR0, VR1);
4933
4934 vand(VR0, VR0, mask_64bit);
4935
4936 /*
4937 * The reflected version of Barrett reduction. Instead of bit
4938 * reflecting our data (which is expensive to do), we bit reflect our
4939 * constants and our algorithm, which means the intermediate data in
4940 * our vector registers goes from 0-63 instead of 63-0. We can reflect
4941 * the algorithm because we don't carry in mod 2 arithmetic.
4942 */
4943 vand(VR1, VR0, mask_32bit); // bottom 32 bits of a
4944 vpmsumd(VR1, VR1, const1); // ma
4945 vand(VR1, VR1, mask_32bit); // bottom 32bits of ma
4946 vpmsumd(VR1, VR1, const2); // qn */
4947 vxor(VR0, VR0, VR1); // a - qn, subtraction is xor in GF(2)
4948
4949 /*
4950 * Since we are bit reflected, the result (ie the low 32 bits) is in
4951 * the high 32 bits. We just need to shift it left 4 bytes
4952 * V0 [ 0 1 X 3 ]
4953 * V0 [ 0 X 2 3 ]
4954 */
4955 vsldoi(VR0, VR0, zeroes, 4); // shift result into top 64 bits of
4956
4957 // Get it into r3
4958 mfvrd(crc, VR0);
4959
4960 BIND(L_end);
4961
4962 offsetInt = 0;
4963 // Restore non-volatile Vector registers (frameless).
4964 offsetInt -= 16; li(offset, -16); lvx(VR20, offset, R1_SP);
4965 offsetInt -= 16; addi(offset, offset, -16); lvx(VR21, offset, R1_SP);
4966 offsetInt -= 16; addi(offset, offset, -16); lvx(VR22, offset, R1_SP);
4967 offsetInt -= 16; addi(offset, offset, -16); lvx(VR23, offset, R1_SP);
4968 offsetInt -= 16; addi(offset, offset, -16); lvx(VR24, offset, R1_SP);
4969 offsetInt -= 16; addi(offset, offset, -16); lvx(VR25, offset, R1_SP);
4970 offsetInt -= 16; addi(offset, offset, -16); lvx(VR26, offset, R1_SP);
4971 offsetInt -= 16; addi(offset, offset, -16); lvx(VR27, offset, R1_SP);
4972 offsetInt -= 16; addi(offset, offset, -16); lvx(VR28, offset, R1_SP);
4973 offsetInt -= 8; ld(R22, offsetInt, R1_SP);
4974 offsetInt -= 8; ld(R23, offsetInt, R1_SP);
4975 offsetInt -= 8; ld(R24, offsetInt, R1_SP);
4976 offsetInt -= 8; ld(R25, offsetInt, R1_SP);
4977 offsetInt -= 8; ld(R26, offsetInt, R1_SP);
4978 offsetInt -= 8; ld(R27, offsetInt, R1_SP);
4979 offsetInt -= 8; ld(R28, offsetInt, R1_SP);
4980 offsetInt -= 8; ld(R29, offsetInt, R1_SP);
4981 offsetInt -= 8; ld(R30, offsetInt, R1_SP);
4982 offsetInt -= 8; ld(R31, offsetInt, R1_SP);
4983 }
4984
4985 void MacroAssembler::kernel_crc32_singleByte(Register crc, Register buf, Register len, Register table, Register tmp, bool invertCRC) {
4986 assert_different_registers(crc, buf, /* len, not used!! */ table, tmp);
4987
4988 BLOCK_COMMENT("kernel_crc32_singleByte:");
4989 if (invertCRC) {
4990 nand(crc, crc, crc); // 1s complement of crc
4991 }
4992
4993 lbz(tmp, 0, buf); // Byte from buffer, zero-extended.
4994 update_byte_crc32(crc, tmp, table);
4995
4996 if (invertCRC) {
4997 nand(crc, crc, crc); // 1s complement of crc
4998 }
4999 }
5000
5001 void MacroAssembler::kernel_crc32_singleByteReg(Register crc, Register val, Register table, bool invertCRC) {
5002 assert_different_registers(crc, val, table);
5003
5004 BLOCK_COMMENT("kernel_crc32_singleByteReg:");
5005 if (invertCRC) {
5006 nand(crc, crc, crc); // 1s complement of crc
5007 }
5008
5009 update_byte_crc32(crc, val, table);
5010
5011 if (invertCRC) {
5012 nand(crc, crc, crc); // 1s complement of crc
5013 }
5014 }
5015
5016 // dest_lo += src1 + src2
5017 // dest_hi += carry1 + carry2
5018 void MacroAssembler::add2_with_carry(Register dest_hi,
5019 Register dest_lo,
5020 Register src1, Register src2) {
5021 li(R0, 0);
5022 addc(dest_lo, dest_lo, src1);
5023 adde(dest_hi, dest_hi, R0);
5024 addc(dest_lo, dest_lo, src2);
5025 adde(dest_hi, dest_hi, R0);
5026 }
5027
5028 // Multiply 64 bit by 64 bit first loop.
5029 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart,
5030 Register x_xstart,
5031 Register y, Register y_idx,
5032 Register z,
5033 Register carry,
5034 Register product_high, Register product,
5035 Register idx, Register kdx,
5036 Register tmp) {
5037 // jlong carry, x[], y[], z[];
5038 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
5039 // huge_128 product = y[idx] * x[xstart] + carry;
5040 // z[kdx] = (jlong)product;
5041 // carry = (jlong)(product >>> 64);
5042 // }
5043 // z[xstart] = carry;
5044
5045 Label L_first_loop, L_first_loop_exit;
5046 Label L_one_x, L_one_y, L_multiply;
5047
5048 addic_(xstart, xstart, -1);
5049 blt(CCR0, L_one_x); // Special case: length of x is 1.
5050
5051 // Load next two integers of x.
5052 sldi(tmp, xstart, LogBytesPerInt);
5053 ldx(x_xstart, x, tmp);
5054 #ifdef VM_LITTLE_ENDIAN
5055 rldicl(x_xstart, x_xstart, 32, 0);
5056 #endif
5057
5058 align(32, 16);
5059 bind(L_first_loop);
5060
5061 cmpdi(CCR0, idx, 1);
5062 blt(CCR0, L_first_loop_exit);
5063 addi(idx, idx, -2);
5064 beq(CCR0, L_one_y);
5065
5066 // Load next two integers of y.
5067 sldi(tmp, idx, LogBytesPerInt);
5068 ldx(y_idx, y, tmp);
5069 #ifdef VM_LITTLE_ENDIAN
5070 rldicl(y_idx, y_idx, 32, 0);
5071 #endif
5072
5073
5074 bind(L_multiply);
5075 multiply64(product_high, product, x_xstart, y_idx);
5076
5077 li(tmp, 0);
5078 addc(product, product, carry); // Add carry to result.
5079 adde(product_high, product_high, tmp); // Add carry of the last addition.
5080 addi(kdx, kdx, -2);
5081
5082 // Store result.
5083 #ifdef VM_LITTLE_ENDIAN
5084 rldicl(product, product, 32, 0);
5085 #endif
5086 sldi(tmp, kdx, LogBytesPerInt);
5087 stdx(product, z, tmp);
5088 mr_if_needed(carry, product_high);
5089 b(L_first_loop);
5090
5091
5092 bind(L_one_y); // Load one 32 bit portion of y as (0,value).
5093
5094 lwz(y_idx, 0, y);
5095 b(L_multiply);
5096
5097
5098 bind(L_one_x); // Load one 32 bit portion of x as (0,value).
5099
5100 lwz(x_xstart, 0, x);
5101 b(L_first_loop);
5102
5103 bind(L_first_loop_exit);
5104 }
5105
5106 // Multiply 64 bit by 64 bit and add 128 bit.
5107 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y,
5108 Register z, Register yz_idx,
5109 Register idx, Register carry,
5110 Register product_high, Register product,
5111 Register tmp, int offset) {
5112
5113 // huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
5114 // z[kdx] = (jlong)product;
5115
5116 sldi(tmp, idx, LogBytesPerInt);
5117 if (offset) {
5118 addi(tmp, tmp, offset);
5119 }
5120 ldx(yz_idx, y, tmp);
5121 #ifdef VM_LITTLE_ENDIAN
5122 rldicl(yz_idx, yz_idx, 32, 0);
5123 #endif
5124
5125 multiply64(product_high, product, x_xstart, yz_idx);
5126 ldx(yz_idx, z, tmp);
5127 #ifdef VM_LITTLE_ENDIAN
5128 rldicl(yz_idx, yz_idx, 32, 0);
5129 #endif
5130
5131 add2_with_carry(product_high, product, carry, yz_idx);
5132
5133 sldi(tmp, idx, LogBytesPerInt);
5134 if (offset) {
5135 addi(tmp, tmp, offset);
5136 }
5137 #ifdef VM_LITTLE_ENDIAN
5138 rldicl(product, product, 32, 0);
5139 #endif
5140 stdx(product, z, tmp);
5141 }
5142
5143 // Multiply 128 bit by 128 bit. Unrolled inner loop.
5144 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart,
5145 Register y, Register z,
5146 Register yz_idx, Register idx, Register carry,
5147 Register product_high, Register product,
5148 Register carry2, Register tmp) {
5149
5150 // jlong carry, x[], y[], z[];
5151 // int kdx = ystart+1;
5152 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
5153 // huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
5154 // z[kdx+idx+1] = (jlong)product;
5155 // jlong carry2 = (jlong)(product >>> 64);
5156 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
5157 // z[kdx+idx] = (jlong)product;
5158 // carry = (jlong)(product >>> 64);
5159 // }
5160 // idx += 2;
5161 // if (idx > 0) {
5162 // product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
5163 // z[kdx+idx] = (jlong)product;
5164 // carry = (jlong)(product >>> 64);
5165 // }
5166
5167 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
5168 const Register jdx = R0;
5169
5170 // Scale the index.
5171 srdi_(jdx, idx, 2);
5172 beq(CCR0, L_third_loop_exit);
5173 mtctr(jdx);
5174
5175 align(32, 16);
5176 bind(L_third_loop);
5177
5178 addi(idx, idx, -4);
5179
5180 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 8);
5181 mr_if_needed(carry2, product_high);
5182
5183 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product_high, product, tmp, 0);
5184 mr_if_needed(carry, product_high);
5185 bdnz(L_third_loop);
5186
5187 bind(L_third_loop_exit); // Handle any left-over operand parts.
5188
5189 andi_(idx, idx, 0x3);
5190 beq(CCR0, L_post_third_loop_done);
5191
5192 Label L_check_1;
5193
5194 addic_(idx, idx, -2);
5195 blt(CCR0, L_check_1);
5196
5197 multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 0);
5198 mr_if_needed(carry, product_high);
5199
5200 bind(L_check_1);
5201
5202 addi(idx, idx, 0x2);
5203 andi_(idx, idx, 0x1);
5204 addic_(idx, idx, -1);
5205 blt(CCR0, L_post_third_loop_done);
5206
5207 sldi(tmp, idx, LogBytesPerInt);
5208 lwzx(yz_idx, y, tmp);
5209 multiply64(product_high, product, x_xstart, yz_idx);
5210 lwzx(yz_idx, z, tmp);
5211
5212 add2_with_carry(product_high, product, yz_idx, carry);
5213
5214 sldi(tmp, idx, LogBytesPerInt);
5215 stwx(product, z, tmp);
5216 srdi(product, product, 32);
5217
5218 sldi(product_high, product_high, 32);
5219 orr(product, product, product_high);
5220 mr_if_needed(carry, product);
5221
5222 bind(L_post_third_loop_done);
5223 } // multiply_128_x_128_loop
5224
5225 void MacroAssembler::multiply_to_len(Register x, Register xlen,
5226 Register y, Register ylen,
5227 Register z, Register zlen,
5228 Register tmp1, Register tmp2,
5229 Register tmp3, Register tmp4,
5230 Register tmp5, Register tmp6,
5231 Register tmp7, Register tmp8,
5232 Register tmp9, Register tmp10,
5233 Register tmp11, Register tmp12,
5234 Register tmp13) {
5235
5236 ShortBranchVerifier sbv(this);
5237
5238 assert_different_registers(x, xlen, y, ylen, z, zlen,
5239 tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
5240 assert_different_registers(x, xlen, y, ylen, z, zlen,
5241 tmp1, tmp2, tmp3, tmp4, tmp5, tmp7);
5242 assert_different_registers(x, xlen, y, ylen, z, zlen,
5243 tmp1, tmp2, tmp3, tmp4, tmp5, tmp8);
5244
5245 const Register idx = tmp1;
5246 const Register kdx = tmp2;
5247 const Register xstart = tmp3;
5248
5249 const Register y_idx = tmp4;
5250 const Register carry = tmp5;
5251 const Register product = tmp6;
5252 const Register product_high = tmp7;
5253 const Register x_xstart = tmp8;
5254 const Register tmp = tmp9;
5255
5256 // First Loop.
5257 //
5258 // final static long LONG_MASK = 0xffffffffL;
5259 // int xstart = xlen - 1;
5260 // int ystart = ylen - 1;
5261 // long carry = 0;
5262 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
5263 // long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
5264 // z[kdx] = (int)product;
5265 // carry = product >>> 32;
5266 // }
5267 // z[xstart] = (int)carry;
5268
5269 mr_if_needed(idx, ylen); // idx = ylen
5270 mr_if_needed(kdx, zlen); // kdx = xlen + ylen
5271 li(carry, 0); // carry = 0
5272
5273 Label L_done;
5274
5275 addic_(xstart, xlen, -1);
5276 blt(CCR0, L_done);
5277
5278 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z,
5279 carry, product_high, product, idx, kdx, tmp);
5280
5281 Label L_second_loop;
5282
5283 cmpdi(CCR0, kdx, 0);
5284 beq(CCR0, L_second_loop);
5285
5286 Label L_carry;
5287
5288 addic_(kdx, kdx, -1);
5289 beq(CCR0, L_carry);
5290
5291 // Store lower 32 bits of carry.
5292 sldi(tmp, kdx, LogBytesPerInt);
5293 stwx(carry, z, tmp);
5294 srdi(carry, carry, 32);
5295 addi(kdx, kdx, -1);
5296
5297
5298 bind(L_carry);
5299
5300 // Store upper 32 bits of carry.
5301 sldi(tmp, kdx, LogBytesPerInt);
5302 stwx(carry, z, tmp);
5303
5304 // Second and third (nested) loops.
5305 //
5306 // for (int i = xstart-1; i >= 0; i--) { // Second loop
5307 // carry = 0;
5308 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
5309 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
5310 // (z[k] & LONG_MASK) + carry;
5311 // z[k] = (int)product;
5312 // carry = product >>> 32;
5313 // }
5314 // z[i] = (int)carry;
5315 // }
5316 //
5317 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
5318
5319 bind(L_second_loop);
5320
5321 li(carry, 0); // carry = 0;
5322
5323 addic_(xstart, xstart, -1); // i = xstart-1;
5324 blt(CCR0, L_done);
5325
5326 Register zsave = tmp10;
5327
5328 mr(zsave, z);
5329
5330
5331 Label L_last_x;
5332
5333 sldi(tmp, xstart, LogBytesPerInt);
5334 add(z, z, tmp); // z = z + k - j
5335 addi(z, z, 4);
5336 addic_(xstart, xstart, -1); // i = xstart-1;
5337 blt(CCR0, L_last_x);
5338
5339 sldi(tmp, xstart, LogBytesPerInt);
5340 ldx(x_xstart, x, tmp);
5341 #ifdef VM_LITTLE_ENDIAN
5342 rldicl(x_xstart, x_xstart, 32, 0);
5343 #endif
5344
5345
5346 Label L_third_loop_prologue;
5347
5348 bind(L_third_loop_prologue);
5349
5350 Register xsave = tmp11;
5351 Register xlensave = tmp12;
5352 Register ylensave = tmp13;
5353
5354 mr(xsave, x);
5355 mr(xlensave, xstart);
5356 mr(ylensave, ylen);
5357
5358
5359 multiply_128_x_128_loop(x_xstart, y, z, y_idx, ylen,
5360 carry, product_high, product, x, tmp);
5361
5362 mr(z, zsave);
5363 mr(x, xsave);
5364 mr(xlen, xlensave); // This is the decrement of the loop counter!
5365 mr(ylen, ylensave);
5366
5367 addi(tmp3, xlen, 1);
5368 sldi(tmp, tmp3, LogBytesPerInt);
5369 stwx(carry, z, tmp);
5370 addic_(tmp3, tmp3, -1);
5371 blt(CCR0, L_done);
5372
5373 srdi(carry, carry, 32);
5374 sldi(tmp, tmp3, LogBytesPerInt);
5375 stwx(carry, z, tmp);
5376 b(L_second_loop);
5377
5378 // Next infrequent code is moved outside loops.
5379 bind(L_last_x);
5380
5381 lwz(x_xstart, 0, x);
5382 b(L_third_loop_prologue);
5383
5384 bind(L_done);
5385 } // multiply_to_len
5386
5387 void MacroAssembler::asm_assert(bool check_equal, const char *msg, int id) {
5388 #ifdef ASSERT
5389 Label ok;
5390 if (check_equal) {
5391 beq(CCR0, ok);
5392 } else {
5393 bne(CCR0, ok);
5394 }
5395 stop(msg, id);
5396 bind(ok);
5397 #endif
5398 }
5399
5400 void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset,
5401 Register mem_base, const char* msg, int id) {
5402 #ifdef ASSERT
5403 switch (size) {
5404 case 4:
5405 lwz(R0, mem_offset, mem_base);
5406 cmpwi(CCR0, R0, 0);
5407 break;
5408 case 8:
5409 ld(R0, mem_offset, mem_base);
5410 cmpdi(CCR0, R0, 0);
5411 break;
5412 default:
5413 ShouldNotReachHere();
5414 }
5415 asm_assert(check_equal, msg, id);
5416 #endif // ASSERT
5417 }
5418
5419 void MacroAssembler::verify_thread() {
5420 if (VerifyThread) {
5421 unimplemented("'VerifyThread' currently not implemented on PPC");
5422 }
5423 }
5424
5425 // READ: oop. KILL: R0. Volatile floats perhaps.
5426 void MacroAssembler::verify_oop(Register oop, const char* msg) {
5427 if (!VerifyOops) {
5428 return;
5429 }
5430
5431 address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
5432 const Register tmp = R11; // Will be preserved.
5433 const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
5434 save_volatile_gprs(R1_SP, -nbytes_save); // except R0
5435
5436 mr_if_needed(R4_ARG2, oop);
5437 save_LR_CR(tmp); // save in old frame
5438 push_frame_reg_args(nbytes_save, tmp);
5439 // load FunctionDescriptor** / entry_address *
5440 load_const_optimized(tmp, fd, R0);
5441 // load FunctionDescriptor* / entry_address
5442 ld(tmp, 0, tmp);
5443 load_const_optimized(R3_ARG1, (address)msg, R0);
5444 // Call destination for its side effect.
5445 call_c(tmp);
5446
5447 pop_frame();
5448 restore_LR_CR(tmp);
5449 restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
5450 }
5451
5452 void MacroAssembler::verify_oop_addr(RegisterOrConstant offs, Register base, const char* msg) {
5453 if (!VerifyOops) {
5454 return;
5455 }
5456
5457 address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
5458 const Register tmp = R11; // Will be preserved.
5459 const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
5460 save_volatile_gprs(R1_SP, -nbytes_save); // except R0
5461
5462 ld(R4_ARG2, offs, base);
5463 save_LR_CR(tmp); // save in old frame
5464 push_frame_reg_args(nbytes_save, tmp);
5465 // load FunctionDescriptor** / entry_address *
5466 load_const_optimized(tmp, fd, R0);
5467 // load FunctionDescriptor* / entry_address
5468 ld(tmp, 0, tmp);
5469 load_const_optimized(R3_ARG1, (address)msg, R0);
5470 // Call destination for its side effect.
5471 call_c(tmp);
5472
5473 pop_frame();
5474 restore_LR_CR(tmp);
5475 restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
5476 }
5477
5478 const char* stop_types[] = {
5479 "stop",
5480 "untested",
5481 "unimplemented",
5482 "shouldnotreachhere"
5483 };
5484
5485 static void stop_on_request(int tp, const char* msg) {
5486 tty->print("PPC assembly code requires stop: (%s) %s\n", stop_types[tp%/*stop_end*/4], msg);
5487 guarantee(false, "PPC assembly code requires stop: %s", msg);
5488 }
5489
5490 // Call a C-function that prints output.
5491 void MacroAssembler::stop(int type, const char* msg, int id) {
5492 #ifndef PRODUCT
5493 block_comment(err_msg("stop: %s %s {", stop_types[type%stop_end], msg));
5494 #else
5495 block_comment("stop {");
5496 #endif
5497
5498 // setup arguments
5499 load_const_optimized(R3_ARG1, type);
5500 load_const_optimized(R4_ARG2, (void *)msg, /*tmp=*/R0);
5501 call_VM_leaf(CAST_FROM_FN_PTR(address, stop_on_request), R3_ARG1, R4_ARG2);
5502 illtrap();
5503 emit_int32(id);
5504 block_comment("} stop;");
5505 }
5506
5507 #ifndef PRODUCT
5508 // Write pattern 0x0101010101010101 in memory region [low-before, high+after].
5509 // Val, addr are temp registers.
5510 // If low == addr, addr is killed.
5511 // High is preserved.
5512 void MacroAssembler::zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) {
5513 if (!ZapMemory) return;
5514
5515 assert_different_registers(low, val);
5516
5517 BLOCK_COMMENT("zap memory region {");
5518 load_const_optimized(val, 0x0101010101010101);
5519 int size = before + after;
5520 if (low == high && size < 5 && size > 0) {
5521 int offset = -before*BytesPerWord;
5522 for (int i = 0; i < size; ++i) {
5523 std(val, offset, low);
5524 offset += (1*BytesPerWord);
5525 }
5526 } else {
5527 addi(addr, low, -before*BytesPerWord);
5528 assert_different_registers(high, val);
5529 if (after) addi(high, high, after * BytesPerWord);
5530 Label loop;
5531 bind(loop);
5532 std(val, 0, addr);
5533 addi(addr, addr, 8);
5534 cmpd(CCR6, addr, high);
5535 ble(CCR6, loop);
5536 if (after) addi(high, high, -after * BytesPerWord); // Correct back to old value.
5537 }
5538 BLOCK_COMMENT("} zap memory region");
5539 }
5540
5541 #endif // !PRODUCT
5542
5543 SkipIfEqualZero::SkipIfEqualZero(MacroAssembler* masm, Register temp, const bool* flag_addr) : _masm(masm), _label() {
5544 int simm16_offset = masm->load_const_optimized(temp, (address)flag_addr, R0, true);
5545 assert(sizeof(bool) == 1, "PowerPC ABI");
5546 masm->lbz(temp, simm16_offset, temp);
5547 masm->cmpwi(CCR0, temp, 0);
5548 masm->beq(CCR0, _label);
5549 }
5550
5551 SkipIfEqualZero::~SkipIfEqualZero() {
5552 _masm->bind(_label);
5553 }