1 /*
2 * Copyright (c) 2002, 2020, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2012, 2015 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #ifndef CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP
27 #define CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP
28
29 #include "asm/assembler.inline.hpp"
30 #include "asm/macroAssembler.hpp"
31 #include "asm/codeBuffer.hpp"
32 #include "code/codeCache.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "oops/accessDecorators.hpp"
36 #include "oops/compressedOops.hpp"
37 #include "runtime/safepointMechanism.hpp"
38 #include "utilities/powerOfTwo.hpp"
39
40 inline bool MacroAssembler::is_ld_largeoffset(address a) {
41 const int inst1 = *(int *)a;
42 const int inst2 = *(int *)(a+4);
43 return (is_ld(inst1)) ||
44 (is_addis(inst1) && is_ld(inst2) && inv_ra_field(inst2) == inv_rt_field(inst1));
45 }
46
47 inline int MacroAssembler::get_ld_largeoffset_offset(address a) {
48 assert(MacroAssembler::is_ld_largeoffset(a), "must be ld with large offset");
49
50 const int inst1 = *(int *)a;
51 if (is_ld(inst1)) {
52 return inv_d1_field(inst1);
53 } else {
54 const int inst2 = *(int *)(a+4);
55 return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2);
56 }
57 }
58
59 inline void MacroAssembler::round_to(Register r, int modulus) {
60 assert(is_power_of_2((jlong)modulus), "must be power of 2");
61 addi(r, r, modulus-1);
62 clrrdi(r, r, log2_long((jlong)modulus));
63 }
64
65 // Move register if destination register and target register are different.
66 inline void MacroAssembler::mr_if_needed(Register rd, Register rs) {
67 if (rs != rd) mr(rd, rs);
68 }
69 inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) {
70 if (rs != rd) fmr(rd, rs);
71 }
72 inline void MacroAssembler::endgroup_if_needed(bool needed) {
73 if (needed) {
74 endgroup();
75 }
76 }
77
78 inline void MacroAssembler::membar(int bits) {
79 // Comment: Usage of elemental_membar(bits) is not recommended for Power 8.
80 // If elemental_membar(bits) is used, disable optimization of acquire-release
81 // (Matcher::post_membar_release where we use PPC64_ONLY(xop == Op_MemBarRelease ||))!
82 if (bits & StoreLoad) { sync(); }
83 else if (bits) { lwsync(); }
84 }
85 inline void MacroAssembler::release() { membar(LoadStore | StoreStore); }
86 inline void MacroAssembler::acquire() { membar(LoadLoad | LoadStore); }
87 inline void MacroAssembler::fence() { membar(LoadLoad | LoadStore | StoreLoad | StoreStore); }
88
89 // Address of the global TOC.
90 inline address MacroAssembler::global_toc() {
91 return CodeCache::low_bound();
92 }
93
94 // Offset of given address to the global TOC.
95 inline int MacroAssembler::offset_to_global_toc(const address addr) {
96 intptr_t offset = (intptr_t)addr - (intptr_t)MacroAssembler::global_toc();
97 assert(Assembler::is_uimm((long)offset, 31), "must be in range");
98 return (int)offset;
99 }
100
101 // Address of current method's TOC.
102 inline address MacroAssembler::method_toc() {
103 return code()->consts()->start();
104 }
105
106 // Offset of given address to current method's TOC.
107 inline int MacroAssembler::offset_to_method_toc(address addr) {
108 intptr_t offset = (intptr_t)addr - (intptr_t)method_toc();
109 assert(Assembler::is_uimm((long)offset, 31), "must be in range");
110 return (int)offset;
111 }
112
113 inline bool MacroAssembler::is_calculate_address_from_global_toc_at(address a, address bound) {
114 const address inst2_addr = a;
115 const int inst2 = *(int *) a;
116
117 // The relocation points to the second instruction, the addi.
118 if (!is_addi(inst2)) return false;
119
120 // The addi reads and writes the same register dst.
121 const int dst = inv_rt_field(inst2);
122 if (inv_ra_field(inst2) != dst) return false;
123
124 // Now, find the preceding addis which writes to dst.
125 int inst1 = 0;
126 address inst1_addr = inst2_addr - BytesPerInstWord;
127 while (inst1_addr >= bound) {
128 inst1 = *(int *) inst1_addr;
129 if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
130 // stop, found the addis which writes dst
131 break;
132 }
133 inst1_addr -= BytesPerInstWord;
134 }
135
136 if (!(inst1 == 0 || inv_ra_field(inst1) == 29 /* R29 */)) return false;
137 return is_addis(inst1);
138 }
139
140 #ifdef _LP64
141 // Detect narrow oop constants.
142 inline bool MacroAssembler::is_set_narrow_oop(address a, address bound) {
143 const address inst2_addr = a;
144 const int inst2 = *(int *)a;
145 // The relocation points to the second instruction, the ori.
146 if (!is_ori(inst2)) return false;
147
148 // The ori reads and writes the same register dst.
149 const int dst = inv_rta_field(inst2);
150 if (inv_rs_field(inst2) != dst) return false;
151
152 // Now, find the preceding addis which writes to dst.
153 int inst1 = 0;
154 address inst1_addr = inst2_addr - BytesPerInstWord;
155 while (inst1_addr >= bound) {
156 inst1 = *(int *) inst1_addr;
157 if (is_lis(inst1) && inv_rs_field(inst1) == dst) return true;
158 inst1_addr -= BytesPerInstWord;
159 }
160 return false;
161 }
162 #endif
163
164
165 inline bool MacroAssembler::is_load_const_at(address a) {
166 const int* p_inst = (int *) a;
167 bool b = is_lis(*p_inst++);
168 if (is_ori(*p_inst)) {
169 p_inst++;
170 b = b && is_rldicr(*p_inst++); // TODO: could be made more precise: `sldi'!
171 b = b && is_oris(*p_inst++);
172 b = b && is_ori(*p_inst);
173 } else if (is_lis(*p_inst)) {
174 p_inst++;
175 b = b && is_ori(*p_inst++);
176 b = b && is_ori(*p_inst);
177 // TODO: could enhance reliability by adding is_insrdi
178 } else return false;
179 return b;
180 }
181
182 inline void MacroAssembler::set_oop_constant(jobject obj, Register d) {
183 set_oop(constant_oop_address(obj), d);
184 }
185
186 inline void MacroAssembler::set_oop(AddressLiteral obj_addr, Register d) {
187 assert(obj_addr.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
188 load_const(d, obj_addr);
189 }
190
191 inline void MacroAssembler::pd_patch_instruction(address branch, address target, const char* file, int line) {
192 jint& stub_inst = *(jint*) branch;
193 stub_inst = patched_branch(target - branch, stub_inst, 0);
194 }
195
196 // Relocation of conditional far branches.
197 inline bool MacroAssembler::is_bc_far_variant1_at(address instruction_addr) {
198 // Variant 1, the 1st instruction contains the destination address:
199 //
200 // bcxx DEST
201 // nop
202 //
203 const int instruction_1 = *(int*)(instruction_addr);
204 const int instruction_2 = *(int*)(instruction_addr + 4);
205 return is_bcxx(instruction_1) &&
206 (inv_bd_field(instruction_1, (intptr_t)instruction_addr) != (intptr_t)(instruction_addr + 2*4)) &&
207 is_nop(instruction_2);
208 }
209
210 // Relocation of conditional far branches.
211 inline bool MacroAssembler::is_bc_far_variant2_at(address instruction_addr) {
212 // Variant 2, the 2nd instruction contains the destination address:
213 //
214 // b!cxx SKIP
215 // bxx DEST
216 // SKIP:
217 //
218 const int instruction_1 = *(int*)(instruction_addr);
219 const int instruction_2 = *(int*)(instruction_addr + 4);
220 return is_bcxx(instruction_1) &&
221 (inv_bd_field(instruction_1, (intptr_t)instruction_addr) == (intptr_t)(instruction_addr + 2*4)) &&
222 is_bxx(instruction_2);
223 }
224
225 // Relocation for conditional branches
226 inline bool MacroAssembler::is_bc_far_variant3_at(address instruction_addr) {
227 // Variant 3, far cond branch to the next instruction, already patched to nops:
228 //
229 // nop
230 // endgroup
231 // SKIP/DEST:
232 //
233 const int instruction_1 = *(int*)(instruction_addr);
234 const int instruction_2 = *(int*)(instruction_addr + 4);
235 return is_nop(instruction_1) &&
236 is_endgroup(instruction_2);
237 }
238
239
240 // Convenience bc_far versions
241 inline void MacroAssembler::blt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, less), L, optimize); }
242 inline void MacroAssembler::bgt_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, greater), L, optimize); }
243 inline void MacroAssembler::beq_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, equal), L, optimize); }
244 inline void MacroAssembler::bso_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs1, bi0(crx, summary_overflow), L, optimize); }
245 inline void MacroAssembler::bge_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, less), L, optimize); }
246 inline void MacroAssembler::ble_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, greater), L, optimize); }
247 inline void MacroAssembler::bne_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, equal), L, optimize); }
248 inline void MacroAssembler::bns_far(ConditionRegister crx, Label& L, int optimize) { MacroAssembler::bc_far(bcondCRbiIs0, bi0(crx, summary_overflow), L, optimize); }
249
250 inline address MacroAssembler::call_stub(Register function_entry) {
251 mtctr(function_entry);
252 bctrl();
253 return pc();
254 }
255
256 inline void MacroAssembler::call_stub_and_return_to(Register function_entry, Register return_pc) {
257 assert_different_registers(function_entry, return_pc);
258 mtlr(return_pc);
259 mtctr(function_entry);
260 bctr();
261 }
262
263 // Get the pc where the last emitted call will return to.
264 inline address MacroAssembler::last_calls_return_pc() {
265 return _last_calls_return_pc;
266 }
267
268 // Read from the polling page, its address is already in a register.
269 inline void MacroAssembler::load_from_polling_page(Register polling_page_address, int offset) {
270 if (USE_POLL_BIT_ONLY) {
271 int encoding = SafepointMechanism::poll_bit();
272 tdi(traptoGreaterThanUnsigned | traptoEqual, polling_page_address, encoding);
273 } else {
274 ld(R0, offset, polling_page_address);
275 }
276 }
277
278 // Trap-instruction-based checks.
279
280 inline void MacroAssembler::trap_null_check(Register a, trap_to_bits cmp) {
281 assert(TrapBasedNullChecks, "sanity");
282 tdi(cmp, a/*reg a*/, 0);
283 }
284
285 inline void MacroAssembler::trap_ic_miss_check(Register a, Register b) {
286 td(traptoGreaterThanUnsigned | traptoLessThanUnsigned, a, b);
287 }
288
289 // Do an explicit null check if access to a+offset will not raise a SIGSEGV.
290 // Either issue a trap instruction that raises SIGTRAP, or do a compare that
291 // branches to exception_entry.
292 // No support for compressed oops (base page of heap). Does not distinguish
293 // loads and stores.
294 inline void MacroAssembler::null_check_throw(Register a, int offset, Register temp_reg,
295 address exception_entry) {
296 if (!ImplicitNullChecks || needs_explicit_null_check(offset) || !os::zero_page_read_protected()) {
297 if (TrapBasedNullChecks) {
298 assert(UseSIGTRAP, "sanity");
299 trap_null_check(a);
300 } else {
301 Label ok;
302 cmpdi(CCR0, a, 0);
303 bne(CCR0, ok);
304 load_const_optimized(temp_reg, exception_entry);
305 mtctr(temp_reg);
306 bctr();
307 bind(ok);
308 }
309 }
310 }
311
312 inline void MacroAssembler::null_check(Register a, int offset, Label *Lis_null) {
313 if (!ImplicitNullChecks || needs_explicit_null_check(offset) || !os::zero_page_read_protected()) {
314 if (TrapBasedNullChecks) {
315 assert(UseSIGTRAP, "sanity");
316 trap_null_check(a);
317 } else if (Lis_null){
318 Label ok;
319 cmpdi(CCR0, a, 0);
320 beq(CCR0, *Lis_null);
321 }
322 }
323 }
324
325 inline void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
326 Register base, RegisterOrConstant ind_or_offs, Register val,
327 Register tmp1, Register tmp2, Register tmp3, bool needs_frame) {
328 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
329 ON_UNKNOWN_OOP_REF)) == 0, "unsupported decorator");
330 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
331 bool as_raw = (decorators & AS_RAW) != 0;
332 decorators = AccessInternal::decorator_fixup(decorators);
333 if (as_raw) {
334 bs->BarrierSetAssembler::store_at(this, decorators, type,
335 base, ind_or_offs, val,
336 tmp1, tmp2, tmp3, needs_frame);
337 } else {
338 bs->store_at(this, decorators, type,
339 base, ind_or_offs, val,
340 tmp1, tmp2, tmp3, needs_frame);
341 }
342 }
343
344 inline void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
345 Register base, RegisterOrConstant ind_or_offs, Register dst,
346 Register tmp1, Register tmp2, bool needs_frame, Label *L_handle_null) {
347 assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
348 ON_PHANTOM_OOP_REF | ON_WEAK_OOP_REF)) == 0, "unsupported decorator");
349 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
350 decorators = AccessInternal::decorator_fixup(decorators);
351 bool as_raw = (decorators & AS_RAW) != 0;
352 if (as_raw) {
353 bs->BarrierSetAssembler::load_at(this, decorators, type,
354 base, ind_or_offs, dst,
355 tmp1, tmp2, needs_frame, L_handle_null);
356 } else {
357 bs->load_at(this, decorators, type,
358 base, ind_or_offs, dst,
359 tmp1, tmp2, needs_frame, L_handle_null);
360 }
361 }
362
363 inline void MacroAssembler::load_heap_oop(Register d, RegisterOrConstant offs, Register s1,
364 Register tmp1, Register tmp2,
365 bool needs_frame, DecoratorSet decorators, Label *L_handle_null) {
366 access_load_at(T_OBJECT, IN_HEAP | decorators, s1, offs, d, tmp1, tmp2, needs_frame, L_handle_null);
367 }
368
369 inline void MacroAssembler::store_heap_oop(Register d, RegisterOrConstant offs, Register s1,
370 Register tmp1, Register tmp2, Register tmp3,
371 bool needs_frame, DecoratorSet decorators) {
372 access_store_at(T_OBJECT, IN_HEAP | decorators, s1, offs, d, tmp1, tmp2, tmp3, needs_frame);
373 }
374
375 inline Register MacroAssembler::encode_heap_oop_not_null(Register d, Register src) {
376 Register current = (src != noreg) ? src : d; // Oop to be compressed is in d if no src provided.
377 if (CompressedOops::base_overlaps()) {
378 sub_const_optimized(d, current, CompressedOops::base(), R0);
379 current = d;
380 }
381 if (CompressedOops::shift() != 0) {
382 rldicl(d, current, 64-CompressedOops::shift(), 32); // Clears the upper bits.
383 current = d;
384 }
385 return current; // Encoded oop is in this register.
386 }
387
388 inline Register MacroAssembler::encode_heap_oop(Register d, Register src) {
389 if (CompressedOops::base() != NULL) {
390 if (VM_Version::has_isel()) {
391 cmpdi(CCR0, src, 0);
392 Register co = encode_heap_oop_not_null(d, src);
393 assert(co == d, "sanity");
394 isel_0(d, CCR0, Assembler::equal);
395 } else {
396 Label isNull;
397 or_(d, src, src); // move and compare 0
398 beq(CCR0, isNull);
399 encode_heap_oop_not_null(d, src);
400 bind(isNull);
401 }
402 return d;
403 } else {
404 return encode_heap_oop_not_null(d, src);
405 }
406 }
407
408 inline Register MacroAssembler::decode_heap_oop_not_null(Register d, Register src) {
409 if (CompressedOops::base_disjoint() && src != noreg && src != d &&
410 CompressedOops::shift() != 0) {
411 load_const_optimized(d, CompressedOops::base(), R0);
412 rldimi(d, src, CompressedOops::shift(), 32-CompressedOops::shift());
413 return d;
414 }
415
416 Register current = (src != noreg) ? src : d; // Compressed oop is in d if no src provided.
417 if (CompressedOops::shift() != 0) {
418 sldi(d, current, CompressedOops::shift());
419 current = d;
420 }
421 if (CompressedOops::base() != NULL) {
422 add_const_optimized(d, current, CompressedOops::base(), R0);
423 current = d;
424 }
425 return current; // Decoded oop is in this register.
426 }
427
428 inline void MacroAssembler::decode_heap_oop(Register d) {
429 Label isNull;
430 bool use_isel = false;
431 if (CompressedOops::base() != NULL) {
432 cmpwi(CCR0, d, 0);
433 if (VM_Version::has_isel()) {
434 use_isel = true;
435 } else {
436 beq(CCR0, isNull);
437 }
438 }
439 decode_heap_oop_not_null(d);
440 if (use_isel) {
441 isel_0(d, CCR0, Assembler::equal);
442 }
443 bind(isNull);
444 }
445
446 // SIGTRAP-based range checks for arrays.
447 inline void MacroAssembler::trap_range_check_l(Register a, Register b) {
448 tw (traptoLessThanUnsigned, a/*reg a*/, b/*reg b*/);
449 }
450 inline void MacroAssembler::trap_range_check_l(Register a, int si16) {
451 twi(traptoLessThanUnsigned, a/*reg a*/, si16);
452 }
453 inline void MacroAssembler::trap_range_check_le(Register a, int si16) {
454 twi(traptoEqual | traptoLessThanUnsigned, a/*reg a*/, si16);
455 }
456 inline void MacroAssembler::trap_range_check_g(Register a, int si16) {
457 twi(traptoGreaterThanUnsigned, a/*reg a*/, si16);
458 }
459 inline void MacroAssembler::trap_range_check_ge(Register a, Register b) {
460 tw (traptoEqual | traptoGreaterThanUnsigned, a/*reg a*/, b/*reg b*/);
461 }
462 inline void MacroAssembler::trap_range_check_ge(Register a, int si16) {
463 twi(traptoEqual | traptoGreaterThanUnsigned, a/*reg a*/, si16);
464 }
465
466 // unsigned integer multiplication 64*64 -> 128 bits
467 inline void MacroAssembler::multiply64(Register dest_hi, Register dest_lo,
468 Register x, Register y) {
469 mulld(dest_lo, x, y);
470 mulhdu(dest_hi, x, y);
471 }
472
473 #if defined(ABI_ELFv2)
474 inline address MacroAssembler::function_entry() { return pc(); }
475 #else
476 inline address MacroAssembler::function_entry() { return emit_fd(); }
477 #endif
478
479 #endif // CPU_PPC_MACROASSEMBLER_PPC_INLINE_HPP