1 /*
2 * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, 2018, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.hpp"
28 #include "code/compiledIC.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "nativeInst_aarch64.hpp"
31 #include "oops/oop.inline.hpp"
32 #include "runtime/handles.hpp"
33 #include "runtime/orderAccess.hpp"
34 #include "runtime/sharedRuntime.hpp"
35 #include "runtime/stubRoutines.hpp"
36 #include "utilities/ostream.hpp"
37 #ifdef COMPILER1
38 #include "c1/c1_Runtime1.hpp"
39 #endif
40
41 void NativeCall::verify() {
42 assert(NativeCall::is_call_at((address)this), "unexpected code at call site");
43 }
44
45 void NativeInstruction::wrote(int offset) {
46 ICache::invalidate_word(addr_at(offset));
47 }
48
49 void NativeLoadGot::report_and_fail() const {
50 tty->print_cr("Addr: " INTPTR_FORMAT, p2i(instruction_address()));
51 fatal("not a indirect rip mov to rbx");
52 }
53
54 void NativeLoadGot::verify() const {
55 assert(is_adrp_at((address)this), "must be adrp");
56 }
57
58 address NativeLoadGot::got_address() const {
59 return MacroAssembler::target_addr_for_insn((address)this);
60 }
61
62 intptr_t NativeLoadGot::data() const {
63 return *(intptr_t *) got_address();
64 }
65
66 address NativePltCall::destination() const {
67 NativeGotJump* jump = nativeGotJump_at(plt_jump());
68 return *(address*)MacroAssembler::target_addr_for_insn((address)jump);
69 }
70
71 address NativePltCall::plt_entry() const {
72 return MacroAssembler::target_addr_for_insn((address)this);
73 }
74
75 address NativePltCall::plt_jump() const {
76 address entry = plt_entry();
77 // Virtual PLT code has move instruction first
78 if (((NativeGotJump*)entry)->is_GotJump()) {
79 return entry;
80 } else {
81 return nativeLoadGot_at(entry)->next_instruction_address();
82 }
83 }
84
85 address NativePltCall::plt_load_got() const {
86 address entry = plt_entry();
87 if (!((NativeGotJump*)entry)->is_GotJump()) {
88 // Virtual PLT code has move instruction first
89 return entry;
90 } else {
91 // Static PLT code has move instruction second (from c2i stub)
92 return nativeGotJump_at(entry)->next_instruction_address();
93 }
94 }
95
96 address NativePltCall::plt_c2i_stub() const {
97 address entry = plt_load_got();
98 // This method should be called only for static calls which has C2I stub.
99 NativeLoadGot* load = nativeLoadGot_at(entry);
100 return entry;
101 }
102
103 address NativePltCall::plt_resolve_call() const {
104 NativeGotJump* jump = nativeGotJump_at(plt_jump());
105 address entry = jump->next_instruction_address();
106 if (((NativeGotJump*)entry)->is_GotJump()) {
107 return entry;
108 } else {
109 // c2i stub 2 instructions
110 entry = nativeLoadGot_at(entry)->next_instruction_address();
111 return nativeGotJump_at(entry)->next_instruction_address();
112 }
113 }
114
115 void NativePltCall::reset_to_plt_resolve_call() {
116 set_destination_mt_safe(plt_resolve_call());
117 }
118
119 void NativePltCall::set_destination_mt_safe(address dest) {
120 // rewriting the value in the GOT, it should always be aligned
121 NativeGotJump* jump = nativeGotJump_at(plt_jump());
122 address* got = (address *) jump->got_address();
123 *got = dest;
124 }
125
126 void NativePltCall::set_stub_to_clean() {
127 NativeLoadGot* method_loader = nativeLoadGot_at(plt_c2i_stub());
128 NativeGotJump* jump = nativeGotJump_at(method_loader->next_instruction_address());
129 method_loader->set_data(0);
130 jump->set_jump_destination((address)-1);
131 }
132
133 void NativePltCall::verify() const {
134 assert(NativeCall::is_call_at((address)this), "unexpected code at call site");
135 }
136
137 address NativeGotJump::got_address() const {
138 return MacroAssembler::target_addr_for_insn((address)this);
139 }
140
141 address NativeGotJump::destination() const {
142 address *got_entry = (address *) got_address();
143 return *got_entry;
144 }
145
146 bool NativeGotJump::is_GotJump() const {
147 NativeInstruction *insn =
148 nativeInstruction_at(addr_at(3 * NativeInstruction::instruction_size));
149 return insn->encoding() == 0xd61f0200; // br x16
150 }
151
152 void NativeGotJump::verify() const {
153 assert(is_adrp_at((address)this), "must be adrp");
154 }
155
156 address NativeCall::destination() const {
157 address addr = (address)this;
158 address destination = instruction_address() + displacement();
159
160 // Do we use a trampoline stub for this call?
161 CodeBlob* cb = CodeCache::find_blob_unsafe(addr); // Else we get assertion if nmethod is zombie.
162 assert(cb && cb->is_nmethod(), "sanity");
163 nmethod *nm = (nmethod *)cb;
164 if (nm->stub_contains(destination) && is_NativeCallTrampolineStub_at(destination)) {
165 // Yes we do, so get the destination from the trampoline stub.
166 const address trampoline_stub_addr = destination;
167 destination = nativeCallTrampolineStub_at(trampoline_stub_addr)->destination();
168 }
169
170 return destination;
171 }
172
173 // Similar to replace_mt_safe, but just changes the destination. The
174 // important thing is that free-running threads are able to execute this
175 // call instruction at all times.
176 //
177 // Used in the runtime linkage of calls; see class CompiledIC.
178 //
179 // Add parameter assert_lock to switch off assertion
180 // during code generation, where no patching lock is needed.
181 void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) {
182 assert(!assert_lock ||
183 (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) ||
184 CompiledICLocker::is_safe(addr_at(0)),
185 "concurrent code patching");
186
187 ResourceMark rm;
188 int code_size = NativeInstruction::instruction_size;
189 address addr_call = addr_at(0);
190 bool reachable = Assembler::reachable_from_branch_at(addr_call, dest);
191 assert(NativeCall::is_call_at(addr_call), "unexpected code at call site");
192
193 // Patch the constant in the call's trampoline stub.
194 address trampoline_stub_addr = get_trampoline();
195 if (trampoline_stub_addr != NULL) {
196 assert (! is_NativeCallTrampolineStub_at(dest), "chained trampolines");
197 nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest);
198 }
199
200 // Patch the call.
201 if (reachable) {
202 set_destination(dest);
203 } else {
204 assert (trampoline_stub_addr != NULL, "we need a trampoline");
205 set_destination(trampoline_stub_addr);
206 }
207
208 ICache::invalidate_range(addr_call, instruction_size);
209 }
210
211 address NativeCall::get_trampoline() {
212 address call_addr = addr_at(0);
213
214 CodeBlob *code = CodeCache::find_blob(call_addr);
215 assert(code != NULL, "Could not find the containing code blob");
216
217 address bl_destination
218 = MacroAssembler::pd_call_destination(call_addr);
219 if (code->contains(bl_destination) &&
220 is_NativeCallTrampolineStub_at(bl_destination))
221 return bl_destination;
222
223 if (code->is_nmethod()) {
224 return trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code);
225 }
226
227 return NULL;
228 }
229
230 // Inserts a native call instruction at a given pc
231 void NativeCall::insert(address code_pos, address entry) { Unimplemented(); }
232
233 //-------------------------------------------------------------------
234
235 void NativeMovConstReg::verify() {
236 if (! (nativeInstruction_at(instruction_address())->is_movz() ||
237 is_adrp_at(instruction_address()) ||
238 is_ldr_literal_at(instruction_address())) ) {
239 fatal("should be MOVZ or ADRP or LDR (literal)");
240 }
241 }
242
243
244 intptr_t NativeMovConstReg::data() const {
245 // das(uint64_t(instruction_address()),2);
246 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
247 if (maybe_cpool_ref(instruction_address())) {
248 return *(intptr_t*)addr;
249 } else {
250 return (intptr_t)addr;
251 }
252 }
253
254 void NativeMovConstReg::set_data(intptr_t x) {
255 if (maybe_cpool_ref(instruction_address())) {
256 address addr = MacroAssembler::target_addr_for_insn(instruction_address());
257 *(intptr_t*)addr = x;
258 } else {
259 // Store x into the instruction stream.
260 MacroAssembler::pd_patch_instruction(instruction_address(), (address)x);
261 ICache::invalidate_range(instruction_address(), instruction_size);
262 }
263
264 // Find and replace the oop/metadata corresponding to this
265 // instruction in oops section.
266 CodeBlob* cb = CodeCache::find_blob(instruction_address());
267 nmethod* nm = cb->as_nmethod_or_null();
268 if (nm != NULL) {
269 RelocIterator iter(nm, instruction_address(), next_instruction_address());
270 while (iter.next()) {
271 if (iter.type() == relocInfo::oop_type) {
272 oop* oop_addr = iter.oop_reloc()->oop_addr();
273 *oop_addr = cast_to_oop(x);
274 break;
275 } else if (iter.type() == relocInfo::metadata_type) {
276 Metadata** metadata_addr = iter.metadata_reloc()->metadata_addr();
277 *metadata_addr = (Metadata*)x;
278 break;
279 }
280 }
281 }
282 }
283
284 void NativeMovConstReg::print() {
285 tty->print_cr(PTR_FORMAT ": mov reg, " INTPTR_FORMAT,
286 p2i(instruction_address()), data());
287 }
288
289 //-------------------------------------------------------------------
290
291 int NativeMovRegMem::offset() const {
292 address pc = instruction_address();
293 unsigned insn = *(unsigned*)pc;
294 if (Instruction_aarch64::extract(insn, 28, 24) == 0b10000) {
295 address addr = MacroAssembler::target_addr_for_insn(pc);
296 return *addr;
297 } else {
298 return (int)(intptr_t)MacroAssembler::target_addr_for_insn(instruction_address());
299 }
300 }
301
302 void NativeMovRegMem::set_offset(int x) {
303 address pc = instruction_address();
304 unsigned insn = *(unsigned*)pc;
305 if (maybe_cpool_ref(pc)) {
306 address addr = MacroAssembler::target_addr_for_insn(pc);
307 *(long*)addr = x;
308 } else {
309 MacroAssembler::pd_patch_instruction(pc, (address)intptr_t(x));
310 ICache::invalidate_range(instruction_address(), instruction_size);
311 }
312 }
313
314 void NativeMovRegMem::verify() {
315 #ifdef ASSERT
316 address dest = MacroAssembler::target_addr_for_insn(instruction_address());
317 #endif
318 }
319
320 //--------------------------------------------------------------------------------
321
322 void NativeJump::verify() { ; }
323
324
325 void NativeJump::check_verified_entry_alignment(address entry, address verified_entry) {
326 }
327
328
329 address NativeJump::jump_destination() const {
330 address dest = MacroAssembler::target_addr_for_insn(instruction_address());
331
332 // We use jump to self as the unresolved address which the inline
333 // cache code (and relocs) know about
334 // As a special case we also use sequence movptr(r,0); br(r);
335 // i.e. jump to 0 when we need leave space for a wide immediate
336 // load
337
338 // return -1 if jump to self or to 0
339 if ((dest == (address)this) || dest == 0) {
340 dest = (address) -1;
341 }
342 return dest;
343 }
344
345 void NativeJump::set_jump_destination(address dest) {
346 // We use jump to self as the unresolved address which the inline
347 // cache code (and relocs) know about
348 if (dest == (address) -1)
349 dest = instruction_address();
350
351 MacroAssembler::pd_patch_instruction(instruction_address(), dest);
352 ICache::invalidate_range(instruction_address(), instruction_size);
353 };
354
355 //-------------------------------------------------------------------
356
357 address NativeGeneralJump::jump_destination() const {
358 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
359 address dest = (address) move->data();
360
361 // We use jump to self as the unresolved address which the inline
362 // cache code (and relocs) know about
363 // As a special case we also use jump to 0 when first generating
364 // a general jump
365
366 // return -1 if jump to self or to 0
367 if ((dest == (address)this) || dest == 0) {
368 dest = (address) -1;
369 }
370 return dest;
371 }
372
373 void NativeGeneralJump::set_jump_destination(address dest) {
374 NativeMovConstReg* move = nativeMovConstReg_at(instruction_address());
375
376 // We use jump to self as the unresolved address which the inline
377 // cache code (and relocs) know about
378 if (dest == (address) -1) {
379 dest = instruction_address();
380 }
381
382 move->set_data((uintptr_t) dest);
383 };
384
385 //-------------------------------------------------------------------
386
387 bool NativeInstruction::is_safepoint_poll() {
388 // a safepoint_poll is implemented in two steps as either
389 //
390 // adrp(reg, polling_page);
391 // ldr(zr, [reg, #offset]);
392 //
393 // or
394 //
395 // mov(reg, polling_page);
396 // ldr(zr, [reg, #offset]);
397 //
398 // or
399 //
400 // ldr(reg, [rthread, #offset]);
401 // ldr(zr, [reg, #offset]);
402 //
403 // however, we cannot rely on the polling page address load always
404 // directly preceding the read from the page. C1 does that but C2
405 // has to do the load and read as two independent instruction
406 // generation steps. that's because with a single macro sequence the
407 // generic C2 code can only add the oop map before the mov/adrp and
408 // the trap handler expects an oop map to be associated with the
409 // load. with the load scheuled as a prior step the oop map goes
410 // where it is needed.
411 //
412 // so all we can do here is check that marked instruction is a load
413 // word to zr
414 return is_ldrw_to_zr(address(this));
415 }
416
417 bool NativeInstruction::is_adrp_at(address instr) {
418 unsigned insn = *(unsigned*)instr;
419 return (Instruction_aarch64::extract(insn, 31, 24) & 0b10011111) == 0b10010000;
420 }
421
422 bool NativeInstruction::is_ldr_literal_at(address instr) {
423 unsigned insn = *(unsigned*)instr;
424 return (Instruction_aarch64::extract(insn, 29, 24) & 0b011011) == 0b00011000;
425 }
426
427 bool NativeInstruction::is_ldrw_to_zr(address instr) {
428 unsigned insn = *(unsigned*)instr;
429 return (Instruction_aarch64::extract(insn, 31, 22) == 0b1011100101 &&
430 Instruction_aarch64::extract(insn, 4, 0) == 0b11111);
431 }
432
433 bool NativeInstruction::is_general_jump() {
434 if (is_movz()) {
435 NativeInstruction* inst1 = nativeInstruction_at(addr_at(instruction_size * 1));
436 if (inst1->is_movk()) {
437 NativeInstruction* inst2 = nativeInstruction_at(addr_at(instruction_size * 2));
438 if (inst2->is_movk()) {
439 NativeInstruction* inst3 = nativeInstruction_at(addr_at(instruction_size * 3));
440 if (inst3->is_blr()) {
441 return true;
442 }
443 }
444 }
445 }
446 return false;
447 }
448
449 bool NativeInstruction::is_movz() {
450 return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b10100101;
451 }
452
453 bool NativeInstruction::is_movk() {
454 return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b11100101;
455 }
456
457 bool NativeInstruction::is_sigill_zombie_not_entrant() {
458 return uint_at(0) == 0xd4bbd5a1; // dcps1 #0xdead
459 }
460
461 void NativeIllegalInstruction::insert(address code_pos) {
462 *(juint*)code_pos = 0xd4bbd5a1; // dcps1 #0xdead
463 }
464
465 bool NativeInstruction::is_stop() {
466 return uint_at(0) == 0xd4bbd5c1; // dcps1 #0xdeae
467 }
468
469 //-------------------------------------------------------------------
470
471 // MT-safe inserting of a jump over a jump or a nop (used by
472 // nmethod::make_not_entrant_or_zombie)
473
474 void NativeJump::patch_verified_entry(address entry, address verified_entry, address dest) {
475
476 assert(dest == SharedRuntime::get_handle_wrong_method_stub(), "expected fixed destination of patch");
477
478 #ifdef ASSERT
479 // This may be the temporary nmethod generated while we're AOT
480 // compiling. Such an nmethod doesn't begin with a NOP but with an ADRP.
481 if (! (CalculateClassFingerprint && UseAOT && is_adrp_at(verified_entry))) {
482 assert(nativeInstruction_at(verified_entry)->is_jump_or_nop()
483 || nativeInstruction_at(verified_entry)->is_sigill_zombie_not_entrant(),
484 "Aarch64 cannot replace non-jump with jump");
485 }
486 #endif
487
488 // Patch this nmethod atomically.
489 if (Assembler::reachable_from_branch_at(verified_entry, dest)) {
490 ptrdiff_t disp = dest - verified_entry;
491 guarantee(disp < 1 << 27 && disp > - (1 << 27), "branch overflow");
492
493 unsigned int insn = (0b000101 << 26) | ((disp >> 2) & 0x3ffffff);
494 *(unsigned int*)verified_entry = insn;
495 } else {
496 // We use an illegal instruction for marking a method as
497 // not_entrant or zombie.
498 NativeIllegalInstruction::insert(verified_entry);
499 }
500
501 ICache::invalidate_range(verified_entry, instruction_size);
502 }
503
504 void NativeGeneralJump::verify() { }
505
506 void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
507 NativeGeneralJump* n_jump = (NativeGeneralJump*)code_pos;
508
509 CodeBuffer cb(code_pos, instruction_size);
510 MacroAssembler a(&cb);
511
512 a.movptr(rscratch1, (uintptr_t)entry);
513 a.br(rscratch1);
514
515 ICache::invalidate_range(code_pos, instruction_size);
516 }
517
518 // MT-safe patching of a long jump instruction.
519 void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
520 ShouldNotCallThis();
521 }
522
523 address NativeCallTrampolineStub::destination(nmethod *nm) const {
524 return ptr_at(data_offset);
525 }
526
527 void NativeCallTrampolineStub::set_destination(address new_destination) {
528 set_ptr_at(data_offset, new_destination);
529 OrderAccess::release();
530 }
531
532 // Generate a trampoline for a branch to dest. If there's no need for a
533 // trampoline, simply patch the call directly to dest.
534 address NativeCall::trampoline_jump(CodeBuffer &cbuf, address dest) {
535 MacroAssembler a(&cbuf);
536 address stub = NULL;
537
538 if (a.far_branches()
539 && ! is_NativeCallTrampolineStub_at(instruction_address() + displacement())) {
540 stub = a.emit_trampoline_stub(instruction_address() - cbuf.insts()->start(), dest);
541 }
542
543 if (stub == NULL) {
544 // If we generated no stub, patch this call directly to dest.
545 // This will happen if we don't need far branches or if there
546 // already was a trampoline.
547 set_destination(dest);
548 }
549
550 return stub;
551 }