1 /*
2 * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
3 * Copyright 2012, 2013 SAP AG. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "asm/macroAssembler.inline.hpp"
30 #include "interp_masm_ppc_64.hpp"
31 #include "interpreter/interpreterRuntime.hpp"
32
33
34 #ifdef PRODUCT
35 #define BLOCK_COMMENT(str) // nothing
36 #else
37 #define BLOCK_COMMENT(str) block_comment(str)
38 #endif
39
40 // Lock object
41 //
42 // Registers alive
43 // monitor - Address of the BasicObjectLock to be used for locking,
44 // which must be initialized with the object to lock.
45 // object - Address of the object to be locked.
46 //
47 void InterpreterMacroAssembler::lock_object(Register monitor, Register object) {
48 if (UseHeavyMonitors) {
49 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
50 monitor, /*check_for_exceptions=*/false);
51 } else {
52 // template code:
53 //
54 // markOop displaced_header = obj->mark().set_unlocked();
55 // monitor->lock()->set_displaced_header(displaced_header);
56 // if (Atomic::cmpxchg_ptr(/*ex=*/monitor, /*addr*/obj->mark_addr(), /*cmp*/displaced_header) == displaced_header) {
57 // // We stored the monitor address into the object's mark word.
58 // } else if (THREAD->is_lock_owned((address)displaced_header))
59 // // Simple recursive case.
60 // monitor->lock()->set_displaced_header(NULL);
61 // } else {
62 // // Slow path.
63 // InterpreterRuntime::monitorenter(THREAD, monitor);
64 // }
65
66 const Register displaced_header = R7_ARG5;
67 const Register object_mark_addr = R8_ARG6;
68 const Register current_header = R9_ARG7;
69 const Register tmp = R10_ARG8;
70
71 Label done;
72 Label slow_case;
73
74 assert_different_registers(displaced_header, object_mark_addr, current_header, tmp);
75
76
77 // markOop displaced_header = obj->mark().set_unlocked();
78
79 // Load markOop from object into displaced_header.
80 ld(displaced_header, oopDesc::mark_offset_in_bytes(), object);
81
82 if (UseBiasedLocking) {
83 biased_locking_enter(CCR0, object, displaced_header, tmp, current_header, done, &slow_case);
84 }
85
86 // Set displaced_header to be (markOop of object | UNLOCK_VALUE).
87 ori(displaced_header, displaced_header, markOopDesc::unlocked_value);
88
89
90 // monitor->lock()->set_displaced_header(displaced_header);
91
92 // Initialize the box (Must happen before we update the object mark!).
93 std(displaced_header, BasicObjectLock::lock_offset_in_bytes() +
94 BasicLock::displaced_header_offset_in_bytes(), monitor);
95
96 // if (Atomic::cmpxchg_ptr(/*ex=*/monitor, /*addr*/obj->mark_addr(), /*cmp*/displaced_header) == displaced_header) {
97
98 // Store stack address of the BasicObjectLock (this is monitor) into object.
99 addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());
100
101 // Must fence, otherwise, preceding store(s) may float below cmpxchg.
102 // CmpxchgX sets CCR0 to cmpX(current, displaced).
103 fence(); // TODO: replace by MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq ?
104 cmpxchgd(/*flag=*/CCR0,
105 /*current_value=*/current_header,
106 /*compare_value=*/displaced_header, /*exchange_value=*/monitor,
107 /*where=*/object_mark_addr,
108 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
109 MacroAssembler::cmpxchgx_hint_acquire_lock());
110
111 // If the compare-and-exchange succeeded, then we found an unlocked
112 // object and we have now locked it.
113 beq(CCR0, done);
114
115
116 // } else if (THREAD->is_lock_owned((address)displaced_header))
117 // // Simple recursive case.
118 // monitor->lock()->set_displaced_header(NULL);
119
120 // We did not see an unlocked object so try the fast recursive case.
121
122 // Check if owner is self by comparing the value in the markOop of object
123 // (current_header) with the stack pointer.
124 sub(current_header, current_header, R1_SP);
125
126 assert(os::vm_page_size() > 0xfff, "page size too small - change the constant");
127 load_const_optimized(tmp,
128 (address) (~(os::vm_page_size()-1) |
129 markOopDesc::lock_mask_in_place));
130
131 and_(R0/*==0?*/, current_header, tmp);
132 // If condition is true we are done and hence we can store 0 in the displaced
133 // header indicating it is a recursive lock.
134 bne(CCR0, slow_case);
135 release();
136 std(R0/*==0!*/, BasicObjectLock::lock_offset_in_bytes() +
137 BasicLock::displaced_header_offset_in_bytes(), monitor);
138 b(done);
139
140
141 // } else {
142 // // Slow path.
143 // InterpreterRuntime::monitorenter(THREAD, monitor);
144
145 // None of the above fast optimizations worked so we have to get into the
146 // slow case of monitor enter.
147 bind(slow_case);
148 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
149 monitor, /*check_for_exceptions=*/false);
150 // }
151
152 bind(done);
153 }
154 }
155
156 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
157 //
158 // Registers alive
159 // monitor - Address of the BasicObjectLock to be used for locking,
160 // which must be initialized with the object to lock.
161 //
162 // Throw IllegalMonitorException if object is not locked by current thread.
163 void InterpreterMacroAssembler::unlock_object(Register monitor) {
164 if (UseHeavyMonitors) {
165 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit),
166 monitor, /*check_for_exceptions=*/false);
167 } else {
168
169 // template code:
170 //
171 // if ((displaced_header = monitor->displaced_header()) == NULL) {
172 // // Recursive unlock. Mark the monitor unlocked by setting the object field to NULL.
173 // monitor->set_obj(NULL);
174 // } else if (Atomic::cmpxchg_ptr(displaced_header, obj->mark_addr(), monitor) == monitor) {
175 // // We swapped the unlocked mark in displaced_header into the object's mark word.
176 // monitor->set_obj(NULL);
177 // } else {
178 // // Slow path.
179 // InterpreterRuntime::monitorexit(THREAD, monitor);
180 // }
181
182 const Register object = R7_ARG5;
183 const Register displaced_header = R8_ARG6;
184 const Register object_mark_addr = R9_ARG7;
185 const Register current_header = R10_ARG8;
186
187 Label no_recursive_unlock;
188 Label slow_case;
189 Label done;
190
191 assert_different_registers(object, displaced_header, object_mark_addr, current_header);
192
193 if (UseBiasedLocking) {
194 // The object address from the monitor is in object.
195 ld(object, BasicObjectLock::obj_offset_in_bytes(), monitor);
196 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
197 biased_locking_exit(CCR0, object, displaced_header, done);
198 }
199
200 // Test first if we are in the fast recursive case.
201 ld(displaced_header, BasicObjectLock::lock_offset_in_bytes() +
202 BasicLock::displaced_header_offset_in_bytes(), monitor);
203
204 // If the displaced header is zero, we have a recursive unlock.
205 cmpdi(CCR0, displaced_header, 0);
206 bne(CCR0, no_recursive_unlock);
207 // Release in recursive unlock is not necessary.
208 // release();
209 std(displaced_header/*==0!*/, BasicObjectLock::obj_offset_in_bytes(), monitor);
210 b(done);
211
212 bind(no_recursive_unlock);
213
214 // } else if (Atomic::cmpxchg_ptr(displaced_header, obj->mark_addr(), monitor) == monitor) {
215 // // We swapped the unlocked mark in displaced_header into the object's mark word.
216 // monitor->set_obj(NULL);
217
218 // If we still have a lightweight lock, unlock the object and be done.
219
220 // The object address from the monitor is in object.
221 ld(object, BasicObjectLock::obj_offset_in_bytes(), monitor);
222 addi(object_mark_addr, object, oopDesc::mark_offset_in_bytes());
223
224 // We have the displaced header in displaced_header. If the lock is still
225 // lightweight, it will contain the monitor address and we'll store the
226 // displaced header back into the object's mark word.
227 // CmpxchgX sets CCR0 to cmpX(current, monitor).
228 cmpxchgd(/*flag=*/CCR0,
229 /*current_value=*/current_header,
230 /*compare_value=*/monitor, /*exchange_value=*/displaced_header,
231 /*where=*/object_mark_addr,
232 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
233 MacroAssembler::cmpxchgx_hint_release_lock());
234 bne(CCR0, slow_case);
235
236 // Exchange worked, do monitor->set_obj(NULL).
237 li(R0, 0);
238 // Must realease earlier (see cmpxchgd above).
239 // release();
240 std(R0, BasicObjectLock::obj_offset_in_bytes(), monitor);
241 b(done);
242
243
244 // } else {
245 // // Slow path.
246 // InterpreterRuntime::monitorexit(THREAD, monitor);
247
248 // The lock has been converted into a heavy lock and hence
249 // we need to get into the slow case.
250 bind(slow_case);
251 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit),
252 monitor, /*check_for_exceptions=*/false);
253 // }
254
255 bind(done);
256 }
257 }
258
259 void InterpreterMacroAssembler::get_method_counters(Register method,
260 Register Rcounters,
261 Label& skip) {
262 BLOCK_COMMENT("Load and ev. allocate counter object {");
263 Label has_counters;
264 ld(Rcounters, in_bytes(Method::method_counters_offset()), method);
265 cmpdi(CCR0, Rcounters, 0);
266 bne(CCR0, has_counters);
267 call_VM(noreg, CAST_FROM_FN_PTR(address,
268 InterpreterRuntime::build_method_counters), method, false);
269 ld(Rcounters, in_bytes(Method::method_counters_offset()), method);
270 cmpdi(CCR0, Rcounters, 0);
271 beq(CCR0, skip); // No MethodCounters, OutOfMemory.
272 BLOCK_COMMENT("} Load and ev. allocate counter object");
273
274 bind(has_counters);
275 }
276
277 void InterpreterMacroAssembler::increment_invocation_counter(Register Rcounters, Register iv_be_count, Register Rtmp_r0) {
278 assert(UseCompiler, "incrementing must be useful");
279 Register invocation_count = iv_be_count;
280 Register backedge_count = Rtmp_r0;
281 int delta = InvocationCounter::count_increment;
282
283 // Load each counter in a register.
284 // ld(inv_counter, Rtmp);
285 // ld(be_counter, Rtmp2);
286 int inv_counter_offset = in_bytes(MethodCounters::invocation_counter_offset() +
287 InvocationCounter::counter_offset());
288 int be_counter_offset = in_bytes(MethodCounters::backedge_counter_offset() +
289 InvocationCounter::counter_offset());
290
291 BLOCK_COMMENT("Increment profiling counters {");
292
293 // Load the backedge counter.
294 lwz(backedge_count, be_counter_offset, Rcounters); // is unsigned int
295 // Mask the backedge counter.
296 Register tmp = invocation_count;
297 li(tmp, InvocationCounter::count_mask_value);
298 andr(backedge_count, tmp, backedge_count); // Cannot use andi, need sign extension of count_mask_value.
299
300 // Load the invocation counter.
301 lwz(invocation_count, inv_counter_offset, Rcounters); // is unsigned int
302 // Add the delta to the invocation counter and store the result.
303 addi(invocation_count, invocation_count, delta);
304 // Store value.
305 stw(invocation_count, inv_counter_offset, Rcounters);
306
307 // Add invocation counter + backedge counter.
308 add(iv_be_count, backedge_count, invocation_count);
309
310 // Note that this macro must leave the backedge_count + invocation_count in
311 // register iv_be_count!
312 BLOCK_COMMENT("} Increment profiling counters");
313 }
314
315 void InterpreterMacroAssembler::verify_oop(Register reg, TosState state) {
316 if (state == atos) { MacroAssembler::verify_oop(reg); }
317 }
318
319 // Inline assembly for:
320 //
321 // if (thread is in interp_only_mode) {
322 // InterpreterRuntime::post_method_entry();
323 // }
324 // if (*jvmpi::event_flags_array_at_addr(JVMPI_EVENT_METHOD_ENTRY ) ||
325 // *jvmpi::event_flags_array_at_addr(JVMPI_EVENT_METHOD_ENTRY2) ) {
326 // SharedRuntime::jvmpi_method_entry(method, receiver);
327 // }
328 void InterpreterMacroAssembler::notify_method_entry() {
329 // JVMTI
330 // Whenever JVMTI puts a thread in interp_only_mode, method
331 // entry/exit events are sent for that thread to track stack
332 // depth. If it is possible to enter interp_only_mode we add
333 // the code to check if the event should be sent.
334 if (JvmtiExport::can_post_interpreter_events()) {
335 Label jvmti_post_done;
336
337 lwz(R0, in_bytes(JavaThread::interp_only_mode_offset()), R16_thread);
338 cmpwi(CCR0, R0, 0);
339 beq(CCR0, jvmti_post_done);
340 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry),
341 /*check_exceptions=*/false);
342
343 bind(jvmti_post_done);
344 }
345 }
346
347
348 // Inline assembly for:
349 //
350 // if (thread is in interp_only_mode) {
351 // // save result
352 // InterpreterRuntime::post_method_exit();
353 // // restore result
354 // }
355 // if (*jvmpi::event_flags_array_at_addr(JVMPI_EVENT_METHOD_EXIT)) {
356 // // save result
357 // SharedRuntime::jvmpi_method_exit();
358 // // restore result
359 // }
360 //
361 // Native methods have their result stored in d_tmp and l_tmp.
362 // Java methods have their result stored in the expression stack.
363 void InterpreterMacroAssembler::notify_method_exit(bool is_native_method, TosState state) {
364 // JVMTI
365 // Whenever JVMTI puts a thread in interp_only_mode, method
366 // entry/exit events are sent for that thread to track stack
367 // depth. If it is possible to enter interp_only_mode we add
368 // the code to check if the event should be sent.
369 if (JvmtiExport::can_post_interpreter_events()) {
370 Label jvmti_post_done;
371
372 lwz(R0, in_bytes(JavaThread::interp_only_mode_offset()), R16_thread);
373 cmpwi(CCR0, R0, 0);
374 beq(CCR0, jvmti_post_done);
375 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit),
376 /*check_exceptions=*/false);
377
378 bind(jvmti_post_done);
379 }
380 }
381
382 // Convert the current TOP_IJAVA_FRAME into a PARENT_IJAVA_FRAME
383 // (using parent_frame_resize) and push a new interpreter
384 // TOP_IJAVA_FRAME (using frame_size).
385 void InterpreterMacroAssembler::push_interpreter_frame(Register top_frame_size, Register parent_frame_resize,
386 Register tmp1, Register tmp2, Register tmp3,
387 Register tmp4, Register pc) {
388 assert_different_registers(top_frame_size, parent_frame_resize, tmp1, tmp2, tmp3, tmp4);
389 ld(tmp1, _top_ijava_frame_abi(frame_manager_lr), R1_SP);
390 mr(tmp2/*top_frame_sp*/, R1_SP);
391 // Move initial_caller_sp.
392 ld(tmp4, _top_ijava_frame_abi(initial_caller_sp), R1_SP);
393 neg(parent_frame_resize, parent_frame_resize);
394 resize_frame(parent_frame_resize/*-parent_frame_resize*/, tmp3);
395
396 // Set LR in new parent frame.
397 std(tmp1, _abi(lr), R1_SP);
398 // Set top_frame_sp info for new parent frame.
399 std(tmp2, _parent_ijava_frame_abi(top_frame_sp), R1_SP);
400 std(tmp4, _parent_ijava_frame_abi(initial_caller_sp), R1_SP);
401
402 // Push new TOP_IJAVA_FRAME.
403 push_frame(top_frame_size, tmp2);
404
405 get_PC_trash_LR(tmp3);
406 std(tmp3, _top_ijava_frame_abi(frame_manager_lr), R1_SP);
407 // Used for non-initial callers by unextended_sp().
408 std(R1_SP, _top_ijava_frame_abi(initial_caller_sp), R1_SP);
409 }
410
411 // Pop the topmost TOP_IJAVA_FRAME and convert the previous
412 // PARENT_IJAVA_FRAME back into a TOP_IJAVA_FRAME.
413 void InterpreterMacroAssembler::pop_interpreter_frame(Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
414 assert_different_registers(tmp1, tmp2, tmp3, tmp4);
415
416 ld(tmp1/*caller's sp*/, _abi(callers_sp), R1_SP);
417 ld(tmp3, _abi(lr), tmp1);
418
419 ld(tmp4, _parent_ijava_frame_abi(initial_caller_sp), tmp1);
420
421 ld(tmp2/*caller's caller's sp*/, _abi(callers_sp), tmp1);
422 // Merge top frame.
423 std(tmp2, _abi(callers_sp), R1_SP);
424
425 ld(tmp2, _parent_ijava_frame_abi(top_frame_sp), tmp1);
426
427 // Update C stack pointer to caller's top_abi.
428 resize_frame_absolute(tmp2/*addr*/, tmp1/*tmp*/, tmp2/*tmp*/);
429
430 // Update LR in top_frame.
431 std(tmp3, _top_ijava_frame_abi(frame_manager_lr), R1_SP);
432
433 std(tmp4, _top_ijava_frame_abi(initial_caller_sp), R1_SP);
434
435 // Store the top-frame stack-pointer for c2i adapters.
436 std(R1_SP, _top_ijava_frame_abi(top_frame_sp), R1_SP);
437 }
438
439 #ifdef CC_INTERP
440 // Turn state's interpreter frame into the current TOP_IJAVA_FRAME.
441 void InterpreterMacroAssembler::pop_interpreter_frame_to_state(Register state, Register tmp1, Register tmp2, Register tmp3) {
442 assert_different_registers(R14_state, R15_prev_state, tmp1, tmp2, tmp3);
443
444 if (state == R14_state) {
445 ld(tmp1/*state's fp*/, state_(_last_Java_fp));
446 ld(tmp2/*state's sp*/, state_(_last_Java_sp));
447 } else if (state == R15_prev_state) {
448 ld(tmp1/*state's fp*/, prev_state_(_last_Java_fp));
449 ld(tmp2/*state's sp*/, prev_state_(_last_Java_sp));
450 } else {
451 ShouldNotReachHere();
452 }
453
454 // Merge top frames.
455 std(tmp1, _abi(callers_sp), R1_SP);
456
457 // Tmp2 is new SP.
458 // Tmp1 is parent's SP.
459 resize_frame_absolute(tmp2/*addr*/, tmp1/*tmp*/, tmp2/*tmp*/);
460
461 // Update LR in top_frame.
462 // Must be interpreter frame.
463 get_PC_trash_LR(tmp3);
464 std(tmp3, _top_ijava_frame_abi(frame_manager_lr), R1_SP);
465 // Used for non-initial callers by unextended_sp().
466 std(R1_SP, _top_ijava_frame_abi(initial_caller_sp), R1_SP);
467 }
468 #endif // CC_INTERP
469
470 // Set SP to initial caller's sp, but before fix the back chain.
471 void InterpreterMacroAssembler::resize_frame_to_initial_caller(Register tmp1, Register tmp2) {
472 ld(tmp1, _parent_ijava_frame_abi(initial_caller_sp), R1_SP);
473 ld(tmp2, _parent_ijava_frame_abi(callers_sp), R1_SP);
474 std(tmp2, _parent_ijava_frame_abi(callers_sp), tmp1); // Fix back chain ...
475 mr(R1_SP, tmp1); // ... and resize to initial caller.
476 }
477
478 #ifdef CC_INTERP
479 // Pop the current interpreter state (without popping the correspoding
480 // frame) and restore R14_state and R15_prev_state accordingly.
481 // Use prev_state_may_be_0 to indicate whether prev_state may be 0
482 // in order to generate an extra check before retrieving prev_state_(_prev_link).
483 void InterpreterMacroAssembler::pop_interpreter_state(bool prev_state_may_be_0)
484 {
485 // Move prev_state to state and restore prev_state from state_(_prev_link).
486 Label prev_state_is_0;
487 mr(R14_state, R15_prev_state);
488
489 // Don't retrieve /*state==*/prev_state_(_prev_link)
490 // if /*state==*/prev_state is 0.
491 if (prev_state_may_be_0) {
492 cmpdi(CCR0, R15_prev_state, 0);
493 beq(CCR0, prev_state_is_0);
494 }
495
496 ld(R15_prev_state, /*state==*/prev_state_(_prev_link));
497 bind(prev_state_is_0);
498 }
499
500 void InterpreterMacroAssembler::restore_prev_state() {
501 // _prev_link is private, but cInterpreter is a friend.
502 ld(R15_prev_state, state_(_prev_link));
503 }
504 #endif // CC_INTERP