1 /*
2 * Copyright (c) 2008, 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "c1/c1_MacroAssembler.hpp"
27 #include "c1/c1_Runtime1.hpp"
28 #include "classfile/systemDictionary.hpp"
29 #include "gc/shared/collectedHeap.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "oops/arrayOop.hpp"
32 #include "oops/markOop.hpp"
33 #include "runtime/basicLock.hpp"
34 #include "runtime/biasedLocking.hpp"
35 #include "runtime/os.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/stubRoutines.hpp"
38
39 // Note: Rtemp usage is this file should not impact C2 and should be
40 // correct as long as it is not implicitly used in lower layers (the
41 // arm [macro]assembler) and used with care in the other C1 specific
42 // files.
43
44 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
45 Label verified;
46 load_klass(Rtemp, receiver);
47 cmp(Rtemp, iCache);
48 b(verified, eq); // jump over alignment no-ops
49 #ifdef AARCH64
50 jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
51 #else
52 jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
53 #endif
54 align(CodeEntryAlignment);
55 bind(verified);
56 }
57
58 void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
59 assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
60 assert((frame_size_in_bytes % StackAlignmentInBytes) == 0, "frame size should be aligned");
61
62 #ifdef AARCH64
63 // Extra nop for MT-safe patching in NativeJump::patch_verified_entry
64 nop();
65 #endif // AARCH64
66
67 arm_stack_overflow_check(bang_size_in_bytes, Rtemp);
68
69 // FP can no longer be used to memorize SP. It may be modified
70 // if this method contains a methodHandle call site
71 raw_push(FP, LR);
72 sub_slow(SP, SP, frame_size_in_bytes);
73 }
74
75 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) {
76 add_slow(SP, SP, frame_size_in_bytes);
77 raw_pop(FP, LR);
78 }
79
80 void C1_MacroAssembler::verified_entry() {
81 if (C1Breakpoint) {
82 breakpoint();
83 }
84 }
85
86 // Puts address of allocated object into register `obj` and end of allocated object into register `obj_end`.
87 void C1_MacroAssembler::try_allocate(Register obj, Register obj_end, Register tmp1, Register tmp2,
88 RegisterOrConstant size_expression, Label& slow_case) {
89 if (UseTLAB) {
90 tlab_allocate(obj, obj_end, tmp1, size_expression, slow_case);
91 } else {
92 eden_allocate(obj, obj_end, tmp1, tmp2, size_expression, slow_case);
93 incr_allocated_bytes(size_expression, tmp1);
94 }
95 }
96
97
98 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register tmp) {
99 assert_different_registers(obj, klass, len, tmp);
100
101 if(UseBiasedLocking && !len->is_valid()) {
102 ldr(tmp, Address(klass, Klass::prototype_header_offset()));
103 } else {
104 mov(tmp, (intptr_t)markOopDesc::prototype());
105 }
106
107 #ifdef AARCH64
108 if (UseCompressedClassPointers) {
109 str(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
110 encode_klass_not_null(tmp, klass); // Take care not to kill klass
111 str_w(tmp, Address(obj, oopDesc::klass_offset_in_bytes()));
112 } else {
113 assert(oopDesc::mark_offset_in_bytes() + wordSize == oopDesc::klass_offset_in_bytes(), "adjust this code");
114 stp(tmp, klass, Address(obj, oopDesc::mark_offset_in_bytes()));
115 }
116 #else
117 str(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
118 str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
119 #endif // AARCH64
120
121 if (len->is_valid()) {
122 str_32(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
123 }
124 #ifdef AARCH64
125 else if (UseCompressedClassPointers) {
126 store_klass_gap(obj);
127 }
128 #endif // AARCH64
129 }
130
131
132 // Cleans object body [base..obj_end]. Clobbers `base` and `tmp` registers.
133 void C1_MacroAssembler::initialize_body(Register base, Register obj_end, Register tmp) {
134 zero_memory(base, obj_end, tmp);
135 }
136
137
138 void C1_MacroAssembler::initialize_object(Register obj, Register obj_end, Register klass,
139 Register len, Register tmp1, Register tmp2,
140 RegisterOrConstant header_size, int obj_size_in_bytes,
141 bool is_tlab_allocated)
142 {
143 assert_different_registers(obj, obj_end, klass, len, tmp1, tmp2);
144 initialize_header(obj, klass, len, tmp1);
145
146 const Register ptr = tmp2;
147
148 if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
149 #ifdef AARCH64
150 if (obj_size_in_bytes < 0) {
151 add_rc(ptr, obj, header_size);
152 initialize_body(ptr, obj_end, tmp1);
153
154 } else {
155 int base = instanceOopDesc::header_size() * HeapWordSize;
156 assert(obj_size_in_bytes >= base, "should be");
157
158 const int zero_bytes = obj_size_in_bytes - base;
159 assert((zero_bytes % wordSize) == 0, "should be");
160
161 if ((zero_bytes % (2*wordSize)) != 0) {
162 str(ZR, Address(obj, base));
163 base += wordSize;
164 }
165
166 const int stp_count = zero_bytes / (2*wordSize);
167
168 if (zero_bytes > 8 * wordSize) {
169 Label loop;
170 add(ptr, obj, base);
171 mov(tmp1, stp_count);
172 bind(loop);
173 subs(tmp1, tmp1, 1);
174 stp(ZR, ZR, Address(ptr, 2*wordSize, post_indexed));
175 b(loop, gt);
176 } else {
177 for (int i = 0; i < stp_count; i++) {
178 stp(ZR, ZR, Address(obj, base + i * 2 * wordSize));
179 }
180 }
181 }
182 #else
183 if (obj_size_in_bytes >= 0 && obj_size_in_bytes <= 8 * BytesPerWord) {
184 mov(tmp1, 0);
185 const int base = instanceOopDesc::header_size() * HeapWordSize;
186 for (int i = base; i < obj_size_in_bytes; i += wordSize) {
187 str(tmp1, Address(obj, i));
188 }
189 } else {
190 assert(header_size.is_constant() || header_size.as_register() == ptr, "code assumption");
191 add(ptr, obj, header_size);
192 initialize_body(ptr, obj_end, tmp1);
193 }
194 #endif // AARCH64
195 }
196
197 // StoreStore barrier required after complete initialization
198 // (headers + content zeroing), before the object may escape.
199 membar(MacroAssembler::StoreStore, tmp1);
200 }
201
202 void C1_MacroAssembler::allocate_object(Register obj, Register tmp1, Register tmp2, Register tmp3,
203 int header_size, int object_size,
204 Register klass, Label& slow_case) {
205 assert_different_registers(obj, tmp1, tmp2, tmp3, klass, Rtemp);
206 assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
207 const int object_size_in_bytes = object_size * BytesPerWord;
208
209 const Register obj_end = tmp1;
210 const Register len = noreg;
211
212 if (Assembler::is_arith_imm_in_range(object_size_in_bytes)) {
213 try_allocate(obj, obj_end, tmp2, tmp3, object_size_in_bytes, slow_case);
214 } else {
215 // Rtemp should be free at c1 LIR level
216 mov_slow(Rtemp, object_size_in_bytes);
217 try_allocate(obj, obj_end, tmp2, tmp3, Rtemp, slow_case);
218 }
219 initialize_object(obj, obj_end, klass, len, tmp2, tmp3, instanceOopDesc::header_size() * HeapWordSize, object_size_in_bytes, /* is_tlab_allocated */ UseTLAB);
220 }
221
222 void C1_MacroAssembler::allocate_array(Register obj, Register len,
223 Register tmp1, Register tmp2, Register tmp3,
224 int header_size, int element_size,
225 Register klass, Label& slow_case) {
226 assert_different_registers(obj, len, tmp1, tmp2, tmp3, klass, Rtemp);
227 const int header_size_in_bytes = header_size * BytesPerWord;
228 const int scale_shift = exact_log2(element_size);
229 const Register obj_size = Rtemp; // Rtemp should be free at c1 LIR level
230
231 #ifdef AARCH64
232 mov_slow(Rtemp, max_array_allocation_length);
233 cmp_32(len, Rtemp);
234 #else
235 cmp_32(len, max_array_allocation_length);
236 #endif // AARCH64
237 b(slow_case, hs);
238
239 bool align_header = ((header_size_in_bytes | element_size) & MinObjAlignmentInBytesMask) != 0;
240 assert(align_header || ((header_size_in_bytes & MinObjAlignmentInBytesMask) == 0), "must be");
241 assert(align_header || ((element_size & MinObjAlignmentInBytesMask) == 0), "must be");
242
243 mov(obj_size, header_size_in_bytes + (align_header ? (MinObjAlignmentInBytes - 1) : 0));
244 add_ptr_scaled_int32(obj_size, obj_size, len, scale_shift);
245
246 if (align_header) {
247 align_reg(obj_size, obj_size, MinObjAlignmentInBytes);
248 }
249
250 try_allocate(obj, tmp1, tmp2, tmp3, obj_size, slow_case);
251 initialize_object(obj, tmp1, klass, len, tmp2, tmp3, header_size_in_bytes, -1, /* is_tlab_allocated */ UseTLAB);
252 }
253
254 int C1_MacroAssembler::lock_object(Register hdr, Register obj,
255 Register disp_hdr, Register tmp1,
256 Label& slow_case) {
257 Label done, fast_lock, fast_lock_done;
258 int null_check_offset = 0;
259
260 const Register tmp2 = Rtemp; // Rtemp should be free at c1 LIR level
261 assert_different_registers(hdr, obj, disp_hdr, tmp1, tmp2);
262
263 assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code");
264 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
265 const int mark_offset = BasicLock::displaced_header_offset_in_bytes();
266
267 if (UseBiasedLocking) {
268 // load object
269 str(obj, Address(disp_hdr, obj_offset));
270 null_check_offset = biased_locking_enter(obj, hdr/*scratched*/, tmp1, false, tmp2, done, slow_case);
271 }
272
273 assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
274
275 #ifdef AARCH64
276
277 str(obj, Address(disp_hdr, obj_offset));
278
279 if (!UseBiasedLocking) {
280 null_check_offset = offset();
281 }
282 ldr(hdr, obj);
283
284 // Test if object is already locked
285 assert(markOopDesc::unlocked_value == 1, "adjust this code");
286 tbnz(hdr, exact_log2(markOopDesc::unlocked_value), fast_lock);
287
288 // Check for recursive locking
289 // See comments in InterpreterMacroAssembler::lock_object for
290 // explanations on the fast recursive locking check.
291 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
292 Assembler::LogicalImmediate imm(mask, false);
293 mov(tmp2, SP);
294 sub(tmp2, hdr, tmp2);
295 ands(tmp2, tmp2, imm);
296 b(slow_case, ne);
297
298 // Recursive locking: store 0 into a lock record
299 str(ZR, Address(disp_hdr, mark_offset));
300 b(fast_lock_done);
301
302 #else // AARCH64
303
304 if (!UseBiasedLocking) {
305 null_check_offset = offset();
306 }
307
308 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
309 // That would be acceptable as ether CAS or slow case path is taken in that case.
310
311 // Must be the first instruction here, because implicit null check relies on it
312 ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
313
314 str(obj, Address(disp_hdr, obj_offset));
315 tst(hdr, markOopDesc::unlocked_value);
316 b(fast_lock, ne);
317
318 // Check for recursive locking
319 // See comments in InterpreterMacroAssembler::lock_object for
320 // explanations on the fast recursive locking check.
321 // -1- test low 2 bits
322 movs(tmp2, AsmOperand(hdr, lsl, 30));
323 // -2- test (hdr - SP) if the low two bits are 0
324 sub(tmp2, hdr, SP, eq);
325 movs(tmp2, AsmOperand(tmp2, lsr, exact_log2(os::vm_page_size())), eq);
326 // If 'eq' then OK for recursive fast locking: store 0 into a lock record.
327 str(tmp2, Address(disp_hdr, mark_offset), eq);
328 b(fast_lock_done, eq);
329 // else need slow case
330 b(slow_case);
331
332 #endif // AARCH64
333
334 bind(fast_lock);
335 // Save previous object header in BasicLock structure and update the header
336 str(hdr, Address(disp_hdr, mark_offset));
337
338 cas_for_lock_acquire(hdr, disp_hdr, obj, tmp2, slow_case);
339
340 bind(fast_lock_done);
341
342 #ifndef PRODUCT
343 if (PrintBiasedLockingStatistics) {
344 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
345 }
346 #endif // !PRODUCT
347
348 bind(done);
349
350 return null_check_offset;
351 }
352
353 void C1_MacroAssembler::unlock_object(Register hdr, Register obj,
354 Register disp_hdr, Register tmp,
355 Label& slow_case) {
356 // Note: this method is not using its 'tmp' argument
357
358 assert_different_registers(hdr, obj, disp_hdr, Rtemp);
359 Register tmp2 = Rtemp;
360
361 assert(BasicObjectLock::lock_offset_in_bytes() == 0, "ajust this code");
362 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
363 const int mark_offset = BasicLock::displaced_header_offset_in_bytes();
364
365 Label done;
366 if (UseBiasedLocking) {
367 // load object
368 ldr(obj, Address(disp_hdr, obj_offset));
369 biased_locking_exit(obj, hdr, done);
370 }
371
372 assert(oopDesc::mark_offset_in_bytes() == 0, "Required by atomic instructions");
373 Label retry;
374
375 // Load displaced header and object from the lock
376 ldr(hdr, Address(disp_hdr, mark_offset));
377 // If hdr is NULL, we've got recursive locking and there's nothing more to do
378 cbz(hdr, done);
379
380 if(!UseBiasedLocking) {
381 // load object
382 ldr(obj, Address(disp_hdr, obj_offset));
383 }
384
385 // Restore the object header
386 cas_for_lock_release(disp_hdr, hdr, obj, tmp2, slow_case);
387
388 bind(done);
389 }
390
391
392 #ifndef PRODUCT
393
394 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
395 if (!VerifyOops) return;
396 verify_oop_addr(Address(SP, stack_offset));
397 }
398
399 void C1_MacroAssembler::verify_not_null_oop(Register r) {
400 Label not_null;
401 cbnz(r, not_null);
402 stop("non-null oop required");
403 bind(not_null);
404 if (!VerifyOops) return;
405 verify_oop(r);
406 }
407
408 #endif // !PRODUCT