1 /*
2 * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * Copyright (c) 2015-2018, Azul Systems, Inc. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "c1/c1_MacroAssembler.hpp"
29 #include "c1/c1_Runtime1.hpp"
30 #include "classfile/systemDictionary.hpp"
31 #include "gc/shared/collectedHeap.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "oops/arrayOop.hpp"
34 #include "oops/markOop.hpp"
35 #include "runtime/basicLock.hpp"
36 #include "runtime/biasedLocking.hpp"
37 #include "runtime/os.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "runtime/stubRoutines.hpp"
40
41 void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
42 FloatRegister f0, FloatRegister f1,
43 Register result)
44 {
45 Label done;
46 if (is_float) {
47 vcmp_f32(f0, f1);
48 } else {
49 vcmp_f64(f0, f1);
50 }
51
52 get_fpsr();
53
54 mov(result, 0);
55 if (unordered_result < 0) {
56 // we want -1 for unordered or less than, 0 for equal and 1 for
57 // greater than.
58 mov(result, 1, NE); // Not equal or unordered
59 neg(result, result, LT); // Less than or unordered
60 } else {
61 // we want -1 for less than, 0 for equal and 1 for unordered or
62 // greater than.
63 mov(result, 1, NE); // Not equal or unordered
64 neg(result, result, LO); // Less than
65 }
66 }
67
68 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
69 const int aligned_mask = BytesPerWord -1;
70 const int hdr_offset = oopDesc::mark_offset_in_bytes();
71 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
72 Label done, fail;
73 int null_check_offset = -1;
74
75 verify_oop(obj);
76
77 // save object being locked into the BasicObjectLock
78 str(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
79
80 if (UseBiasedLocking) {
81 assert(scratch != noreg, "should have scratch register at this point");
82 null_check_offset = biased_locking_enter(obj, hdr, scratch, rscratch1, false, done, &slow_case);
83 } else {
84 null_check_offset = offset();
85 }
86
87 // Load object header
88 ldr(hdr, Address(obj, hdr_offset));
89 // and mark it as unlocked
90 orr(hdr, hdr, markOopDesc::unlocked_value);
91 // save unlocked object header into the displaced header location on the stack
92 str(hdr, Address(disp_hdr, 0));
93 // test if object header is still the same (i.e. unlocked), and if so, store the
94 // displaced header address in the object header - if it is not the same, get the
95 // object header instead
96 lea(rscratch2, Address(obj, hdr_offset));
97 cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
98 // if the object header was the same, we're done
99 // if the object header was not the same, it is now in the hdr register
100 // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
101 //
102 // 1) (hdr & aligned_mask) == 0
103 // 2) sp <= hdr
104 // 3) hdr <= sp + page_size
105 //
106 // these 3 tests can be done by evaluating the following expression:
107 //
108 // (hdr - sp) & (aligned_mask - page_size)
109 //
110 // assuming both the stack pointer and page_size have their least
111 // significant 2 bits cleared and page_size is a power of 2
112 mov(rscratch1, sp);
113 sub(hdr, hdr, rscratch1);
114 mov(rscratch2, aligned_mask - os::vm_page_size());
115 ands(hdr, hdr, rscratch2);
116 // for recursive locking, the result is zero => save it in the displaced header
117 // location (NULL in the displaced hdr location indicates recursive locking)
118 str(hdr, Address(disp_hdr, 0));
119 // otherwise we don't care about the result and handle locking via runtime call
120 cbnz(hdr, slow_case);
121 // done
122 bind(done);
123 if (PrintBiasedLockingStatistics) {
124 lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
125 addmw(Address(rscratch2, 0), 1, rscratch1);
126 }
127 return null_check_offset;
128 }
129
130
131 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
132 const int aligned_mask = BytesPerWord -1;
133 const int hdr_offset = oopDesc::mark_offset_in_bytes();
134 assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
135 Label done;
136
137 if (UseBiasedLocking) {
138 // load object
139 ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
140 biased_locking_exit(obj, hdr, done);
141 }
142
143 // load displaced header
144 ldr(hdr, Address(disp_hdr, 0));
145 // if the loaded hdr is NULL we had recursive locking
146 // if we had recursive locking, we are done
147 cbz(hdr, done);
148 if (!UseBiasedLocking) {
149 // load object
150 ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
151 }
152 verify_oop(obj);
153 // test if object header is pointing to the displaced header, and if so, restore
154 // the displaced header in the object - if the object header is not pointing to
155 // the displaced header, get the object header instead
156 // if the object header was not pointing to the displaced header,
157 // we do unlocking via runtime call
158 if (hdr_offset) {
159 lea(rscratch1, Address(obj, hdr_offset));
160 cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
161 } else {
162 cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
163 }
164 // done
165 bind(done);
166 }
167
168
169 // Defines obj, preserves var_size_in_bytes
170 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
171 if (UseTLAB) {
172 tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
173 } else {
174 eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
175 }
176 }
177
178 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
179 assert_different_registers(obj, klass, len);
180 if (UseBiasedLocking && !len->is_valid()) {
181 assert_different_registers(obj, klass, len, t1, t2);
182 ldr(t1, Address(klass, Klass::prototype_header_offset()));
183 } else {
184 // This assumes that all prototype bits fit in an int32_t
185 mov(t1, (int32_t)(intptr_t)markOopDesc::prototype());
186 }
187 str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
188 str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
189
190 if (len->is_valid()) {
191 str(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
192 }
193 }
194
195 // preserves obj, destroys len_in_bytes
196 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1) {
197 assert(hdr_size_in_bytes >= 0, "header size must be positive or 0");
198 Label done;
199
200 // len_in_bytes is positive and ptr sized
201 subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
202 b(done, Assembler::EQ);
203
204 // Preserve obj
205 if (hdr_size_in_bytes)
206 add(obj, obj, hdr_size_in_bytes);
207 zero_memory(obj, len_in_bytes, t1);
208 if (hdr_size_in_bytes)
209 sub(obj, obj, hdr_size_in_bytes);
210
211 bind(done);
212 }
213
214
215 void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) {
216 assert_different_registers(obj, t1, t2); // XXX really?
217 assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
218
219 try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case);
220
221 initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2, UseTLAB);
222 }
223
224 // This method clobbers t1, t2, and rscratch1 registers.
225 void C1_MacroAssembler::initialize_object(Register obj, Register klass,
226 Register var_size_in_bytes,
227 int con_size_in_bytes,
228 Register t1, Register t2,
229 bool is_tlab_allocated) {
230 assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
231 "con_size_in_bytes is not multiple of alignment");
232
233 const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
234
235 initialize_header(obj, klass, noreg, t1, t2);
236
237 if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
238 // Null out rest of allocated space
239 const Register index = t2;
240 const int threshold = 8 * BytesPerWord;
241 if (var_size_in_bytes != noreg) {
242 mov(index, var_size_in_bytes);
243 initialize_body(obj, index, hdr_size_in_bytes, t1);
244 } else if (con_size_in_bytes <= threshold) {
245 // Emit required number of str instructions (unroll loop completely)
246 mov(t1, 0);
247 for (int i = hdr_size_in_bytes; i < con_size_in_bytes; i += BytesPerWord) {
248 str(t1, Address(obj, i));
249 }
250 } else if (con_size_in_bytes > hdr_size_in_bytes) {
251 block_comment("zero memory");
252 // Use loop to null out fields
253 int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord;
254 mov(t1, 0);
255
256 const int unroll = 4; // Number of str instructions we'll unroll
257 mov(index, words / unroll);
258 int remainder = words % unroll;
259 lea(rscratch1, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord));
260
261 Label entry_point, loop;
262 b(entry_point);
263 bind(loop);
264 sub(index, index, 1);
265 for (int i = -unroll; i < 0; i++) {
266 if (-i == remainder) {
267 bind(entry_point);
268 }
269 str(t1, Address(rscratch1, i * BytesPerWord));
270 }
271 if (remainder == 0) {
272 bind(entry_point);
273 }
274 add(rscratch1, rscratch1, unroll * BytesPerWord);
275 cbnz(index, loop);
276 }
277 }
278
279 membar(StoreStore);
280
281 if (CURRENT_ENV->dtrace_alloc_probes()) {
282 assert(obj == r0, "must be");
283 far_call(RuntimeAddress(Runtime1::entry_for(
284 Runtime1::dtrace_object_alloc_id)));
285 }
286
287 verify_oop(obj);
288 }
289
290 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
291 assert_different_registers(obj, len, t1, t2, klass);
292
293 // determine alignment mask
294 assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
295
296 // check for negative or excessive length
297 mov(rscratch1, (int32_t)max_array_allocation_length);
298 cmp(len, rscratch1);
299 b(slow_case, Assembler::HS);
300
301 const Register arr_size = t2; // okay to be the same
302 // align object end
303 mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
304 add(arr_size, arr_size, len, Assembler::lsl(f));
305 mov(t1, ~MinObjAlignmentInBytesMask);
306 andr(arr_size, arr_size, t1);
307
308 try_allocate(obj, arr_size, 0, t1, t2, slow_case);
309
310 initialize_header(obj, klass, len, t1, t2);
311
312 // clear rest of allocated space
313 const Register len_zero = len;
314 initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
315
316 membar(StoreStore);
317
318 if (CURRENT_ENV->dtrace_alloc_probes()) {
319 assert(obj == r0, "must be");
320 far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
321 }
322
323 verify_oop(obj);
324 }
325
326
327 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
328 verify_oop(receiver);
329 // explicit NULL check not needed since load from [klass_offset] causes a trap
330 // check against inline cache
331 assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
332
333 cmp_klass(receiver, iCache, rscratch1);
334 }
335
336 void C1_MacroAssembler::build_frame(int frame_size_in_bytes,
337 int bang_size_in_bytes) {
338 assert(bang_size_in_bytes >= frame_size_in_bytes,
339 "stack bang size incorrect");
340
341 // If we have to make this method not-entrant, we'll overwrite its first
342 // instruction with a jump. For this action to be legal we must ensure that
343 // this first instruction is a B, BL, NOP, BKPT, or SVC. Make it a NOP
344 nop();
345
346 // Make sure there is enough stack space for this method's activation
347 generate_stack_overflow_check(bang_size_in_bytes);
348
349 // Push lr, rfp, and optionally update rfp. rfp points to the first stack
350 // word used by the new frame.
351
352 if (FrameAPCS) {
353 mov(rscratch2, sp);
354 stmdb(sp, RegSet::of(rfp, rscratch2, lr, r15_pc).bits());
355 add(rfp, sp, 3 * wordSize);
356 } else {
357 stmdb(sp, RegSet::of(rfp, lr).bits());
358 if (PreserveFramePointer) {
359 add(rfp, sp, BytesPerWord);
360 }
361 }
362
363 // Create frame. frame_size_in_bytes always comes from
364 // LIR_Assembler::initial_frame_size_in_bytes() method, and it already
365 // takes into account two stack words spent on saving lr and rfp.
366 decrement(sp, frame_size_in_bytes);
367 }
368
369 void C1_MacroAssembler::remove_frame(int frame_size_in_bytes) {
370 if (FrameAPCS) {
371 ldmea(rfp, RegSet::of(rfp, sp, lr).bits(), false/*wb*/);
372 } else {
373 // Remove frame. frame_size_in_bytes always comes from
374 // LIR_Assembler::initial_frame_size_in_bytes() method, and it already
375 // takes into account two stack words spent on saving lr and rfp.
376 increment(sp, frame_size_in_bytes);
377 // Pop rfp and lr
378 ldmia(sp, RegSet::of(rfp, lr).bits());
379 }
380 }
381
382 void C1_MacroAssembler::verified_entry() {
383 }
384
385 void C1_MacroAssembler::patchable_load(Register reg, address addr) {
386 nop();
387 membar(Assembler::LoadLoad);
388 far_load(reg, addr);
389 }
390
391 void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg) {
392 //// Not APCS
393 // - 1: link
394 // fp 0: return address
395 // + 1: argument with offset 0
396 // + 2: argument with offset 1
397 // + 3: ...
398 //// APCS
399 // - 3: link
400 // - 2: sp
401 // - 1: return address
402 // fp 0: pc
403 // + 1: argument with offset 0
404 // + 2: argument with offset 1
405 // + 3: ...
406
407 ldr(reg, Address(rfp, (offset_in_words + 1) * BytesPerWord));
408 }
409
410 #ifndef PRODUCT
411
412 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
413 if (!VerifyOops) return;
414 verify_oop_addr(Address(sp, stack_offset), "oop");
415 }
416
417 void C1_MacroAssembler::verify_not_null_oop(Register r) {
418 if (!VerifyOops) return;
419 Label not_null;
420 cbnz(r, not_null);
421 stop("non-null oop required");
422 bind(not_null);
423 verify_oop(r);
424 }
425
426 void C1_MacroAssembler::invalidate_registers(bool inv_r0, bool inv_r2, bool inv_r3) {
427 #ifdef ASSERT
428 static int nn;
429 if (inv_r0) mov(r0, 0xDEAD);
430 if (inv_r2) mov(r2, nn++);
431 if (inv_r3) mov(r3, 0xDEAD);
432 #endif
433 }
434 #endif // ifndef PRODUCT