1 /*
  2  * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 #include "c1/c1_MacroAssembler.hpp"
 28 #include "c1/c1_Runtime1.hpp"
 29 #include "classfile/systemDictionary.hpp"
 30 #include "gc/shared/collectedHeap.hpp"
 31 #include "interpreter/interpreter.hpp"
 32 #include "oops/arrayOop.hpp"
 33 #include "oops/markOop.hpp"
 34 #include "runtime/basicLock.hpp"
 35 #include "runtime/biasedLocking.hpp"
 36 #include "runtime/os.hpp"
 37 #include "runtime/sharedRuntime.hpp"
 38 #include "runtime/stubRoutines.hpp"
 39 
 40 void C1_MacroAssembler::float_cmp(bool is_float, int unordered_result,
 41                                   FloatRegister f0, FloatRegister f1,
 42                                   Register result)
 43 {
 44   Label done;
 45   if (is_float) {
 46     fcmps(f0, f1);
 47   } else {
 48     fcmpd(f0, f1);
 49   }
 50   if (unordered_result < 0) {
 51     // we want -1 for unordered or less than, 0 for equal and 1 for
 52     // greater than.
 53     cset(result, NE);  // Not equal or unordered
 54     cneg(result, result, LT);  // Less than or unordered
 55   } else {
 56     // we want -1 for less than, 0 for equal and 1 for unordered or
 57     // greater than.
 58     cset(result, NE);  // Not equal or unordered
 59     cneg(result, result, LO);  // Less than
 60   }
 61 }
 62 
 63 int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register scratch, Label& slow_case) {
 64   const int aligned_mask = BytesPerWord -1;
 65   const int hdr_offset = oopDesc::mark_offset_in_bytes();
 66   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
 67   Label done, fail;
 68   int null_check_offset = -1;
 69 
 70   verify_oop(obj);
 71 
 72   // save object being locked into the BasicObjectLock
 73   str(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
 74 
 75   if (UseBiasedLocking) {
 76     assert(scratch != noreg, "should have scratch register at this point");
 77     null_check_offset = biased_locking_enter(disp_hdr, obj, hdr, scratch, false, done, &slow_case);
 78   } else {
 79     null_check_offset = offset();
 80   }
 81 
 82   // Load object header
 83   ldr(hdr, Address(obj, hdr_offset));
 84   // and mark it as unlocked
 85   orr(hdr, hdr, markOopDesc::unlocked_value);
 86   // save unlocked object header into the displaced header location on the stack
 87   str(hdr, Address(disp_hdr, 0));
 88   // test if object header is still the same (i.e. unlocked), and if so, store the
 89   // displaced header address in the object header - if it is not the same, get the
 90   // object header instead
 91   lea(rscratch2, Address(obj, hdr_offset));
 92   cmpxchgptr(hdr, disp_hdr, rscratch2, rscratch1, done, /*fallthough*/NULL);
 93   // if the object header was the same, we're done
 94   // if the object header was not the same, it is now in the hdr register
 95   // => test if it is a stack pointer into the same stack (recursive locking), i.e.:
 96   //
 97   // 1) (hdr & aligned_mask) == 0
 98   // 2) sp <= hdr
 99   // 3) hdr <= sp + page_size
100   //
101   // these 3 tests can be done by evaluating the following expression:
102   //
103   // (hdr - sp) & (aligned_mask - page_size)
104   //
105   // assuming both the stack pointer and page_size have their least
106   // significant 2 bits cleared and page_size is a power of 2
107   mov(rscratch1, sp);
108   sub(hdr, hdr, rscratch1);
109   ands(hdr, hdr, aligned_mask - os::vm_page_size());
110   // for recursive locking, the result is zero => save it in the displaced header
111   // location (NULL in the displaced hdr location indicates recursive locking)
112   str(hdr, Address(disp_hdr, 0));
113   // otherwise we don't care about the result and handle locking via runtime call
114   cbnz(hdr, slow_case);
115   // done
116   bind(done);
117   if (PrintBiasedLockingStatistics) {
118     lea(rscratch2, ExternalAddress((address)BiasedLocking::fast_path_entry_count_addr()));
119     addmw(Address(rscratch2, 0), 1, rscratch1);
120   }
121   return null_check_offset;
122 }
123 
124 
125 void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_hdr, Label& slow_case) {
126   const int aligned_mask = BytesPerWord -1;
127   const int hdr_offset = oopDesc::mark_offset_in_bytes();
128   assert(hdr != obj && hdr != disp_hdr && obj != disp_hdr, "registers must be different");
129   Label done;
130 
131   if (UseBiasedLocking) {
132     // load object
133     ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
134     biased_locking_exit(obj, hdr, done);
135   }
136 
137   // load displaced header
138   ldr(hdr, Address(disp_hdr, 0));
139   // if the loaded hdr is NULL we had recursive locking
140   // if we had recursive locking, we are done
141   cbz(hdr, done);
142   if (!UseBiasedLocking) {
143     // load object
144     ldr(obj, Address(disp_hdr, BasicObjectLock::obj_offset_in_bytes()));
145   }
146   verify_oop(obj);
147   // test if object header is pointing to the displaced header, and if so, restore
148   // the displaced header in the object - if the object header is not pointing to
149   // the displaced header, get the object header instead
150   // if the object header was not pointing to the displaced header,
151   // we do unlocking via runtime call
152   if (hdr_offset) {
153     lea(rscratch1, Address(obj, hdr_offset));
154     cmpxchgptr(disp_hdr, hdr, rscratch1, rscratch2, done, &slow_case);
155   } else {
156     cmpxchgptr(disp_hdr, hdr, obj, rscratch2, done, &slow_case);
157   }
158   // done
159   bind(done);
160 }
161 
162 
163 // Defines obj, preserves var_size_in_bytes
164 void C1_MacroAssembler::try_allocate(Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, Label& slow_case) {
165   if (UseTLAB) {
166     tlab_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
167   } else {
168     eden_allocate(obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
169     incr_allocated_bytes(noreg, var_size_in_bytes, con_size_in_bytes, t1);
170   }
171 }
172 
173 void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register len, Register t1, Register t2) {
174   assert_different_registers(obj, klass, len);
175   if (UseBiasedLocking && !len->is_valid()) {
176     assert_different_registers(obj, klass, len, t1, t2);
177     ldr(t1, Address(klass, Klass::prototype_header_offset()));
178   } else {
179     // This assumes that all prototype bits fit in an int32_t
180     mov(t1, (int32_t)(intptr_t)markOopDesc::prototype());
181   }
182   str(t1, Address(obj, oopDesc::mark_offset_in_bytes()));
183 
184   if (UseCompressedClassPointers) { // Take care not to kill klass
185     encode_klass_not_null(t1, klass);
186     strw(t1, Address(obj, oopDesc::klass_offset_in_bytes()));
187   } else {
188     str(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
189   }
190 
191   if (len->is_valid()) {
192     strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
193   } else if (UseCompressedClassPointers) {
194     store_klass_gap(obj, zr);
195   }
196 }
197 
198 // preserves obj, destroys len_in_bytes
199 void C1_MacroAssembler::initialize_body(Register obj, Register len_in_bytes, int hdr_size_in_bytes, Register t1) {
200   assert(hdr_size_in_bytes >= 0, "header size must be positive or 0");
201   Label done;
202 
203   // len_in_bytes is positive and ptr sized
204   subs(len_in_bytes, len_in_bytes, hdr_size_in_bytes);
205   br(Assembler::EQ, done);
206 
207   // Preserve obj
208   if (hdr_size_in_bytes)
209     add(obj, obj, hdr_size_in_bytes);
210   zero_memory(obj, len_in_bytes, t1);
211   if (hdr_size_in_bytes)
212     sub(obj, obj, hdr_size_in_bytes);
213 
214   bind(done);
215 }
216 
217 
218 void C1_MacroAssembler::allocate_object(Register obj, Register t1, Register t2, int header_size, int object_size, Register klass, Label& slow_case) {
219   assert_different_registers(obj, t1, t2); // XXX really?
220   assert(header_size >= 0 && object_size >= header_size, "illegal sizes");
221 
222   try_allocate(obj, noreg, object_size * BytesPerWord, t1, t2, slow_case);
223 
224   initialize_object(obj, klass, noreg, object_size * HeapWordSize, t1, t2, UseTLAB);
225 }
226 
227 void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register var_size_in_bytes, int con_size_in_bytes, Register t1, Register t2, bool is_tlab_allocated) {
228   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0,
229          "con_size_in_bytes is not multiple of alignment");
230   const int hdr_size_in_bytes = instanceOopDesc::header_size() * HeapWordSize;
231 
232   initialize_header(obj, klass, noreg, t1, t2);
233 
234   if (!(UseTLAB && ZeroTLAB && is_tlab_allocated)) {
235      // clear rest of allocated space
236      const Register index = t2;
237      const int threshold = 16 * BytesPerWord;   // approximate break even point for code size (see comments below)
238      if (var_size_in_bytes != noreg) {
239        mov(index, var_size_in_bytes);
240        initialize_body(obj, index, hdr_size_in_bytes, t1);
241      } else if (con_size_in_bytes <= threshold) {
242        // use explicit null stores
243        int i = hdr_size_in_bytes;
244        if (i < con_size_in_bytes && (con_size_in_bytes % (2 * BytesPerWord))) {
245          str(zr, Address(obj, i));
246          i += BytesPerWord;
247        }
248        for (; i < con_size_in_bytes; i += 2 * BytesPerWord)
249          stp(zr, zr, Address(obj, i));
250      } else if (con_size_in_bytes > hdr_size_in_bytes) {
251        block_comment("zero memory");
252       // use loop to null out the fields
253 
254        int words = (con_size_in_bytes - hdr_size_in_bytes) / BytesPerWord;
255        mov(index,  words / 8);
256 
257        const int unroll = 8; // Number of str(zr) instructions we'll unroll
258        int remainder = words % unroll;
259        lea(rscratch1, Address(obj, hdr_size_in_bytes + remainder * BytesPerWord));
260 
261        Label entry_point, loop;
262        b(entry_point);
263 
264        bind(loop);
265        sub(index, index, 1);
266        for (int i = -unroll; i < 0; i++) {
267          if (-i == remainder)
268            bind(entry_point);
269          str(zr, Address(rscratch1, i * wordSize));
270        }
271        if (remainder == 0)
272          bind(entry_point);
273        add(rscratch1, rscratch1, unroll * wordSize);
274        cbnz(index, loop);
275 
276      }
277   }
278 
279   membar(StoreStore);
280 
281   if (CURRENT_ENV->dtrace_alloc_probes()) {
282     assert(obj == r0, "must be");
283     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
284   }
285 
286   verify_oop(obj);
287 }
288 void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
289   assert_different_registers(obj, len, t1, t2, klass);
290 
291   // determine alignment mask
292   assert(!(BytesPerWord & 1), "must be a multiple of 2 for masking code to work");
293 
294   // check for negative or excessive length
295   mov(rscratch1, (int32_t)max_array_allocation_length);
296   cmp(len, rscratch1);
297   br(Assembler::HS, slow_case);
298 
299   const Register arr_size = t2; // okay to be the same
300   // align object end
301   mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
302   add(arr_size, arr_size, len, ext::uxtw, f);
303   andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
304 
305   try_allocate(obj, arr_size, 0, t1, t2, slow_case);
306 
307   initialize_header(obj, klass, len, t1, t2);
308 
309   // clear rest of allocated space
310   const Register len_zero = len;
311   initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
312 
313   membar(StoreStore);
314 
315   if (CURRENT_ENV->dtrace_alloc_probes()) {
316     assert(obj == r0, "must be");
317     far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::dtrace_object_alloc_id)));
318   }
319 
320   verify_oop(obj);
321 }
322 
323 
324 void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
325   verify_oop(receiver);
326   // explicit NULL check not needed since load from [klass_offset] causes a trap
327   // check against inline cache
328   assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
329 
330   cmp_klass(receiver, iCache, rscratch1);
331 }
332 
333 
334 void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
335   // If we have to make this method not-entrant we'll overwrite its
336   // first instruction with a jump.  For this action to be legal we
337   // must ensure that this first instruction is a B, BL, NOP, BKPT,
338   // SVC, HVC, or SMC.  Make it a NOP.
339   nop();
340   assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
341   // Make sure there is enough stack space for this method's activation.
342   // Note that we do this before doing an enter().
343   generate_stack_overflow_check(bang_size_in_bytes);
344   MacroAssembler::build_frame(framesize + 2 * wordSize);
345   if (NotifySimulator) {
346     notify(Assembler::method_entry);
347   }
348 }
349 
350 void C1_MacroAssembler::remove_frame(int framesize) {
351   MacroAssembler::remove_frame(framesize + 2 * wordSize);
352   if (NotifySimulator) {
353     notify(Assembler::method_reentry);
354   }
355 }
356 
357 
358 void C1_MacroAssembler::verified_entry() {
359 }
360 
361 #ifndef PRODUCT
362 
363 void C1_MacroAssembler::verify_stack_oop(int stack_offset) {
364   if (!VerifyOops) return;
365   verify_oop_addr(Address(sp, stack_offset), "oop");
366 }
367 
368 void C1_MacroAssembler::verify_not_null_oop(Register r) {
369   if (!VerifyOops) return;
370   Label not_null;
371   cbnz(r, not_null);
372   stop("non-null oop required");
373   bind(not_null);
374   verify_oop(r);
375 }
376 
377 void C1_MacroAssembler::invalidate_registers(bool inv_r0, bool inv_r19, bool inv_r2, bool inv_r3, bool inv_r4, bool inv_r5) {
378 #ifdef ASSERT
379   static int nn;
380   if (inv_r0) mov(r0, 0xDEAD);
381   if (inv_r19) mov(r19, 0xDEAD);
382   if (inv_r2) mov(r2, nn++);
383   if (inv_r3) mov(r3, 0xDEAD);
384   if (inv_r4) mov(r4, 0xDEAD);
385   if (inv_r5) mov(r5, 0xDEAD);
386 #endif
387 }
388 #endif // ifndef PRODUCT