1 /*
   2  * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shared/barrierSetAssembler.hpp"
  29 #include "gc/shared/barrierSetNMethod.hpp"
  30 #include "gc/shared/barrierSetRuntime.hpp"
  31 #include "gc/shared/collectedHeap.hpp"
  32 #include "interpreter/interp_masm.hpp"
  33 #include "memory/universe.hpp"
  34 #include "runtime/jniHandles.hpp"
  35 #include "runtime/sharedRuntime.hpp"
  36 #include "runtime/thread.hpp"
  37 
  38 #define __ masm->
  39 
  40 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
  41                                   Register dst, Address src, Register tmp1, Register tmp_thread) {
  42   bool in_heap = (decorators & IN_HEAP) != 0;
  43   bool in_native = (decorators & IN_NATIVE) != 0;
  44   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
  45   bool atomic = (decorators & MO_RELAXED) != 0;
  46 
  47   assert(type != T_VALUETYPE, "Not supported yet");
  48   switch (type) {
  49   case T_OBJECT:
  50   case T_ARRAY: {
  51     if (in_heap) {
  52 #ifdef _LP64
  53       if (UseCompressedOops) {
  54         __ movl(dst, src);
  55         if (is_not_null) {
  56           __ decode_heap_oop_not_null(dst);
  57         } else {
  58           __ decode_heap_oop(dst);
  59         }
  60       } else
  61 #endif
  62       {
  63         __ movptr(dst, src);
  64       }
  65     } else {
  66       assert(in_native, "why else?");
  67       __ movptr(dst, src);
  68     }
  69     break;
  70   }
  71   case T_BOOLEAN: __ load_unsigned_byte(dst, src);  break;
  72   case T_BYTE:    __ load_signed_byte(dst, src);    break;
  73   case T_CHAR:    __ load_unsigned_short(dst, src); break;
  74   case T_SHORT:   __ load_signed_short(dst, src);   break;
  75   case T_INT:     __ movl  (dst, src);              break;
  76   case T_ADDRESS: __ movptr(dst, src);              break;
  77   case T_FLOAT:
  78     assert(dst == noreg, "only to ftos");
  79     __ load_float(src);
  80     break;
  81   case T_DOUBLE:
  82     assert(dst == noreg, "only to dtos");
  83     __ load_double(src);
  84     break;
  85   case T_LONG:
  86     assert(dst == noreg, "only to ltos");
  87 #ifdef _LP64
  88     __ movq(rax, src);
  89 #else
  90     if (atomic) {
  91       __ fild_d(src);               // Must load atomically
  92       __ subptr(rsp,2*wordSize);    // Make space for store
  93       __ fistp_d(Address(rsp,0));
  94       __ pop(rax);
  95       __ pop(rdx);
  96     } else {
  97       __ movl(rax, src);
  98       __ movl(rdx, src.plus_disp(wordSize));
  99     }
 100 #endif
 101     break;
 102   default: Unimplemented();
 103   }
 104 }
 105 
 106 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 107                                    Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
 108   bool in_heap = (decorators & IN_HEAP) != 0;
 109   bool in_native = (decorators & IN_NATIVE) != 0;
 110   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 111   bool atomic = (decorators & MO_RELAXED) != 0;
 112 
 113   assert(type != T_VALUETYPE, "Not supported yet");
 114   switch (type) {
 115   case T_OBJECT:
 116   case T_ARRAY: {
 117     if (in_heap) {
 118       if (val == noreg) {
 119         assert(!is_not_null, "inconsistent access");
 120 #ifdef _LP64
 121         if (UseCompressedOops) {
 122           __ movl(dst, (int32_t)NULL_WORD);
 123         } else {
 124           __ movslq(dst, (int32_t)NULL_WORD);
 125         }
 126 #else
 127         __ movl(dst, (int32_t)NULL_WORD);
 128 #endif
 129       } else {
 130 #ifdef _LP64
 131         if (UseCompressedOops) {
 132           assert(!dst.uses(val), "not enough registers");
 133           if (is_not_null) {
 134             __ encode_heap_oop_not_null(val);
 135           } else {
 136             __ encode_heap_oop(val);
 137           }
 138           __ movl(dst, val);
 139         } else
 140 #endif
 141         {
 142           __ movptr(dst, val);
 143         }
 144       }
 145     } else {
 146       assert(in_native, "why else?");
 147       assert(val != noreg, "not supported");
 148       __ movptr(dst, val);
 149     }
 150     break;
 151   }
 152   case T_BOOLEAN:
 153     __ andl(val, 0x1);  // boolean is true if LSB is 1
 154     __ movb(dst, val);
 155     break;
 156   case T_BYTE:
 157     __ movb(dst, val);
 158     break;
 159   case T_SHORT:
 160     __ movw(dst, val);
 161     break;
 162   case T_CHAR:
 163     __ movw(dst, val);
 164     break;
 165   case T_INT:
 166     __ movl(dst, val);
 167     break;
 168   case T_LONG:
 169     assert(val == noreg, "only tos");
 170 #ifdef _LP64
 171     __ movq(dst, rax);
 172 #else
 173     if (atomic) {
 174       __ push(rdx);
 175       __ push(rax);                 // Must update atomically with FIST
 176       __ fild_d(Address(rsp,0));    // So load into FPU register
 177       __ fistp_d(dst);              // and put into memory atomically
 178       __ addptr(rsp, 2*wordSize);
 179     } else {
 180       __ movptr(dst, rax);
 181       __ movptr(dst.plus_disp(wordSize), rdx);
 182     }
 183 #endif
 184     break;
 185   case T_FLOAT:
 186     assert(val == noreg, "only tos");
 187     __ store_float(dst);
 188     break;
 189   case T_DOUBLE:
 190     assert(val == noreg, "only tos");
 191     __ store_double(dst);
 192     break;
 193   case T_ADDRESS:
 194     __ movptr(dst, val);
 195     break;
 196   default: Unimplemented();
 197   }
 198 }
 199 
 200 void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators,
 201                                      Register src, Register dst, Register value_klass) {
 202   // value_copy implementation is fairly complex, and there are not any
 203   // "short-cuts" to be made from asm. What there is, appears to have the same
 204   // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
 205   // of hand-rolled instructions...
 206   if (decorators & IS_DEST_UNINITIALIZED) {
 207     __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass);
 208   } else {
 209     __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass);
 210   }
 211 }
 212 
 213 
 214 #ifndef _LP64
 215 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 216                                      Address obj1, jobject obj2) {
 217   __ cmpoop_raw(obj1, obj2);
 218 }
 219 
 220 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 221                                      Register obj1, jobject obj2) {
 222   __ cmpoop_raw(obj1, obj2);
 223 }
 224 #endif
 225 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 226                                      Register obj1, Address obj2) {
 227   __ cmpptr(obj1, obj2);
 228 }
 229 
 230 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 231                                      Register obj1, Register obj2) {
 232   __ cmpptr(obj1, obj2);
 233 }
 234 
 235 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 236                                                         Register obj, Register tmp, Label& slowpath) {
 237   __ clear_jweak_tag(obj);
 238   __ movptr(obj, Address(obj, 0));
 239 }
 240 
 241 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
 242                                         Register thread, Register obj,
 243                                         Register var_size_in_bytes,
 244                                         int con_size_in_bytes,
 245                                         Register t1,
 246                                         Register t2,
 247                                         Label& slow_case) {
 248   assert_different_registers(obj, t1, t2);
 249   assert_different_registers(obj, var_size_in_bytes, t1);
 250   Register end = t2;
 251   if (!thread->is_valid()) {
 252 #ifdef _LP64
 253     thread = r15_thread;
 254 #else
 255     assert(t1->is_valid(), "need temp reg");
 256     thread = t1;
 257     __ get_thread(thread);
 258 #endif
 259   }
 260 
 261   __ verify_tlab();
 262 
 263   __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
 264   if (var_size_in_bytes == noreg) {
 265     __ lea(end, Address(obj, con_size_in_bytes));
 266   } else {
 267     __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
 268   }
 269   __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
 270   __ jcc(Assembler::above, slow_case);
 271 
 272   // update the tlab top pointer
 273   __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
 274 
 275   // recover var_size_in_bytes if necessary
 276   if (var_size_in_bytes == end) {
 277     __ subptr(var_size_in_bytes, obj);
 278   }
 279   __ verify_tlab();
 280 }
 281 
 282 // Defines obj, preserves var_size_in_bytes
 283 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm,
 284                                         Register thread, Register obj,
 285                                         Register var_size_in_bytes,
 286                                         int con_size_in_bytes,
 287                                         Register t1,
 288                                         Label& slow_case) {
 289   assert(obj == rax, "obj must be in rax, for cmpxchg");
 290   assert_different_registers(obj, var_size_in_bytes, t1);
 291   if (!Universe::heap()->supports_inline_contig_alloc()) {
 292     __ jmp(slow_case);
 293   } else {
 294     Register end = t1;
 295     Label retry;
 296     __ bind(retry);
 297     ExternalAddress heap_top((address) Universe::heap()->top_addr());
 298     __ movptr(obj, heap_top);
 299     if (var_size_in_bytes == noreg) {
 300       __ lea(end, Address(obj, con_size_in_bytes));
 301     } else {
 302       __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
 303     }
 304     // if end < obj then we wrapped around => object too long => slow case
 305     __ cmpptr(end, obj);
 306     __ jcc(Assembler::below, slow_case);
 307     __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
 308     __ jcc(Assembler::above, slow_case);
 309     // Compare obj with the top addr, and if still equal, store the new top addr in
 310     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
 311     // it otherwise. Use lock prefix for atomicity on MPs.
 312     __ locked_cmpxchgptr(end, heap_top);
 313     __ jcc(Assembler::notEqual, retry);
 314     incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1);
 315   }
 316 }
 317 
 318 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread,
 319                                                Register var_size_in_bytes,
 320                                                int con_size_in_bytes,
 321                                                Register t1) {
 322   if (!thread->is_valid()) {
 323 #ifdef _LP64
 324     thread = r15_thread;
 325 #else
 326     assert(t1->is_valid(), "need temp reg");
 327     thread = t1;
 328     __ get_thread(thread);
 329 #endif
 330   }
 331 
 332 #ifdef _LP64
 333   if (var_size_in_bytes->is_valid()) {
 334     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
 335   } else {
 336     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
 337   }
 338 #else
 339   if (var_size_in_bytes->is_valid()) {
 340     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
 341   } else {
 342     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
 343   }
 344   __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
 345 #endif
 346 }
 347 
 348 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
 349   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
 350   if (bs_nm == NULL) {
 351     return;
 352   }
 353 #ifndef _LP64
 354   ShouldNotReachHere();
 355 #else
 356   Label continuation;
 357   Register thread = LP64_ONLY(r15_thread);
 358   Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset()));
 359   __ align(8);
 360   __ cmpl(disarmed_addr, 0);
 361   __ jcc(Assembler::equal, continuation);
 362   __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
 363   __ bind(continuation);
 364 #endif
 365 }
 366 
 367 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
 368   BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
 369   if (bs == NULL) {
 370     return;
 371   }
 372 
 373   Label bad_call;
 374   __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
 375   __ jcc(Assembler::equal, bad_call);
 376 
 377   // Pointer chase to the method holder to find out if the method is concurrently unloading.
 378   Label method_live;
 379   __ load_method_holder_cld(rscratch1, rbx);
 380 
 381   // Is it a strong CLD?
 382   __ movl(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset()));
 383   __ cmpptr(rscratch2, 0);
 384   __ jcc(Assembler::greater, method_live);
 385 
 386   // Is it a weak but alive CLD?
 387   __ movptr(rscratch1, Address(rscratch1, ClassLoaderData::holder_offset()));
 388   __ resolve_weak_handle(rscratch1, rscratch2);
 389   __ cmpptr(rscratch1, 0);
 390   __ jcc(Assembler::notEqual, method_live);
 391 
 392   __ bind(bad_call);
 393   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 394   __ bind(method_live);
 395 }