1 /*
   2  * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shared/barrierSet.hpp"
  27 #include "gc/shared/barrierSetAssembler.hpp"
  28 #include "gc/shared/barrierSetNMethod.hpp"
  29 #include "gc/shared/collectedHeap.hpp"
  30 #include "interpreter/interp_masm.hpp"
  31 #include "memory/universe.hpp"
  32 #include "runtime/jniHandles.hpp"
  33 #include "runtime/sharedRuntime.hpp"
  34 #include "runtime/thread.hpp"
  35 
  36 #define __ masm->
  37 
  38 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
  39                                   Register dst, Address src, Register tmp1, Register tmp_thread) {
  40   bool in_heap = (decorators & IN_HEAP) != 0;
  41   bool in_native = (decorators & IN_NATIVE) != 0;
  42   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
  43   bool atomic = (decorators & MO_RELAXED) != 0;
  44 
  45   switch (type) {
  46   case T_OBJECT:
  47   case T_ARRAY: {
  48     if (in_heap) {
  49 #ifdef _LP64
  50       if (UseCompressedOops) {
  51         __ movl(dst, src);
  52         if (is_not_null) {
  53           __ decode_heap_oop_not_null(dst);
  54         } else {
  55           __ decode_heap_oop(dst);
  56         }
  57       } else
  58 #endif
  59       {
  60         __ movptr(dst, src);
  61       }
  62     } else {
  63       assert(in_native, "why else?");
  64       __ movptr(dst, src);
  65     }
  66     break;
  67   }
  68   case T_BOOLEAN: __ load_unsigned_byte(dst, src);  break;
  69   case T_BYTE:    __ load_signed_byte(dst, src);    break;
  70   case T_CHAR:    __ load_unsigned_short(dst, src); break;
  71   case T_SHORT:   __ load_signed_short(dst, src);   break;
  72   case T_INT:     __ movl  (dst, src);              break;
  73   case T_ADDRESS: __ movptr(dst, src);              break;
  74   case T_FLOAT:
  75     assert(dst == noreg, "only to ftos");
  76     __ load_float(src);
  77     break;
  78   case T_DOUBLE:
  79     assert(dst == noreg, "only to dtos");
  80     __ load_double(src);
  81     break;
  82   case T_LONG:
  83     assert(dst == noreg, "only to ltos");
  84 #ifdef _LP64
  85     __ movq(rax, src);
  86 #else
  87     if (atomic) {
  88       __ fild_d(src);               // Must load atomically
  89       __ subptr(rsp,2*wordSize);    // Make space for store
  90       __ fistp_d(Address(rsp,0));
  91       __ pop(rax);
  92       __ pop(rdx);
  93     } else {
  94       __ movl(rax, src);
  95       __ movl(rdx, src.plus_disp(wordSize));
  96     }
  97 #endif
  98     break;
  99   default: Unimplemented();
 100   }
 101 }
 102 
 103 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 104                                    Address dst, Register val, Register tmp1, Register tmp2) {
 105   bool in_heap = (decorators & IN_HEAP) != 0;
 106   bool in_native = (decorators & IN_NATIVE) != 0;
 107   bool is_not_null = (decorators & IS_NOT_NULL) != 0;
 108   bool atomic = (decorators & MO_RELAXED) != 0;
 109 
 110   switch (type) {
 111   case T_OBJECT:
 112   case T_ARRAY: {
 113     if (in_heap) {
 114       if (val == noreg) {
 115         assert(!is_not_null, "inconsistent access");
 116 #ifdef _LP64
 117         if (UseCompressedOops) {
 118           __ movl(dst, (int32_t)NULL_WORD);
 119         } else {
 120           __ movslq(dst, (int32_t)NULL_WORD);
 121         }
 122 #else
 123         __ movl(dst, (int32_t)NULL_WORD);
 124 #endif
 125       } else {
 126 #ifdef _LP64
 127         if (UseCompressedOops) {
 128           assert(!dst.uses(val), "not enough registers");
 129           if (is_not_null) {
 130             __ encode_heap_oop_not_null(val);
 131           } else {
 132             __ encode_heap_oop(val);
 133           }
 134           __ movl(dst, val);
 135         } else
 136 #endif
 137         {
 138           __ movptr(dst, val);
 139         }
 140       }
 141     } else {
 142       assert(in_native, "why else?");
 143       assert(val != noreg, "not supported");
 144       __ movptr(dst, val);
 145     }
 146     break;
 147   }
 148   case T_BOOLEAN:
 149     __ andl(val, 0x1);  // boolean is true if LSB is 1
 150     __ movb(dst, val);
 151     break;
 152   case T_BYTE:
 153     __ movb(dst, val);
 154     break;
 155   case T_SHORT:
 156     __ movw(dst, val);
 157     break;
 158   case T_CHAR:
 159     __ movw(dst, val);
 160     break;
 161   case T_INT:
 162     __ movl(dst, val);
 163     break;
 164   case T_LONG:
 165     assert(val == noreg, "only tos");
 166 #ifdef _LP64
 167     __ movq(dst, rax);
 168 #else
 169     if (atomic) {
 170       __ push(rdx);
 171       __ push(rax);                 // Must update atomically with FIST
 172       __ fild_d(Address(rsp,0));    // So load into FPU register
 173       __ fistp_d(dst);              // and put into memory atomically
 174       __ addptr(rsp, 2*wordSize);
 175     } else {
 176       __ movptr(dst, rax);
 177       __ movptr(dst.plus_disp(wordSize), rdx);
 178     }
 179 #endif
 180     break;
 181   case T_FLOAT:
 182     assert(val == noreg, "only tos");
 183     __ store_float(dst);
 184     break;
 185   case T_DOUBLE:
 186     assert(val == noreg, "only tos");
 187     __ store_double(dst);
 188     break;
 189   case T_ADDRESS:
 190     __ movptr(dst, val);
 191     break;
 192   default: Unimplemented();
 193   }
 194 }
 195 
 196 #ifndef _LP64
 197 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 198                                      Address obj1, jobject obj2) {
 199   __ cmpoop_raw(obj1, obj2);
 200 }
 201 
 202 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 203                                      Register obj1, jobject obj2) {
 204   __ cmpoop_raw(obj1, obj2);
 205 }
 206 #endif
 207 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 208                                      Register obj1, Address obj2) {
 209   __ cmpptr(obj1, obj2);
 210 }
 211 
 212 void BarrierSetAssembler::obj_equals(MacroAssembler* masm,
 213                                      Register obj1, Register obj2) {
 214   __ cmpptr(obj1, obj2);
 215 }
 216 
 217 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
 218                                                         Register obj, Register tmp, Label& slowpath) {
 219   __ clear_jweak_tag(obj);
 220   __ movptr(obj, Address(obj, 0));
 221 }
 222 
 223 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
 224                                         Register thread, Register obj,
 225                                         Register var_size_in_bytes,
 226                                         int con_size_in_bytes,
 227                                         Register t1,
 228                                         Register t2,
 229                                         Label& slow_case) {
 230   assert_different_registers(obj, t1, t2);
 231   assert_different_registers(obj, var_size_in_bytes, t1);
 232   Register end = t2;
 233   if (!thread->is_valid()) {
 234 #ifdef _LP64
 235     thread = r15_thread;
 236 #else
 237     assert(t1->is_valid(), "need temp reg");
 238     thread = t1;
 239     __ get_thread(thread);
 240 #endif
 241   }
 242 
 243   __ verify_tlab();
 244 
 245   __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
 246   if (var_size_in_bytes == noreg) {
 247     __ lea(end, Address(obj, con_size_in_bytes));
 248   } else {
 249     __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
 250   }
 251   __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
 252   __ jcc(Assembler::above, slow_case);
 253 
 254   // update the tlab top pointer
 255   __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
 256 
 257   // recover var_size_in_bytes if necessary
 258   if (var_size_in_bytes == end) {
 259     __ subptr(var_size_in_bytes, obj);
 260   }
 261   __ verify_tlab();
 262 }
 263 
 264 // Defines obj, preserves var_size_in_bytes
 265 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm,
 266                                         Register thread, Register obj,
 267                                         Register var_size_in_bytes,
 268                                         int con_size_in_bytes,
 269                                         Register t1,
 270                                         Label& slow_case) {
 271   assert(obj == rax, "obj must be in rax, for cmpxchg");
 272   assert_different_registers(obj, var_size_in_bytes, t1);
 273   if (!Universe::heap()->supports_inline_contig_alloc()) {
 274     __ jmp(slow_case);
 275   } else {
 276     Register end = t1;
 277     Label retry;
 278     __ bind(retry);
 279     ExternalAddress heap_top((address) Universe::heap()->top_addr());
 280     __ movptr(obj, heap_top);
 281     if (var_size_in_bytes == noreg) {
 282       __ lea(end, Address(obj, con_size_in_bytes));
 283     } else {
 284       __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
 285     }
 286     // if end < obj then we wrapped around => object too long => slow case
 287     __ cmpptr(end, obj);
 288     __ jcc(Assembler::below, slow_case);
 289     __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
 290     __ jcc(Assembler::above, slow_case);
 291     // Compare obj with the top addr, and if still equal, store the new top addr in
 292     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
 293     // it otherwise. Use lock prefix for atomicity on MPs.
 294     __ locked_cmpxchgptr(end, heap_top);
 295     __ jcc(Assembler::notEqual, retry);
 296     incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1);
 297   }
 298 }
 299 
 300 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread,
 301                                                Register var_size_in_bytes,
 302                                                int con_size_in_bytes,
 303                                                Register t1) {
 304   if (!thread->is_valid()) {
 305 #ifdef _LP64
 306     thread = r15_thread;
 307 #else
 308     assert(t1->is_valid(), "need temp reg");
 309     thread = t1;
 310     __ get_thread(thread);
 311 #endif
 312   }
 313 
 314 #ifdef _LP64
 315   if (var_size_in_bytes->is_valid()) {
 316     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
 317   } else {
 318     __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
 319   }
 320 #else
 321   if (var_size_in_bytes->is_valid()) {
 322     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
 323   } else {
 324     __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
 325   }
 326   __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
 327 #endif
 328 }
 329 
 330 #ifdef _LP64
 331 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
 332   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
 333   if (bs_nm == NULL) {
 334     return;
 335   }
 336   Label continuation;
 337   Register thread = r15_thread;
 338   Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset()));
 339   __ align(8);
 340   __ cmpl(disarmed_addr, 0);
 341   __ jcc(Assembler::equal, continuation);
 342   __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
 343   __ bind(continuation);
 344 }
 345 #else
 346 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) {
 347   BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
 348   if (bs_nm == NULL) {
 349     return;
 350   }
 351 
 352   Label continuation;
 353 
 354   Register tmp = rdi;
 355   __ push(tmp);
 356   __ movptr(tmp, (intptr_t)bs_nm->disarmed_value_address());
 357   Address disarmed_addr(tmp, 0);
 358   __ align(4);
 359   __ cmpl(disarmed_addr, 0);
 360   __ pop(tmp);
 361   __ jcc(Assembler::equal, continuation);
 362   __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier()));
 363   __ bind(continuation);
 364 }
 365 #endif
 366 
 367 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) {
 368   BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod();
 369   if (bs == NULL) {
 370     return;
 371   }
 372 
 373   Label bad_call;
 374   __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters.
 375   __ jcc(Assembler::equal, bad_call);
 376 
 377   // Pointer chase to the method holder to find out if the method is concurrently unloading.
 378   Label method_live;
 379   __ load_method_holder_cld(rscratch1, rbx);
 380 
 381   // Is it a strong CLD?
 382   __ movl(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset()));
 383   __ cmpptr(rscratch2, 0);
 384   __ jcc(Assembler::greater, method_live);
 385 
 386   // Is it a weak but alive CLD?
 387   __ movptr(rscratch1, Address(rscratch1, ClassLoaderData::holder_offset()));
 388   __ resolve_weak_handle(rscratch1, rscratch2);
 389   __ cmpptr(rscratch1, 0);
 390   __ jcc(Assembler::notEqual, method_live);
 391 
 392   __ bind(bad_call);
 393   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 394   __ bind(method_live);
 395 }